--- a/lurid3/notes.txt	Tue Mar 25 19:48:02 2025 +0000
+++ b/lurid3/notes.txt	Mon Mar 31 12:41:58 2025 +0100
@@ -1982,11 +1982,185 @@
 So 
   >: echo $((6418769 - 6414380))
   4389
+  >: fgrep -v warc.gz w2c_errs |wc -l
+  4389
 So lengths of lmh.cdb_in match, given that ks.tsv has string dates and
 lmh.cdb_in has dates successfully converted to numeric timestamps, and
 non-conversion counts match.
+
+Same for 15...
+  >: cut -f 2 ks_errs.tsv | sus
+     5420 cannot unpack non-iterable NoneType object
+     2670 list index out of range
+      113 year 641471 is out of range
+       20 year 641474 is out of range
+	9 hour must be in 0..23
+	9 year 642435 is out of range
+	3 year 642437 is out of range
+	1 year 4262575 is out of range
+	1 year 641769 is out of range
+	1 year 642434 is out of range
+  >: cut -f 2 <(sed 's/value or format .*/value or format/' w2c_errs)|fgrep -v warc.gz| sus
+     8090 Invalid date value or format
+      113 year 641471 is out of range
+       20 year 641474 is out of range
+	9 hour must be in 0..23
+	9 year 642435 is out of range
+	3 year 642437 is out of range
+	1 year 4262575 is out of range
+	1 year 641769 is out of range
+	1 year 642434 is out of range
+  >: wc -l ks_errs.tsv
+  8247 ks_errs.tsv
   >: fgrep -v warc.gz w2c_errs |wc -l
-  4389
+  8247
+  >: wc -l< lmh.cdb_in
+  5324617
+  >: seq 0 8 | parallel -j 9 'for i in {00..99}; do uz *00{}$i.warc.gz | fgrep -c "       "; done | btot | tee /dev/stderr' | btot
+  593009
+  591105
+  593544
+  591136
+  591189
+  594861
+  593214
+  592556
+  592249
+  5332863
+  >: echo $((5332863 - 5324616))
+  8247
+
+And for 1
+
+  >: seq 0 8 | parallel -j 9 'for i in {00..99}; do uz *00{}$i.warc.gz | fgrep -c "       "; done | btot | tee /dev/stderr' | btot
+  685262
+  686242
+  684822
+  684268
+  684967
+  684771
+  684520
+  685418
+  686420
+  6166690
+  >: wc -l < lmh.cdb_in
+  6160222
+  >: echo $((6166690 - 6160221))
+  6469
+  >: fgrep -v warc.gz w2c_errs |wc -l
+  6469
+
+And for 2?
+
+  >: wc -l {,surplus/}2/lmh.cdb_in
+    4670389 2/lmh.cdb_in
+      52530 surplus/2/lmh.cdb_in
+    4722919 total
+  sing<4564>: seq 0 8 | parallel -j 9 'for i in {00..99}; do uz 2/*00{}$i.warc.gz | fgrep -c "     "; done | btot | tee /dev/stderr' | btot
+  659931
+  658839
+  659020
+  660117
+  659896
+  659771
+  659374
+  660426
+  660040
+  5937414
+
+So surplus/2 is short?
+
+Maybe, but _2_ is short???
+
+  >: fgrep beegfs 2/w2c_errs |cut -f 8 -d / | tr -d \' | cut -f 5 -d - | cut -f 1 -d .|sort | parallel -j 9 'uz 2/*{}.warc.gz | fgrep -c "        " | tee /dev/stderr' | btot
+  4678728
+  >: echo $((4678728 - 4670388))
+  8340
+  >: fgrep -vh warc.gz 2/w2c_errs |wc -l
+  5130
+
+And so is 4???
+  >: fgrep beegfs 4/w2c_errs |cut -f 8 -d / | tr -d \' | cut -f 5 -d - | cut -f 1 -d .|wc -l
+  716
+  >: fgrep beegfs 4/w2c_errs |cut -f 8 -d / | tr -d \' | cut -f 5 -d - | cut -f 1 -d .|sort | parallel -j 9 'uz 4/*{}.warc.gz | fgrep -c "        "' |  btot
+  5003586
+  sing<4577>: fgrep -vh warc.gz 4/w2c_errs |wc -l
+  4890
+  sing<4578>: wc -l < 4/lmh.cdb_in
+  4975964
+  sing<4579>: echo $((5003586 - 4975963))
+  27623
+
+Mystery.  Just have to redo 2--14.
+Cirrus slurm doesn't allow forcing more than one node, e.g. with
+--nodes=2-7, says you should use --exclusive :-(
+
+So, just force it by hand
+
+  >: sbatch --output=slurm_w2c_2-7.out --time=04:00:00 --ntasks=7 -c 3 $HOME/bin/runme.sh $PWD -t 3 -i 'share_by_task.sh -s 2 7  $n $task' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh
+echo $PYTHONPATH
+mkdir -p rerun/$arg
+python3 -c "import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))"  2023-40 $arg ./rerun 2> rerun/$arg/w2c_errs'
+
+  >: sbatch --output=slurm_w2c_8-14.out --time=04:00:00 --ntasks=7 -c 3 $HOME/bin/runme.sh $PWD -t 3 -i 'share_by_task.sh -s 8 14  $n $task' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh
+echo $PYTHONPATH
+mkdir -p rerun/$arg
+python3 -c "import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))"  2023-40 $arg ./rerun 2> rerun/$arg/w2c_errs'
+
+  >: python3 -c "print((900.0/(626/6))*35)"
+  301.91693290734827
+
+Going too slowly, killed the 2nd one after 39 minutes with less than
+110 files processed by the first in 35 minutes, hope it might now just
+squeak in under 240 minutes ...
+
+  >: scancel --signal=HUP 7330184
+  >: python3 -c "print((900.0/(925/6))*45.5)"
+  265.62162162162167
+
+  >: python3 -c "print((900.0/(1058/6))*52.25)"
+  266.68241965973533
+
+Nope:
+  >: fgrep -ch beegfs rerun/{2..7}/w2c_errs
+  680
+  804
+  825
+  816
+  825
+  809
+  >: cd rerun
+  sing<4601>: for i in {2..7}; do fgrep beegfs $i/w2c_errs |cut -f 8 -d / | tr -d \' | cut -f 5 -d - | cut -f 1 -d .|sort | diff - <(seq -f '%05g' 0 899)|~/lib/python/cc/lmh/hack.py > $i.pat ; done
+  sing<4602>: less 2.pat
+  sing<4603>: fgrep -c warc.gz 4/w2c_errs
+  825
+  sing<4604>: cat 2.pat
+  {{680..899}}
+
+less than 10% speed-up after cancelling the 2nd job:
+  >: python3 -c "print((900.0/(4759/6))*240)"
+  272.32611893254887
+
+Rerun 4 separately to have one last go at debugging the problem with
+partial runs:
+
+  >: time python3 -c 'import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))'  2023-40 4 ./surplus 2> surplus/4/w2c_errs &
+  ./surplus/4/lmh.cdb_in
+
+  real    229m55.344s
+  user    164m5.765s
+  sys     11m24.657s
+
+Is 4/rerun/w2c_errs the same as surplus/4/w2c_errs through file 00824?
+Yes:
+  >: fgrep -c beegfs {rerun,surplus}/4/w2c_errs
+  rerun/4/w2c_errs:825
+  surplus/4/w2c_errs:900
+  >: diff -bw <(fgrep beegfs rerun/4/w2c_errs)  <(fgrep beegfs surplus/4/w2c_errs)|head -2
+  825a826,900
+  > /beegfs/common_crawl/CC-MAIN-2023-40/1695233511021.4/orig/warc/CC-MAIN-20231002200740-20231002230740-00825.warc.gz
+  >: diff -bw <(fgrep beegfs rerun/4/w2c_errs)  <(fgrep beegfs surplus/4/w2c_errs)|tail -1
+  > /beegfs/common_crawl/CC-MAIN-2023-40/1695233511021.4/orig/warc/CC-MAIN-20231002200740-20231002230740-00899.warc.gz
 ================
 
 Try it with the existing _per segment_ index we have for 2019-35