--- a/lurid3/notes.txt	Wed Mar 19 21:48:44 2025 -0400
+++ b/lurid3/notes.txt	Tue Mar 25 19:48:02 2025 +0000
@@ -1852,6 +1852,141 @@
   Thu Mar 20 12:05:50 AM GMT 2025
 After an hour, we're only at about 160 per process, and the job only
 has 3:30 to run, so I'm going to kill 8-14.
+
+0-7 didn't finish:
+
+reran the who
+
+  >: time python3 -c 'import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))'  2023-40 0 . 2> 0/w2c_errsle of 0
+  ./0/lmh.cdb_in
+
+  real    224m28.784s
+  user    169m21.796s
+  sys     12m25.583s
+
+Update warc2cdb to accept a rangm patter, use it to try getting the
+rest of segment 1:
+
+  >: time python3 -c 'import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))'  2023-40 1 . '{685..899}' 2>> 1/w2c_errs &
+
+Oops, complete fail, forgot that the earlier version used glob.glob
+which does _not_ produce a sorted list of files, so need to refetch
+all of 1 as well:
+
+  >: time python3 -c 'import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))'  2023-40 1 . 2> 1/w2c_errs
+  ./1/lmh.cdb_in
+
+  real    210m56.444s
+  user    154m29.608s
+  sys     11m49.656s
+
+Here and subsequently, explicity sort the input file list.
+
+Compute the missing bits, use to fill the gaps:
+
+  >: fgrep beegfs 2/w2c_errs |cut -f 8 -d / | tr -d \' | cut -f 5 -d - | cut -f 1 -d .|sort | diff - <(seq -f '%05g' 0 899)|~/lib/python/cc/lmh/hack.py > /tmp/hst/2.pat
+
+  >: echo python3 -c 'import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))'  2023-40 2 surplus $(cat /tmp/hst/2.pat) 2> surplus/2/w2c_errs
+time python3 -c 'import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))' 2023-40 2 surplus '{017,024,027,031,{034..035},038,040,043,046,{051..052},059,061,064,084,092,103,105,{108..109},111,114,117,119,{122..123},135,137,139,144,{146..147},150,153,162,186,195,197,206,208,211,{224..225},227,243,251,253,{264..265},269,285,{292..293},296,304,{307..308},{316..317},328,337,341,344,{360..361},373,380,{386..387},392,397,406,{418..419},426,439,445,454,456,459,462,465,467,473,480,{482..483},{486..487},489,{491..492},499,501,506,512,521,523,528,540,{545..546},550,555,562,568,573,{577..578},584,586,588,593,598,{607..608},621,625,{628..629},637,639,{645..647},649,{656..657},659,662,666,669,671,673,{678..679},683,{688..689},{693..694},702,704,706,{709..710},714,730,{740..741},{745..747},{750..751},754,758,763,767,771,{785..787},791,{794..795},800,804,809,{824..825},831,833,{841..842},846,{854..855},862,864,{875..878},880,883,890,{894..896}}' 2> surplus/2/w2c_errs &
+  surplus/2/lmh.cdb_in
+
+  real    47m35.809s
+  user    35m10.748s
+  sys     2m54.211s
+
+  >: fgrep -h beegfs {,surplus/}2/w2c_errs | wc -l
+  900
+
+Checked that the last 8 of the ones that _were_ logged from the
+original aborted run of segment 2 are complete:
+
+  >: fgrep beegfs 2/w2c_errs |tail -8
+  /beegfs/common_crawl/CC-MAIN-2023-40/...-00030.warc.gz
+  /beegfs/common_crawl/CC-MAIN-2023-40/...-00756.warc.gz
+  /beegfs/common_crawl/CC-MAIN-2023-40/...-00249.warc.gz
+  /beegfs/common_crawl/CC-MAIN-2023-40/...-00301.warc.gz
+  /beegfs/common_crawl/CC-MAIN-2023-40/...-00158.warc.gz
+  /beegfs/common_crawl/CC-MAIN-2023-40/...-00009.warc.gz
+  /beegfs/common_crawl/CC-MAIN-2023-40/...-00294.warc.gz
+  /beegfs/common_crawl/CC-MAIN-2023-40/...-00576.warc.gz
+
+  >: python3 -c 'import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))' 2023-40 2 surplus '{030,756,249,301,158,009,294,576}' 2> surplus/2x/w2c_errs
+  >: fgrep -n beegfs 2/w2c_errs |tail -8
+  5765:/beegfs/common_crawl/CC-MAIN-2023-40/1695233506658.2/orig/warc/CC-MAIN-20230924155422-20230924185422-00030.warc.gz
+  ...
+  5821:/beegfs/common_crawl/CC-MAIN-2023-40/1695233506658.2/orig/warc/CC-MAIN-20230924155422-20230924185422-00576.warc.gz
+  >: wc -l 2/w2c_errs
+  5839
+  >: echo $((5839 - 5765))
+  74
+  sing<4424>: tail -75 2/w2c_errs|sort |diff -bw - <(sort surplus/2x/w2c_errs)
+  [nothing]
+
+Something is wrong?
+  >: wc -l 0/lmh.cdb_in
+  6414381 0/lmh.cdb_in
+  >: seq 0 8 | parallel -j 9 'for i in {00..99}; do uz 0/*00{}$i.warc.gz | egrep  -c "    .*      "; done | btot | tee /dev/stderr' | btot
+  713980
+  714754
+  712334
+  712640
+  713943
+  714063
+  712428
+  712630
+  711997
+  6418769
+  >: echo $((6418769 - 6414381))
+  4388
+
+  >: seq 0 8 | parallel -j 9 'for i in {00..99}; do uz 15/*00{}$i.warc.gz | egrep  -c "   .*      "; done | btot | tee /dev/stderr' | btot
+  591136
+  591189
+  591105
+  593544
+  593214
+  594861
+  592556
+  592249
+  593009
+  5332863
+  >: echo $((5332863 - 5324617))
+  8246
+
+Ah, maybe OK:
+  wc -l < 0/ks_errs.tsv
+  4389
+  >: cut -f 2 0/ks_errs.tsv | sus
+     3188 cannot unpack non-iterable NoneType object
+     1193 list index out of range
+	1 year 4263994 is out of range
+	1 year 4278897 is out of range
+	1 year 4355304 is out of range
+	1 year 4361928 is out of range
+	1 year 4370444 is out of range
+	1 year 4430056 is out of range
+	1 year 4433368 is out of range
+	1 year 53310 is out of range
+  >: cut -f 2 <(sed 's/value or format .*/value or format/' w2c_errs)|fgrep -v warc.gz| sus
+     4381 Invalid date value or format
+	1 year 4263994 is out of range
+	1 year 4278897 is out of range
+	1 year 4355304 is out of range
+	1 year 4361928 is out of range
+	1 year 4370444 is out of range
+	1 year 4430056 is out of range
+	1 year 4433368 is out of range
+	1 year 53310 is out of range
+
+And last line of lmh.cdb_in is a blank EOF indicator
+So 
+  >: echo $((6418769 - 6414380))
+  4389
+So lengths of lmh.cdb_in match, given that ks.tsv has string dates and
+lmh.cdb_in has dates successfully converted to numeric timestamps, and
+non-conversion counts match.
+  >: fgrep -v warc.gz w2c_errs |wc -l
+  4389
 ================
 
 Try it with the existing _per segment_ index we have for 2019-35