--- a/lurid3/notes.txt	Mon Apr 14 15:22:32 2025 +0100
+++ b/lurid3/notes.txt	Fri Apr 18 19:18:52 2025 +0100
@@ -2743,6 +2743,184 @@
 
 47:46 elapsed, mid-day weekday, all on one node...
 Both cross-checks succeed
+
+Where do we stand _now_?
+  >: ff w2c_errs | xargs -n 1 bash -c 'printf "%s\t%s\t%s\n" "$0" $(fgrep -c beegfs/ $0) "$(ls -l $0 | tr -s \  | cut -f 5,6,7,8 -d \  )"' |fgrep Apr | sort -k5nr,5 2>&1
+  ./fixu/x0/0/w2c_errs    180     97561 Apr 14 15:02
+  ./fixu/x0/4/w2c_errs    180     119330 Apr 14 09:41
+  ./fixu/x1/0/w2c_errs    180     92775 Apr 14 15:03
+  ./fixu/x1/4/w2c_errs    180     121103 Apr 14 09:39
+  ./fixu/x2/0/w2c_errs    180     99136 Apr 14 15:02
+  ./fixu/x2/4/w2c_errs    180     122217 Apr 14 09:40
+  ./fixu/x3/0/w2c_errs    180     98527 Apr 14 15:03
+  ./fixu/x3/4/w2c_errs    180     117299 Apr 14 09:41
+  ./fixu/x4/0/w2c_errs    180     97644 Apr 14 15:02
+  ./fixu/x4/4/w2c_errs    180     120875 Apr 14 09:41
+  ./fixu/x0/15/w2c_errs   180     141002 Apr 13 15:52
+  ./fixu/x1/15/w2c_errs   180     142454 Apr 13 15:46
+  ./fixu/x2/15/w2c_errs   180     139796 Apr 13 15:46
+  ./fixu/x3/15/w2c_errs   180     142030 Apr 13 15:45
+  ./fixu/x4/15/w2c_errs   180     141123 Apr 13 15:45
+The above show 0, 4, 15 run in parts, confirmed to match 2024 results
+exactly except for extra log lines from warc and warc2cdb in w2c_errs.
+  ./0/w2c_errs    900     485975 Apr 11 03:24 ***
+  ./15/w2c_errs   900     706412 Apr 11 03:38 ***
+  ./1/w2c_errs    900     607072 Apr 11 03:30 ***
+  ./2/w2c_errs    900     637082 Apr 11 03:35 ***
+  ./4/w2c_errs    900     601209 Apr 11 03:31 ***
+Concatenating errs for 0,4,15 match except for warc max counts after the first
+180, which makes sense:
+  >: diff <(cat fixu/x{1,2,3,4,0}/0/w*) 0/w2c_errs|head
+  1361c1361
+  < 114802 records, max record: 1056834, max header: 4513
+  ---
+  > 114802 records, max record: 1075973, max header: 10302
+  1368c1368
+  < 115126 records, max record: 1058589, max header: 5466
+  ---
+  > 115126 records, max record: 1075973, max header: 10302
+  1377c1377
+  < 114040 records, max record: 1058589, max header: 7318
+  >: head -1360 0/w2c_errs |tail -4
+  b''     Invalid date value or format ""
+  115030 records, max record: 1075973, max header: 10302
+  7148 LM headers, 7146 win, 2 lose, 2 non-ASCII URIs
+  /beegfs/common_crawl/CC-MAIN-2023-40/1695233510671.0/orig/warc/CC-MAIN-20230930082033-20230930112033-00179.warc.gz
+However, these runs must have been done before the \u fix
+  >: diff <(cat fixu/x{1,2,3,4,0}/4/l*) 4/lmh.cdb_in|egrep -c '^[0-9].*a'
+  0
+  >: diff <(cat fixu/x{1,2,3,4,0}/4/l*) 4/lmh.cdb_in|egrep -c '^[0-9].*d'
+  4 [these are the blank lines at the end of the first 4 parts]
+  >: diff <(cat fixu/x{1,2,3,4,0}/4/l*) 4/lmh.cdb_in|egrep -c '^[0-9].*c'
+  1435
+  >: diff <(cat fixu/x{1,2,3,4,0}/4/l*) 4/lmh.cdb_in|egrep -c '^<'
+  1495
+  >: diff <(cat fixu/x{1,2,3,4,0}/4/l*) 4/lmh.cdb_in|egrep -c '^>'
+  1491
+  >: diff <(cat fixu/x{1,2,3,4,0}/4/l*) 4/lmh.cdb_in|egrep -ic '[\\]u'
+  1491
+  >: diff <(cat fixu/x{1,2,3,4,0}/4/l*) 4/lmh.cdb_in|egrep -ic '^<.*[\\]u'
+  1491
+Similarly for 0, 15
+  ./plus/11/w2c_errs      3       1295 Apr 10 15:36
+  ./x0/11/w2c_errs        180     119337 Apr 10 16:31
+  ./x1/11/w2c_errs        180     118047 Apr 10 16:31
+  ./x2/11/w2c_errs        180     109509 Apr 10 16:35
+  ./x3/11/w2c_errs        180     118879 Apr 10 16:27
+  ./x4/11/w2c_errs        180     116116 Apr 10 16:30
+That batch was done to test the fix to warc.py for the last buffer
+load being missed, but _not_ the \u fix:
+  >: egrep -ic '[\\]u' x?/11/lmh.cdb_in
+  x0/11/lmh.cdb_in:0
+  x1/11/lmh.cdb_in:0
+  x2/11/lmh.cdb_in:0
+  x3/11/lmh.cdb_in:0
+  x4/11/lmh.cdb_in:0
+So the runs marked *** above must have been done before that fix and are _not valid_.
+And everything else below here is too early to be any use
+  ./10/w2c_errs   900     473108 Apr 3 17:10
+  ./11/w2c_errs   188     103194 Apr 3 14:03
+  ./12/w2c_errs   900     605957 Apr 3 17:09
+  ./8/w2c_errs    900     468236 Apr 3 17:01
+  ./9/w2c_errs    900     453757 Apr 3 17:03
+  ./plus/2/w2c_errs       220     131136 Apr 3 12:06
+  ./plus/3/w2c_errs       96      52260 Apr 3 11:39
+  ./plus/5/w2c_errs       84      49769 Apr 3 11:35
+  ./plus/6/w2c_errs       75      40819 Apr 3 11:33
+  ./plus/7/w2c_errs       91      54141 Apr 3 11:36
+  ./rerun/4/a_w2c_errs    75      43572 Apr 2 12:48
+Next steps:
+ Check robots.txt for LM headers: Yes
+ Fetched robotstxt for 2019-35
+ Ran warc2cdb on 2019-35 robotstxt, crawldiagnostics (after fixing
+  bug) and warc:
+  >: time ~/lib/python/cc/w2c.py 2019-35 0 . crawldiagnostics 2> 0/crawldiagnostics/w2c_errs
+
+  ./0/crawldiagnostics/lmh.cdb_in
+  real    7m13.989s
+  user    6m24.400s
+  sys     0m15.013s
+
+  >: time ~/lib/python/cc/w2c.py 2019-35 0 . robotstxt 2> 0/robotstxt/w2c_errs
+  ./0/robotstxt/lmh.cdb_in
+  real    1m11.277s
+  user    0m57.372s
+  sys     0m3.942s
+
+  >: time ~/lib/python/cc/w2c.py 2019-35 0 ./x0 warc 000,185 2> x0/0/warc/w2c_errs &
+  >: less x0/0/warc/w2c_errs
+  >: time ~/lib/python/cc/w2c.py 2019-35 0 ./x1 warc 186,371 2> x1/0/warc/w2c_errs &
+  >: time ~/lib/python/cc/w2c.py 2019-35 0 ./x2 warc 372,559 2> x2/0/warc/w2c_errs &
+  ./x0/0/warc/lmh.cdb_in
+  real    54m59.239s
+  user    34m34.656s
+  sys     2m39.790s
+
+  ./x1/0/warc/lmh.cdb_in
+  real    55m15.233s
+  user    34m54.959s
+  sys     2m42.281s
+
+  ./x2/0/warc/lmh.cdb_in
+  real    55m48.226s
+  user    35m14.087s
+  sys     2m45.434s
+
+Cross-check warc output:
+  >: wc -l x?/0/warc/l*
+    1763032 x0/0/warc/lmh.cdb_in
+    1763752 x1/0/warc/lmh.cdb_in
+    1781459 x2/0/warc/lmh.cdb_in
+    5308243 total
+  >: wc -l 0/ks.tsv
+  5308240 0/ks.tsv
+
+  >: wc -l 0/ks_errs.tsv
+  21462 0/ks_errs.tsv
+  >: wc -l x?/0/warc/w*
+     7621 x0/0/warc/w2c_errs
+     7768 x1/0/warc/w2c_errs
+     7753 x2/0/warc/w2c_errs
+    23142 total
+  >: echo $(((23142 - 21462)))
+  1680
+  >: echo $((560*3))
+  1680
+  >: cut -f 2 0/ks_errs.tsv | sus
+    14749 cannot unpack non-iterable NoneType object
+     6604 list index out of range
+       26 year 641480 is out of range
+       24 year 641788 is out of range
+       22 year 641785 is out of range
+       21 year 641787 is out of range
+	5 hour must be in 0..23
+	4 year 641791 is out of range
+	3 year 641474 is out of range
+	2 year 641484 is out of range
+	1 year 640818 is out of range
+	1 year 641778 is out of range
+  >: echo $((14749 + 6604))
+  21353
+  >: cut -f 2 <(sed 's/value or format .*/value or format/' ./x?/0/warc/w2c_errs)|egrep -v 'warc.gz|LM headers|records,'| sus
+    21353 Invalid date value or format
+       26 year 641480 is out of range
+       24 year 641788 is out of range
+       22 year 641785 is out of range
+       21 year 641787 is out of range
+	5 hour must be in 0..23
+	4 year 641791 is out of range
+	3 year 641474 is out of range
+	2 year 641484 is out of range
+	1 year 640818 is out of range
+	1 year 641778 is out of range
+All good
+
+ Fetch c_d and, if needed r_t for 2023-40
+ Remove all w2c_errs and ldb.cdb_in from *** downwards
+ Run a parallel full segment job for 1 2 11 12
+ Do whatever checks are possible on the results
+ Fill in r_t? and c_d ldb.cdb_in for 0 1 2 4 11 12 15
+ Take stock: more 2023-40, or back to 2019-35 to add c_d (and r_t)?
 ================
 
 Try it with the existing _per segment_ index we have for 2019-35