Mercurial > hg > cc > work

changeset 82:9fdeb9b3e58c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
taking stock
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 02 Apr 2025 16:48:27 +0100
parents b589ae992146
children d41cef3d4582
files lurid3/notes.txt
diffstat 1 files changed, 90 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/lurid3/notes.txt	Mon Mar 31 12:41:58 2025 +0100
+++ b/lurid3/notes.txt	Wed Apr 02 16:48:27 2025 +0100
@@ -2161,6 +2161,96 @@
   > /beegfs/common_crawl/CC-MAIN-2023-40/1695233511021.4/orig/warc/CC-MAIN-20231002200740-20231002230740-00825.warc.gz
   >: diff -bw <(fgrep beegfs rerun/4/w2c_errs)  <(fgrep beegfs surplus/4/w2c_errs)|tail -1
   > /beegfs/common_crawl/CC-MAIN-2023-40/1695233511021.4/orig/warc/CC-MAIN-20231002200740-20231002230740-00899.warc.gz
+
+Trashed the above rerun by mistake
+
+Successfully ran the balance anyway just to check...
+  >: time bash -c "python3 -c 'import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))'  2023-40 4 rerun '{825..899}' 2> rerun/4/a_w2c_errs
+  >: diff -bw <(fgrep beegfs rerun/4/a_w2c_errs)  <(fgrep beegfs surplus/4/w2c_errs)|less
+  0a1,825
+  > /beegfs/common_crawl/CC-MAIN-2023-40/1695233511021.4/orig/warc/CC-MAIN-20231002200740-20231002230740-00000.warc.gz
+
+Check 4:
+
+  >: cut -f 2 4/ks_errs.tsv | sus
+     4605 cannot unpack non-iterable NoneType object
+     1502 list index out of range
+	8 year 539500 is out of range
+	7 year 539498 is out of range
+	2 year 55722 is out of range
+	1 year 53337 is out of range
+	1 year 539499 is out of range
+	1 year 54618 is out of range
+	1 year 54752 is out of range
+  >: cut -f 2 <(sed 's/value or format .*/value or format/' surplus/4/w2c_errs)|fgrep -v warc.gz| sus
+     6107 Invalid date value or format
+	8 year 539500 is out of range
+	7 year 539498 is out of range
+	2 year 55722 is out of range
+	1 Growing buffer 2113089 > 2097152  [! from where???]
+	1 year 53337 is out of range
+	1 year 539499 is out of range
+	1 year 54618 is out of range
+	1 year 54752 is out of range
+  >: wc -l 4/ks_errs.tsv
+  6128 4/ks_errs.tsv
+  >: fgrep -v warc.gz surplus/4/w2c_errs |wc -l
+  6129 
+That's the 'Growing buffer ???'
+
+  >: wc -l surplus/4/lmh.cdb_in
+  6281718 surplus/4/lmh.cdb_in
+  >: seq 0 8 | parallel -j 9 'for i in {00..99}; do uz 4/*00{}$i.warc.gz | fgrep -c "     "; done | btot | tee /dev/stderr' | btot
+  697990
+  698141
+  697561
+  699427
+  698286
+  699690
+  699133
+  698055
+  699562
+  6287845
+  >: echo $((6287845 - 6281717))
+  6128
+OK
+
+Where do we stand?
+  >: ff w2c_errs | xargs -n 1 bash -c 'printf "%s\t%s\t%s\n" "$0" $(fgrep -c beegfs/ $0) "$(ls -l $0 | tr -s \  | cut -f 5,6,7,8 -d \  )"' |sort -k2nr,2 2>&1
+  ./0/w2c_errs    900     392410 Mar 24 16:52
+  ./15/w2c_errs   900     609663 Mar 19 22:56
+  ./1/w2c_errs    900     510538 Mar 25 19:02
+  ./surplus/4/w2c_errs    900     504252 Mar 27 02:12
+  ./rerun/4/w2c_errs      825     460572 Mar 26 18:22
+  ./rerun/6/w2c_errs      825     482598 Mar 26 18:22
+  ./rerun/5/w2c_errs      816     487423 Mar 26 18:22
+  ./rerun/7/w2c_errs      809     461695 Mar 26 18:22
+  ./rerun/3/w2c_errs      804     467730 Mar 26 18:22
+  ./4/w2c_errs    716     403837 Mar 20 03:33
+  ./6/w2c_errs    712     414801 Mar 20 03:33
+  ./2/w2c_errs    709     428774 Mar 20 03:33
+  ./7/w2c_errs    704     405495 Mar 20 03:33
+  ./5/w2c_errs    703     420718 Mar 20 03:33
+  ./3/w2c_errs    696     402308 Mar 20 03:33
+  ./rerun/2/w2c_errs      680     409265 Mar 26 18:22
+  ./surplus/2/w2c_errs    191     111807 Mar 24 15:16
+  ./8/w2c_errs    173     90504 Mar 20 01:09
+  ./11/w2c_errs   170     92570 Mar 20 01:09
+  ./9/w2c_errs    168     93293 Mar 20 01:09
+  ./12/w2c_errs   166     107497 Mar 20 01:09
+  ./13/w2c_errs   166     102016 Mar 20 01:09
+  ./14/w2c_errs   166     71934 Mar 20 01:09
+  ./10/w2c_errs   162     86168 Mar 20 01:09
+  ./rerun/9/w2c_errs      124     61805 Mar 26 15:01
+  ./rerun/11/w2c_errs     122     65295 Mar 26 15:01
+  ./rerun/12/w2c_errs     121     84178 Mar 26 15:01
+  ./rerun/13/w2c_errs     121     65592 Mar 26 15:01
+  ./rerun/14/w2c_errs     121     54378 Mar 26 15:01
+  ./rerun/10/w2c_errs     120     62900 Mar 26 15:01
+  ./rerun/8/w2c_errs      107     53748 Mar 26 15:01
+  ./rerun/4/a_w2c_errs    75      43572 Apr 2 12:48
+  ./surplus/2x/w2c_errs   8       5534 Mar 24 19:10
+
 ================
 
 Try it with the existing _per segment_ index we have for 2019-35