Mercurial > hg > cc > work

changeset 83:d41cef3d4582

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
new simpler warc.py did a whole segment, divided in 5 equal parts
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 09 Apr 2025 21:53:55 +0100
parents 9fdeb9b3e58c
children 5aa8b75aa9f0
files lurid3/notes.txt
diffstat 1 files changed, 148 insertions(+), 14 deletions(-) [+]
line wrap: on
line diff
--- a/lurid3/notes.txt	Wed Apr 02 16:48:27 2025 +0100
+++ b/lurid3/notes.txt	Wed Apr 09 21:53:55 2025 +0100
@@ -2226,21 +2226,21 @@
   ./rerun/5/w2c_errs      816     487423 Mar 26 18:22
   ./rerun/7/w2c_errs      809     461695 Mar 26 18:22
   ./rerun/3/w2c_errs      804     467730 Mar 26 18:22
-  ./4/w2c_errs    716     403837 Mar 20 03:33
-  ./6/w2c_errs    712     414801 Mar 20 03:33
-  ./2/w2c_errs    709     428774 Mar 20 03:33
-  ./7/w2c_errs    704     405495 Mar 20 03:33
-  ./5/w2c_errs    703     420718 Mar 20 03:33
-  ./3/w2c_errs    696     402308 Mar 20 03:33
   ./rerun/2/w2c_errs      680     409265 Mar 26 18:22
-  ./surplus/2/w2c_errs    191     111807 Mar 24 15:16
-  ./8/w2c_errs    173     90504 Mar 20 01:09
-  ./11/w2c_errs   170     92570 Mar 20 01:09
-  ./9/w2c_errs    168     93293 Mar 20 01:09
-  ./12/w2c_errs   166     107497 Mar 20 01:09
-  ./13/w2c_errs   166     102016 Mar 20 01:09
-  ./14/w2c_errs   166     71934 Mar 20 01:09
-  ./10/w2c_errs   162     86168 Mar 20 01:09
+      ./4/w2c_errs    716     403837 Mar 20 03:33
+      ./6/w2c_errs    712     414801 Mar 20 03:33
+      ./2/w2c_errs    709     428774 Mar 20 03:33
+      ./7/w2c_errs    704     405495 Mar 20 03:33
+      ./5/w2c_errs    703     420718 Mar 20 03:33
+      ./3/w2c_errs    696     402308 Mar 20 03:33
+    ./surplus/2/w2c_errs    191     111807 Mar 24 15:16
+    ./8/w2c_errs    173     90504 Mar 20 01:09
+    ./11/w2c_errs   170     92570 Mar 20 01:09
+    ./9/w2c_errs    168     93293 Mar 20 01:09
+    ./12/w2c_errs   166     107497 Mar 20 01:09
+    ./13/w2c_errs   166     102016 Mar 20 01:09
+    ./14/w2c_errs   166     71934 Mar 20 01:09
+    ./10/w2c_errs   162     86168 Mar 20 01:09
   ./rerun/9/w2c_errs      124     61805 Mar 26 15:01
   ./rerun/11/w2c_errs     122     65295 Mar 26 15:01
   ./rerun/12/w2c_errs     121     84178 Mar 26 15:01
@@ -2251,6 +2251,140 @@
   ./rerun/4/a_w2c_errs    75      43572 Apr 2 12:48
   ./surplus/2x/w2c_errs   8       5534 Mar 24 19:10
 
+Do the extras for 2,3,5,6,7
+
+  >: sbatch --output=slurm_w2c_plus.out --time=04:00:00 --ntasks=5 -c 2 $HOME/bin/runme.sh $PWD -t 2 -i 'echo 2 3 5 6 7 | xargs -n 1 echo|share_by_task.sh $n $task' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh
+echo $PYTHONPATH
+mkdir -p plus/$arg
+python3 -c "import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))"  2023-40 $arg ./plus "$(cat $arg.pat)" 2> plus/$arg/w2c_errs'
+  >: fgrep -c beegfs plus/*/w*
+  plus/2/w2c_errs:220
+  plus/3/w2c_errs:96
+  plus/5/w2c_errs:84
+  plus/6/w2c_errs:75
+  plus/7/w2c_errs:91
+  >: ls -lt plus/*/w*
+  -rw-r--r-- 1 hst dc007 131136 Apr  3 12:06 plus/2/w2c_errs
+  -rw-r--r-- 1 hst dc007  52260 Apr  3 11:39 plus/3/w2c_errs
+  -rw-r--r-- 1 hst dc007  54141 Apr  3 11:36 plus/7/w2c_errs
+  -rw-r--r-- 1 hst dc007  49769 Apr  3 11:35 plus/5/w2c_errs
+  -rw-r--r-- 1 hst dc007  40819 Apr  3 11:33 plus/6/w2c_errs
+  (reverse-i-search)`': ^C
+  >: cat [23567].pat
+  {680..899}{804..899}{816..899}{825..899}{809..899}
+  >: head -1 slurm_w2c_plus.out
+  Thu Apr 3 11:12:52 BST 2025 ...
+  >: tail -1 slurm_w2c_plus.out
+  Thu Apr 3 12:07:02 BST 2025 Finished
+Looks good...
+
+Duration 54 minutes =~ 244 per hour for segment 2.  Interesting,
+because 3 different nodes were used...
+
+Launch another 5 full runs...
+  >: sbatch --output=slurm_w2c_8-12.out --time=05:00:00 --ntasks=5 -c 2 $HOME/bin/runme.sh $PWD -t 2 -i 'share_by_task.sh -s 8 12 $n $task' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh
+echo $PYTHONPATH
+mkdir -p $arg
+python3 -c "import sys,warc2cdb; sys.exit(warc2cdb.main(*sys.argv[1:]))"  2023-40 $arg . 2> $arg/w2c_errs'
+  >: head -1 slurm_w2c_8-12.out
+  Thu Apr 3 13:13:49 BST 2025 ...
+
+There's a bug in warc.py, if, approximately, the first line of a WARC
+record is the last line in the buffer, see
+  >: python3 test_warc.py 1 /beegfs/common_crawl/CC-MAIN-2023-40/1695233506559.11/orig/warc/CC-MAIN-20230924023050-20230924053050-00186.warc.gz 1 | egrep -ac '^WARC-Type: response'
+  Traceback (most recent call last):
+    File "/work/dc007/dc007/hst/lib/python/cc/test_warc.py", line 25, in <module>
+      warc.warc(sys.argv[1],showme,[b'response','warcinfo','request','metadata'],parts=int(sys.argv[2]),debug=debug)
+    File "warc.py", line 141, in warc.warc
+      raise ValueError("Done but need more! %s + %s > %s in %s"%(bp,
+  ValueError: Done but need more! 1050941 + 278 > 1050538 in /beegfs/common_crawl/CC-MAIN-2023-40/1695233506559.11/orig/warc/CC-MAIN-20230924023050-20230924053050-00186.warc.gz
+  38660
+
+After thrashing around for a few days trying to be clever (see
+abandoned revisions 287 and 288), try a bigger buffer and a trivial
+refill strategy: always have enough room for the largest possible
+record before trying to read one.
+
+Get some baseline timings (cythonized) before trying this:
+  >: time python3 test_warc.py 1 /beegfs/common_crawl/CC-MAIN-2023-40/1695233506559.11/orig/warc/CC-MAIN-20230924023050-20230924053050-00186.warc.gz 1 |wc -l
+  Traceback (most recent call last):
+  ...
+  1199124
+
+  real    0m14.601s
+  user    0m9.006s
+  sys     0m1.158s
+  >: time python3 test_warc.py 1 /beegfs/common_crawl/CC-MAIN-2023-40/1695233506559.11/orig/warc/CC-MAIN-20230924023050-20230924053050-00187.warc.gz 1 |wc -l
+  1192992
+
+  real    0m15.055s
+  user    0m9.140s
+  sys     0m1.207s
+  >: time python3 test_warc.py 1 /beegfs/common_crawl/CC-MAIN-2023-40/1695233506559.11/orig/warc/CC-MAIN-20230924023050-20230924053050-00187.warc.gz 1 >/dev/null
+
+  real    0m14.873s
+  user    0m9.358s
+  sys     0m1.060s
+
+Working, including with problem case, but no faster :-(
+
+  >: time python3 test_warc.py 1 /beegfs/common_crawl/CC-MAIN-2023-40/1695233506559.11/orig/warc/CC-MAIN-20230924023050-20230924053050-00186.warc.gz 1 | wc -l
+  115981 records, max record: 1075305, max header: 4726
+  1199124
+
+  real    0m15.324s
+  user    0m10.308s
+  sys     0m1.153s
+  >: time python3 test_warc.py 1 /beegfs/common_crawl/CC-MAIN-2023-40/1695233506559.11/orig/warc/CC-MAIN-20230924023050-20230924053050-00187.warc.gz 1 | wc -l
+  115381 records, max record: 1056240, max header: 3849
+  1192992
+
+  real    0m14.939s
+  user    0m10.399s
+  sys     0m1.135s
+
+Try to do 1 segment using multiple nodes:
+
+  >: cat 11n.pats
+  {000..179}
+  {180..359}
+  {360..539}
+  {540..719}
+  {720..899}
+  >: sbatch --output=slurm_w2c_11n.out --time=02:00:00 --ntasks=5 -c 2 $HOME/bin/runme.sh $PWD -d -t 2 -i 'cat 11n.pats |share_by_task.sh $n $task' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh
+mkdir -p ./x$task/11
+~/lib/python/cc/w2c.py 2023-40 11 ./x$task "$arg"  2> ./x$task/11/w2c_errs'
+
+Found one more bug, gave up on .., now seems to be working:
+  >: cat 11n.pats
+  000,179
+  180,359
+  360,539
+  540,719
+  720,899
+  >: head -1 slurm_*11n*
+  Wed Apr 9 20:40:09 BST 2025 Launching 3 nodes in /mnt/lustre/e1000/home/dc007/dc007/hst/results/CC-MAIN-2023-40/warc_lmhx for runme /work/dc007/dc007/hst/results/CC-MAIN-2023-40/warc_lmhx -t 2 -i cat 11n.pats |share_by_task.sh $n $task export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh
+  >: date
+  Wed Apr  9 20:50:13 BST 2025
+  cirrus-login1<6025>: fgrep -c beegfs x?/11/w*
+  x0/11/w2c_errs:31
+  x1/11/w2c_errs:31
+  x2/11/w2c_errs:31
+  x3/11/w2c_errs:33
+  x4/11/w2c_errs:33
+But not much faster, 4:40 expected runtime for a whole segment of 900
+files at that rate:
+  (/ 900.0 (* 32 6)) -> 4.69
+And so it proved:
+  >: fgrep -c beegfs x?/11/w*
+  x0/11/w2c_errs:180
+  x1/11/w2c_errs:180
+  x2/11/w2c_errs:180
+  x3/11/w2c_errs:180
+  x4/11/w2c_errs:180
+  >: tail -1 slurm_*11n*
+  Wed Apr 9 21:39:53 BST 2025 Finished
+Need some sanity checks before gearing up production...
 ================
 
 Try it with the existing _per segment_ index we have for 2019-35