Mercurial > hg > cc > cirrus_work

--- a/bin/_runme.sh	Thu Mar 26 17:27:53 2026 +0000
+++ b/bin/_runme.sh	Tue Apr 07 15:04:09 2026 +0100
@@ -11,12 +11,12 @@
 node=$SLURM_NODEID
 task=$SLURM_PROCID

-# Following has been removed from service for now...
+# Not working
 if [ "$1" = "-a" ]
 then
  shift
  xarg="$1"
- xa="run $1 "
+ xa="job $1 "
  shift
 else
  xarg=
@@ -43,10 +43,10 @@

 pjobs=$((c / $threadsPerJob))

-echo $(date) ${xa}task $n.$task on $nodename:$N.$node start $pjobs jobs 1>&2
+echo $(date) ${xa}task $n.$task on $nodename:$N.$node start $pjobs 1>&2

 PATH=$W/$USER/bin:$W/shared/bin:$PATH
-export task PATH n node xarg
+export task PATH n node #xarg

 if [ "$1" = "-b" ]
 then
@@ -74,7 +74,7 @@
 doit () {
  arg="$1"
  i="$2"
- echo $(date) start $task $i "|$node|$arg|"
+ echo $(date) start $task $i "|$node|$arg|" # |$xarg
  eval "$cmd"
  echo $(date) end $task $i
 }
--- a/bin/cdbp.sh	Thu Mar 26 17:27:53 2026 +0000
+++ b/bin/cdbp.sh	Tue Apr 07 15:04:09 2026 +0100
@@ -1,13 +1,12 @@
 #!/usr/bin/bash
-# One cdb pipeline step, doing 1/17th of the work
+# One cdb pipeline step, doing 1/6 of the work
 # Usage: cdbp.sh i
-# where i is the 0-origin index of a 17th, so e.g. cdbp.sh 2 will run
-#   test_cdbp.py ... ks_%d-%d.cdb 2 3 12 18
+# where i is the 0-origin index of a 6th, so e.g. cdbp.sh 2 will run
+#   test_cdbp.py ... ks_%d-%d.cdb lmh_%d-%d.cdb 34 50
 i=$1
-k1=$((i >= 16 ?  ((i * 6) - 1) : (i * 6)))
-k2=$((i < 15 ? (i + 1) * 6 : 90 + (i - 14) * 5))
-#k2=$((k2 > 100 ? 100 : k2))
-echo $(printf lmh_%d-%d.cdb $k1 $((k2 - 1))) 1>&2
-cat $(printf lmh_%d-%d.cdb $k1 $((k2 - 1))) > /dev/null
+k1=$((i < 4 ?  (i * 17)  : (4 +(i * 16))))
+k2=$((i < 4 ? k1 + 16 : k1 + 15))
+#echo $(printf lmh_%d-%d.cdb $k1 $((k2 - 1))) 1>&2
+#cat $(printf lmh_%d-%d.cdb $k1 $((k2 - 1))) > /dev/null
 python3 -c 'import cdb_one;cdb_one.mainp()' lmh_%d-%d.cdb $k1 $k2
--- a/lib/python/cc/lmh/cdb_one.py	Thu Mar 26 17:27:53 2026 +0000
+++ b/lib/python/cc/lmh/cdb_one.py	Tue Apr 07 15:04:09 2026 +0100
@@ -1,14 +1,9 @@
 #!/usr/bin/python3
 # cython: profile=False, language_level=3str
 '''Lookup an original cdx entry and add a LastModified time if there is one.
-Usage: uz .../cdx-00nnn.gz | cdb_one.py cdbpat S E DP | igzip -c > cdx-00101.gz
+Usage: uz .../cdx-00nnn.gz | cdb_one.py cdbpat S E | igzip -c > cdx-00nnn.gz
 cdbpat identifies a shrunken cdb file, e.g. .../cdb/ks_%d-%d.cdb
 for segments in S..E.
-The input cdb entries have only a crawl date suffix wrt to a
- prefix with DP.  For example
-                       a value of 234
-         given a DP of 20230922103943
- gives a full value of 20230922103234
 '''

 import cython, typing, timeit, re, sys, array
@@ -16,9 +11,11 @@

 from db import CCdb

+SD: int = -1
+
 def mainp() -> None:
   #PAT: typing.Pattern[cython.bytes] = re.compile(b'\{"url": "([^"]*)",.*, "filename": ".*/segments/[0-9]*\.([0-9][0-9]?)/')
-  C_PAT: typing.Pattern[cython.bytes] = re.compile(b'[^ ]* ([^ ]*) .*{"url": "(http[^"]*).*"filename": "[^"]*\.([0-9][0-9]?)/(warc|robotstxt|crawldiagnostics)/')
+  C_PAT: typing.Pattern[cython.bytes] = re.compile(b'[^ ]* [^ ]* .*, "offset": "([0-9]*)", "filename": "[^"]*\.([0-9][0-9]?)/(warc|robotstxt|crawldiagnostics)/.*([0-9]{3,3})\.warc\.gz')

   N: int = 0
   M: int = 0
@@ -45,26 +42,24 @@
     wdate: cython.bytes
     kind: cython.bytes
     segb: cython.bytes
+    offsetb: cython.bytes
+    fnob: cython.bytes
     ts: array.array
     seg: int
     res: int
     i: int
     nd: cython.bytes
     if (m:=C_PAT.match(l)):
-      (wdate, uri, segb, kind) = m.groups()
+      (segb, kind, offsetb, fnob) = m.groups()
     else:
       raise ValueError(l)
     N += 1
     seg = int(segb)
     if (seg >= S and seg < E):
       M += 1
-      nd = shrink_key.shrink_date(wdate, DP)
-      if kind == 'robotstxt':
-        nd += seg
-      if not uri.startswith(b'http'):
-        raise ValueError(uri)
-      #print(nd,uri[4:],file=sys.stderr)
-      if (res := C.find(nd+uri[4:])) == 1:
+      shrink_key.set_sd_from_bytes(kind)
+      key = shrink_key.shrink_key(seg, int(fnob), int(offsetb))
+      if (res := C.find(key) == 1:
         hits += 1
         cdx_out.write(memoryview(l)[:-2])
         cdx_out.write(b', "lastmod": "')
--- a/lib/python/cc/lmh/shrink_key.py	Thu Mar 26 17:27:53 2026 +0000
+++ b/lib/python/cc/lmh/shrink_key.py	Tue Apr 07 15:04:09 2026 +0100
@@ -2,15 +2,18 @@
 # cython: profile=False, language_level=3str
 '''Construct a shrunken cdb key from an original cdx entry
    If run as a script, output this to stdout
-   Usage: uz .../cdx-00nnn.gz | ~/lib/python/cc/lmh/shrink_key.py DP
-   The output cdb entries have only a crawl date suffix wrt to a
-    prefix with DP.  For example
-         given a DP of 20230922103943
-         gives a shrunken value of 234
+   Usage: uz .../cdx-00nnn.gz | ~/lib/python/cc/lmh/shrink_key.py
+   The new version of this uses 4 items to construct a 7-byte key for a warc record:
+     segment: 1 byte
+     file number plus 1024 for a crawldiagnostics and
+                      2048 for a robotstxt:
+                      2 bytes
+     record offset: 4 bytes
+   and converts a last-modified unix-numeric datetime to a 4-byte integer
+
+WARNING: If a crawl ever has more than 1023 files per segment, this
+approach won't work any more'''
 '''
-
-
-
 import cython, typing, timeit, re, sys, array

 LAST_DATE: bytes = int.to_bytes(2147483647,4)
@@ -18,7 +21,25 @@

 C_PAT: typing.Pattern[bytes] = re.compile(b'[^ ]* ([^ ]*) .*{"url": "(http[^"]*).*"filename": "[^"]*[.]([0-9][0-9]?)/(warc|robotstxt|crawldiagnostics)/')

-def shrink_key(seg: int, fileno: int, offset: int, lm: int) -> tuple[bytes,bytes]:
+SD: int = -1
+WARC: int = 0
+ROBOTSTXT: int = 1
+CRAWLDIAGNOSTICS: int = 2
+SUBDIR_CODES={b'w':WARC,b'c':CRAWLDIAGNOSTICS,b'r':ROBOTSTXT}
+
+def set_sd_from_string(str: subdir):
+  global SD, SUBDIR_CODES
+  SD = (shrink_key.SUBDIR_CODES[b'w'] if subdir.startswith('w')
+                     else (shrink_key.SUBDIR_CODES[b'c'] if subdir.startswith('c')
+                           else shrink_key.SUBDIR_CODES(bytes(ord(subdir[0])))))
+
+def set_sd_from_bytes(bytes: subdir):
+  global SD, SUBDIR_CODES
+  SD = (shrink_key.SUBDIR_CODES[b'w'] if subdir.startswith(b'w')
+                     else (shrink_key.SUBDIR_CODES[b'c'] if subdir.startswith('bc')
+                           else shrink_key.SUBDIR_CODES(subdir[0])))
+
+def shrink_entry(seg: int, fileno: int, offset: int, lm: int) -> tuple[bytes,bytes]:
   lmb: bytes
   if lm >= 0:
     if lm <= 2147483647:
@@ -30,8 +51,12 @@
       lmb = FIRST_DATE
     else:
       lmb = int.to_bytes(lm, 4, signed=True)
+  return(shrink_key(seg, fileno, offset), lm)

-  return (int.to_bytes(seg,1)+int.to_bytes(fileno,2)+int.to_bytes(offset,4),lmb)
+def shrink_key(seg: int, fileno: int, offset: int) -> bytes
+  global SD
+
+  return (int.to_bytes(seg,1)+int.to_bytes(fileno+(SD << 10),2)+int.to_bytes(offset,4),lmb)

 def shrink_date(wdate: bytes, dp: bytes) -> bytes:
   dpl: int = len(dp)
--- a/lib/python/cc/lmh/warc2cdb.py	Thu Mar 26 17:27:53 2026 +0000
+++ b/lib/python/cc/lmh/warc2cdb.py	Tue Apr 07 15:04:09 2026 +0100
@@ -7,7 +7,7 @@
    warc_file_range is (literally) '???' or from,to'''

 import re, warc, sys
-import typing # cython
+import typing, cython
 import email.utils
 import subprocess
 import shrink_key
@@ -57,15 +57,9 @@
         print(dateTime.rstrip(),e,sep='\t',file=sys.stderr)
         LOSE += 1
         return
-      (key,val) = shrink_key.shrink_key(SEG, FILENO, OFFSET, lmi)
+      (key,val) = shrink_key.shrink_entry(SEG, FILENO, OFFSET, lmi)
       WIN += 1
-      vl: int = 4 # len(val)
-      kl: int = 9 # len(key)
-      OUT.write(b'+')
-      OUT.write(b'%d'%kl)
-      OUT.write(b',')
-      OUT.write(b'%d'%vl)
-      OUT.write(b':')
+      OUT.write(b'+7,4:')
       OUT.write(key)
       OUT.write(b'->')
       OUT.write(val)
@@ -74,6 +68,7 @@
 def main(CCdate: str, segment: str, outdir: str, subdir: str, fpat: str ):
   global OUT, N, WIN, LOSE, SEG
   global FNOPAT, FILENO
+  global SD, WARC, ROBOTSTXT, CRAWLDIAGNOSTICS

   m: typing.Match[bytes] | None
   infile_name: str
@@ -82,6 +77,16 @@
   if fpat != '???':
     fpat = ("{%s..%s}"%tuple(fpat.split(','))) if ',' in fpat else fpat
   infile_pat='bash -c "ls $CCC/CC-MAIN-%s/*.%s/orig/%s/*00%s.warc.gz | sort -k8"'%(CCdate, segment, subdir, fpat)
+
+  try:
+    shrink_key.SD = (shrink_key.SUBDIR_CODES[b'w'] if subdir.startswith('w')
+                     else (shrink_key.SUBDIR_CODES[b'c'] if subdir.startswith('c')
+                           else shrink_key.SUBDIR_CODES(bytes(ord(subdir[0])))))
+
+  except KeyError:
+    print('bogus type %s'%subdir, file=sys.stderr)
+    exit(1)
+
   with open((outfile_name:="%s/%s/%s/lmh.cdb_in"%(outdir, segment, subdir)),'wb') as OUT:
     for infile_name in subprocess.run(infile_pat, shell=True,
                                    stdout=subprocess.PIPE).stdout.decode('utf8').split():
@@ -92,13 +97,10 @@
         print('bogus warc file number %s'%infile_name, file=sys.stderr)
         exit(2)
       WIN = LOSE = N = 0
-      if subdir in ['warc','robotstxt']:
+      if shrink_key.SD == shrink_key.WARC or shrink_key.SD == shrink_key.ROBOTSTXT:
         warc.warc(infile_name, LMHline, [warc.RESP], parts = 3, block = True)
-      elif subdir == 'crawldiagnostics':
+      elif shrink_key.SD == shrink_key.CRAWLDIAGNOSTICS:
         warc.warc(infile_name, LMHline, [warc.RESP, warc.REVISIT], parts = 3, block = True)
-      else:
-        print('bogus type %s'%subdir, file=sys.stderr)
-        exit(1)
       print('%d LM headers, %d good LM string, %d bad LM string'%(N, WIN,LOSE), file=sys.stderr)

   print(outfile_name)