Mercurial > hg > cc > cirrus_work

changeset 376:af18b5074464 plus

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
get new shrink_key working with warc2cdb and vv
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 22 Apr 2026 18:47:51 +0100
parents a74a3335e0d2
children b3ba45af45c9
files lib/python/cc/lmh/shrink_key.py lib/python/cc/lmh/warc2cdb.py
diffstat 2 files changed, 11 insertions(+), 8 deletions(-) [+]
line wrap: on
line diff
--- a/lib/python/cc/lmh/shrink_key.py	Wed Apr 22 15:09:30 2026 +0100
+++ b/lib/python/cc/lmh/shrink_key.py	Wed Apr 22 18:47:51 2026 +0100
@@ -29,17 +29,20 @@
 
 def set_sd_from_string(subdir: str):
   global SD, SUBDIR_CODES
-  SD = (shrink_key.SUBDIR_CODES[b'w'] if subdir.startswith('w') 
-                     else (shrink_key.SUBDIR_CODES[b'c'] if subdir.startswith('c')
-                           else shrink_key.SUBDIR_CODES[bytes(ord(subdir[0]))]))
+  SD = (SUBDIR_CODES[b'w'] if subdir.startswith('w') 
+        else (SUBDIR_CODES[b'c'] if subdir.startswith('c')
+              else (SUBDIR_CODES[b'r'] if subdir.startswith('r')
+                    else -1)))
+  if SD == -1:
+    raise KeyError(subdir)
 
 def set_sd_from_bytes(subdir: bytes):
   global SD, SUBDIR_CODES
   z: int = subdir[0]
   SD = (SUBDIR_CODES[b'w'] if z == 119
-                     else (SUBDIR_CODES[b'c'] if z == 99
-                           else (SUBDIR_CODES[b'r'] if z == 114
-                                 else -1)))
+        else (SUBDIR_CODES[b'c'] if z == 99
+              else (SUBDIR_CODES[b'r'] if z == 114
+                    else -1)))
   if SD == -1:
     raise KeyError(subdir)
 
@@ -55,7 +58,7 @@
       lmb = FIRST_DATE
     else:
       lmb = int.to_bytes(lm, 4, signed=True)
-  return(shrink_key(seg, fileno, offset), lm)
+  return(shrink_key(seg, fileno, offset), lmb)
 
 def shrink_key(seg: int, fileno: int, offset: int) -> bytes:
   global SD
--- a/lib/python/cc/lmh/warc2cdb.py	Wed Apr 22 15:09:30 2026 +0100
+++ b/lib/python/cc/lmh/warc2cdb.py	Wed Apr 22 18:47:51 2026 +0100
@@ -79,7 +79,7 @@
   infile_pat='bash -c "ls $CCC/CC-MAIN-%s/*.%s/orig/%s/*00%s.warc.gz | sort -k8"'%(CCdate, segment, subdir, fpat)
 
   try:
-    shrink_key.set_ftype_from_string(subdir)
+    shrink_key.set_sd_from_string(subdir)
   except KeyError:
     print('bogus type %s'%subdir, file=sys.stderr)
     exit(1)