# HG changeset patch # User Henry S. Thompson # Date 1776880071 -3600 # Node ID af18b50744649484d66a8c52e9855228580f0e63 # Parent a74a3335e0d2b20b9c413c62fa14ae2c6bab10ce get new shrink_key working with warc2cdb and vv diff -r a74a3335e0d2 -r af18b5074464 lib/python/cc/lmh/shrink_key.py --- a/lib/python/cc/lmh/shrink_key.py Wed Apr 22 15:09:30 2026 +0100 +++ b/lib/python/cc/lmh/shrink_key.py Wed Apr 22 18:47:51 2026 +0100 @@ -29,17 +29,20 @@ def set_sd_from_string(subdir: str): global SD, SUBDIR_CODES - SD = (shrink_key.SUBDIR_CODES[b'w'] if subdir.startswith('w') - else (shrink_key.SUBDIR_CODES[b'c'] if subdir.startswith('c') - else shrink_key.SUBDIR_CODES[bytes(ord(subdir[0]))])) + SD = (SUBDIR_CODES[b'w'] if subdir.startswith('w') + else (SUBDIR_CODES[b'c'] if subdir.startswith('c') + else (SUBDIR_CODES[b'r'] if subdir.startswith('r') + else -1))) + if SD == -1: + raise KeyError(subdir) def set_sd_from_bytes(subdir: bytes): global SD, SUBDIR_CODES z: int = subdir[0] SD = (SUBDIR_CODES[b'w'] if z == 119 - else (SUBDIR_CODES[b'c'] if z == 99 - else (SUBDIR_CODES[b'r'] if z == 114 - else -1))) + else (SUBDIR_CODES[b'c'] if z == 99 + else (SUBDIR_CODES[b'r'] if z == 114 + else -1))) if SD == -1: raise KeyError(subdir) @@ -55,7 +58,7 @@ lmb = FIRST_DATE else: lmb = int.to_bytes(lm, 4, signed=True) - return(shrink_key(seg, fileno, offset), lm) + return(shrink_key(seg, fileno, offset), lmb) def shrink_key(seg: int, fileno: int, offset: int) -> bytes: global SD diff -r a74a3335e0d2 -r af18b5074464 lib/python/cc/lmh/warc2cdb.py --- a/lib/python/cc/lmh/warc2cdb.py Wed Apr 22 15:09:30 2026 +0100 +++ b/lib/python/cc/lmh/warc2cdb.py Wed Apr 22 18:47:51 2026 +0100 @@ -79,7 +79,7 @@ infile_pat='bash -c "ls $CCC/CC-MAIN-%s/*.%s/orig/%s/*00%s.warc.gz | sort -k8"'%(CCdate, segment, subdir, fpat) try: - shrink_key.set_ftype_from_string(subdir) + shrink_key.set_sd_from_string(subdir) except KeyError: print('bogus type %s'%subdir, file=sys.stderr) exit(1)