--- a/lib/python/cc/lmh/shrink_key.py	Mon Mar 16 19:06:17 2026 +0000
+++ b/lib/python/cc/lmh/shrink_key.py	Mon Mar 16 19:07:10 2026 +0000
@@ -13,14 +13,22 @@
 
 import cython, typing, timeit, re, sys, array
 
+LAST_DATE: bytes = int.to_bytes(2147483647,4)
+FIRST_DATE: bytes = int.to_bytes(-2147483647,4,signed=True)
+
 C_PAT: typing.Pattern[bytes] = re.compile(b'[^ ]* ([^ ]*) .*{"url": "(http[^"]*).*"filename": "[^"]*[.]([0-9][0-9]?)/(warc|robotstxt|crawldiagnostics)/')
 
-def shrink_key(wdate: bytes, dp: bytes, lmb: bytes) -> tuple[bytes,bytes]:
-  lm = int(lmb)
+def shrink_key(seg: int, fileno: int, offset: int, lm: int) -> tuple[bytes,bytes]:
+  lmb: bytes
   if (lm >= 0 and lm <= 2147483647):
     lmb=int.to_bytes(lm,4)
+  else:
+    if lm < -2147483647:
+      lmb = FIRST_DATE
+    if lm > 2147483647:
+      lmb = LAST_DATE
   #print(nd,uri[4:],file=sys.stderr)
-  return (shrink_date(wdate,dp),lmb)
+  return (int.to_bytes(seg,1)+int.to_bytes(fileno,2)+int.to_bytes(offset,4),lmb)
 
 def shrink_date(wdate: bytes, dp: bytes) -> bytes:
   dpl: int = len(dp)

--- a/lib/python/cc/lmh/warc2cdb.py	Mon Mar 16 19:06:17 2026 +0000
+++ b/lib/python/cc/lmh/warc2cdb.py	Mon Mar 16 19:07:10 2026 +0000
@@ -14,22 +14,13 @@
 import subprocess
 import shrink_key
 
-TUPAT: typing.Pattern[bytes] = re.compile(b'^WARC-Target-URI: (.*?)\r?$',re.MULTILINE)
-DPAT: typing.Pattern[bytes] = re.compile(b'^WARC-Date: (.*?)\r?$',re.MULTILINE)
 LMPAT: typing.Pattern[bytes] = re.compile(b'^Last-Modified: (.*?)\r?$',re.MULTILINE)
 FFPAT: typing.Pattern[bytes] = re.compile(b'([^ ])GMT$')
 
-DTAB: bytearray = bytearray(range(256))
-DDEL: bytes = b'TZ-:'
-
 OUT: typing.BinaryIO
 SEG: bytes
 R_T: bool = False
 
-URI: bytes
-DATE: bytes
-LM: bytes
-
 WIN: int = 0
 LOSE: int = 0
 N: int = 0
@@ -52,26 +43,20 @@
 
 codecs.register_error('java_unicode',java_unicode_encode)
 
-def LMHline(wtype: int, buf: memoryview, part: int) -> None:
-  global TUPAT, DPAT, LMPAT, FFPAT, DTAB, DDEL, OUT, WIN, LOSE, NON_HTTP, NON_MONTH
+def LMHline(wtype: int, buf: memoryview, part: int, brange: tuple[int, int]) -> None:
+  global LMPAT, FFPAT, OUT, WIN, LOSE, NON_MONTH
   global N, UERRS, SEG, R_T, C_BASE, C_BASE_L
-  global DATE, URI, LM
+  global DATE, URI
+
+  lmi: int
+  offset: int
   m: typing.Match[cython.bytes] | None
   mm: typing.Match[cython.bytes] | None
-  lmi: cython.bytes
-  if wtype==warc.REQ and part==1:
-    if (m:=TUPAT.search(buf)):
-      URI=m[1]
-    else:
-      raise ValueError(b"No target URI in %s ??"%buf)
-  else:
+  key: cython.bytes
+  val: cython.bytes
     # Response
-    if part==1:
-      # WARC headers
-      if (md:=DPAT.search(buf)):
-        DATE=md[1]
-      else:
-        raise ValueError(b"No date in %s ??"%buf)
+    if part == 1:
+      offset = (brange[0]
     else:
       # HTTP headers
       mm=LMPAT.search(buf)
@@ -83,27 +68,15 @@
             dateTime = dateTime[:-3]+b' GMT' # FFPAT.sub(b'\\1 GMT',dateTime)
         try:
           try:
-            lmi = b'%d'%int(email.utils.parsedate_to_datetime(dateTime.decode('utf8')).timestamp())
+            lmi = int(email.utils.parsedate_to_datetime(dateTime.decode('utf8')).timestamp())
           except OverflowError:
-            lmi = b'32535215999'
+            lmi = 2147483647
         except (TypeError,IndexError,ValueError) as e:
           print(dateTime.rstrip(),e,sep='\t',file=sys.stderr)
           LOSE += 1
           return
-        DATE=(DATE.translate(DTAB,DDEL))
-        (DATE,LM) = shrink_key.shrink_key(DATE,C_BASE,lmi)
+        (key,val) = shrink_key.shrink_key(SEG, FILENO, offset, lmi)
         WIN += 1
-        try:
-          URI.decode('ascii')
-        except UnicodeDecodeError:
-          UERRS += 1
-          # Try just fixing the non-ASCII:
-          URI = URI.decode('utf-8').encode('ascii', errors='java_unicode')
-        # Could just assume http, but let's check
-        if URI.startswith(b'http'):
-          URI=URI[4:]
-        else:
-          NON_HTTP += 1
         l: int = len(LM)
         kl: int = (len(DATE)+len(URI)+(len(SEG) if R_T else 0))
         OUT.write(b'+')
@@ -119,14 +92,11 @@
         OUT.write(LM)
         OUT.write(b'\n')
 
-def main(CCdate: str, segment: str, outdir: str, subdir: str, fpat: str, dp: str ):
-  global OUT, N, WIN, LOSE, UERRS, SEG, R_T
-  global NON_HTTP, C_BASE, C_BASE_L
+def main(segment: str, outdir: str, subdir: str, fpat: str ):
+  global OUT, N, WIN, LOSE, SEG, R_T
 
   SEG = segment.encode('utf8')
   R_T = (subdir == 'robotstxt')
-  C_BASE = dp.encode('ascii')
-  C_BASE_L = len(dp)
   if fpat != '???':
     fpat = ("{%s..%s}"%tuple(fpat.split(','))) if ',' in fpat else fpat
   infile_pat='bash -c "ls $CCC/CC-MAIN-%s/*.%s/orig/%s/*00%s.warc.gz | sort -k8"'%(CCdate, segment, subdir, fpat)
@@ -134,16 +104,16 @@
     for infile_name in subprocess.run(infile_pat, shell=True,
                                    stdout=subprocess.PIPE).stdout.decode('utf8').split():
       print(infile_name,file=sys.stderr)
-      WIN = LOSE = N = UERRS = NON_HTTP = 0
+      WIN = LOSE = N = 0
       if subdir in ['warc','robotstxt']:
-        warc.warc(infile_name,LMHline,[warc.REQ,warc.RESP],parts=3)
+        warc.warc(infile_name, LMHline, [warc.RESP], parts = 3, block = True)
       elif subdir == 'crawldiagnostics':
-        warc.warc(infile_name,LMHline,[warc.REQ,warc.RESP, warc.REVISIT],parts=3)
+        warc.warc(infile_name, LMHline, [warc.RESP, warc.REVISIT], parts = 3, block = True)
       else:
-        print('bogus type %s'%subdir,file=sys.stderr)
+        print('bogus type %s'%subdir, file=sys.stderr)
         exit(1)
-      print('%d LM headers, %d win, %d lose, %d non-ASCII URIs, %d dodgy schemes'%(N,WIN,LOSE,UERRS,NON_HTTP),
-                                                                 file=sys.stderr)
+      print('%d LM headers, %d win, %d lose, %d non-ASCII URIs, %d dodgy schemes'%(N,
+                             WIN,LOSE), file=sys.stderr)
 
   print(outfile_name)