# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1768583553 0
# Node ID 439cbfd6cbda5dd212ef3ee53f52ece8a1f90de2
# Parent  2502b81838da203e45fb7c1f5c5255bf2f5329c4
works

diff -r 2502b81838da -r 439cbfd6cbda lib/python/cc/lmh/shrink_key.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/python/cc/lmh/shrink_key.py	Fri Jan 16 17:12:33 2026 +0000
@@ -0,0 +1,57 @@
+#!/usr/bin/python3
+# cython: profile=False, language_level=3str
+'''Construct a shrunken cdb key from an original cdx entry
+   If run as a script, output this to stdout
+   Usage: uz .../cdx-00nnn.gz | ~/lib/python/cc/lmh/shrink_key.py DP
+   The output cdb entries have only a crawl date suffix wrt to a
+    prefix with DP.  For example
+         given a DP of 20230922103943
+         gives a shrunken value of 234
+'''
+
+
+
+import cython, typing, timeit, re, sys, array
+
+C_PAT: typing.Pattern[bytes] = re.compile(b'[^ ]* ([^ ]*) .*{"url": "(http[^"]*).*"filename": "[^"]*[.]([0-9][0-9]?)/(warc|robotstxt|crawldiagnostics)/')
+
+def shrink(entry: bytes, dp: bytes) -> bytes:
+  m: typing.Match[bytes] | None
+
+  dpl: int = len(dp)
+  uri: bytes
+  wdate: bytes
+  kind: bytes
+  segb: bytes
+  ts: array.array
+  seg: int
+  res: int
+  i: int
+  nd: bytes
+  if (m:=C_PAT.match(entry)):
+    (wdate, uri, segb, kind) = m.groups()
+  else:
+    raise ValueError(l)
+  seg = int(segb)
+  for i in range(dpl):
+    if wdate[i] != dp[i]:
+      break
+  else:
+    i+=1
+  if i > 0:
+    nd = wdate[i:]
+  else:
+    nd = dp
+  if kind == 'robotstxt':
+    nd += seg
+  if not uri.startswith(b'http'):
+    raise ValueError(uri)
+  #print(nd,uri[4:],file=sys.stderr)
+  return nd+uri[4:]
+
+if __name__ == "__main__":
+  cdx_in: typing.BinaryIO = sys.stdin.buffer
+  l: bytes
+  DP: bytes = sys.argv[1].encode('ASCII')
+  for l in cdx_in:
+    sys.stdout.buffer.write(shrink(l,DP))