# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1782140331 -3600
# Node ID b9b301fd064d0d2bfba297f2469dd2e22a103b19
# Parent  af49ea848756a63e4daf658eb87076b34dad4521
sic

diff -r af49ea848756 -r b9b301fd064d lib/python/cc/ex6.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/python/cc/ex6.py	Mon Jun 22 15:58:51 2026 +0100
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# cython: profile=False, language_level=3str
+'''tabulate '''
+
+import iwarc, re, sys
+from collections import defaultdict
+
+import typing
+
+MDPAT: typing.Pattern[bytes] = re.compile(b'^WARC-Identified-Payload-Type: (.*?)\r?$',re.MULTILINE)
+L1PAT: typing.Pattern[bytes] = re.compile(b'(.*)\r\n')
+
+HCOUNTS: defaultdict[int] = defaultdict(int)
+L1COUNTS: defaultdict[int] = defaultdict(int)
+N: int = 0
+WIN: int = 0
+
+
+def tabml(wtype: int, buf: memoryview, part: int) -> None:
+  global LMPAT
+  global N, WIN, HCOUNTS, L1COUNTS
+  global OFFSET
+
+  lmi: int
+  mm: typing.Match[bytes] | None
+  mpat: bytes
+  el1: int
+  # HTTP headers
+  if part == 1:
+    mm=MDPAT.search(buf)
+    if mm:
+      N += 1
+      mpat = mm[1]
+      if mpat in [b'text/html',b'application/xhtml+xml',b'application/xml']:
+        WIN += 1
+        HCOUNTS[mpat.decode('utf8')]+=1
+  else:
+    # part = 4
+    
+    if (mm := L1PAT.match(buf)):
+      L1COUNTS[(mm[1][:min(10,len(mm[1]))]).decode('utf8')] += 1
+
+def main(inpath: str):
+  global N, WIN, HCOUNTS, L1COUNTS
+  iwarc.warc(inpath, tabml, [iwarc.RESP], parts = 5)
+  print("%s out of %s html-ish responses"%(WIN,N))
+  for k,v in sorted(HCOUNTS.items(),key=lambda x:x[1],reverse=True):
+    print('%10d %s'%(v,k))
+  print("Counts > 1 of the first 10 characters in the body")
+  for k,v in sorted(L1COUNTS.items(),key=lambda x:x[1],reverse=True):
+    if v == 1:
+      break
+    print('%10d %s'%(v,k))
+
+if __name__ == '__main__':
+  sys.exit(main(*sys.argv[1:]))