# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1772290265 0
# Node ID a6c7f23cca94638349ff5f18db6611d0d3c2297b

adding callback functionality

diff -r 000000000000 -r a6c7f23cca94 bin/cdx2tsv.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/cdx2tsv.py	Sat Feb 28 14:51:05 2026 +0000
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+'''Extract named fields, with optional post-processing, in order,
+                         from a Common Crawl index row
+
+   Field specs on command line are either an atom or
+                                             a tuple of atom and expression with
+                                             free variable f
+   For example
+    > cdx2tsv.py mime '(url,f.split(":",maxsplit=1)[0])' < xyzzy.cdx
+   will output media type and URI scheme
+
+   Note that 'key' and 'timestamp' are also available as global variables,
+   so e.g. '(key,key.split(",")[0])' will yield the TLD.
+   For a callback instead of printing, include -c cb-mod for a module to
+     and within which to call init() once and the cbf function for each index line
+'''
+import json,sys
+
+index = cbf = None
+while len(sys.argv)>2 and sys.argv[1][0]=='-':
+  if sys.argv[1] == '-i':
+    sys.argv.pop(1)
+    index=str(int(sys.argv.pop(1)))
+  elif sys.argv[1] == '-c':
+    sys.argv.pop(1)
+    cbmodname = sys.argv.pop(1)
+    from importlib import import_module
+    cbm = import_module(cbmodname)
+    cbf=cbm.cbf
+    cbm.init()
+  else:
+    break
+
+if len(sys.argv)>1 and sys.argv[1][0]=='-':
+  print("""Reads index lines from stdin and extracts values from json dict part
+
+  Usage: cdx2tsv.py [-i n] [-c callback-fn] fieldspecs...
+
+  fieldspec is either a name or a quoted python tuple of a name and
+    an expression with free variable f which will be evaluated with f
+    having the field value.
+
+  For example 
+    cdx2tsv.py mime '(url,f.split(":",maxsplit=1)[0])' < xyzzy.cdx
+  will output media type and URI scheme
+
+   -i n will include n, typically the index number, as 1st column
+
+   -c cbfile will apply cbfile:cbf to each list of values instead of printing
+          [this is probably a better approach than using tuple args with
+           lots of tricky lambdas]
+  """,
+        file=sys.stderr)
+  exit(1)
+
+fields=sys.argv[1:]
+
+stash={} # This can be used to save an expensive field computation for re-use:
+         # E.g. '(filename,(lambda g:(stash.__setitem__("T1",g) or g)[3].split(".")[1])(f.split("/",maxsplit=5)))'  '(filename,(stash["T1"])[4][0])
+
+fields=[((lambda x,y:(x,eval("lambda f:%s"%y,globals())))(*(f[1:-1].split(',',maxsplit=1))) if f[0]=='(' else f) for f in fields]
+for l in sys.stdin:
+  (surt,stamp,jj)=l.rstrip().split(' ',maxsplit=2)
+  key = surt.split(')')
+  ja=json.loads(jj)
+  args = ((ja.get(f,'NA') if isinstance(f,str) else
+           (f[1](ja[f[0]] if f[0] in ja else 'NA'))) for f in (ja.keys()
+                                                      if fields==["*"] else fields))
+  if cbf:
+    cbf(*args)
+  else:
+    if index:
+      print(index,end='\t')
+    print('\t'.join(args))
+
+if cbf:
+  cbm.finish()
+