#!/usr/bin/env python3
'''Extract named fields, with optional post-processing, in order,
                         from a Common Crawl index row

   Field specs on command line are either an atom or
                                             a tuple of atom and expression with
                                             free variable f
   For example
    > cdx2tsv.py mime '(url,f.split(":",maxsplit=1)[0])' < xyzzy.cdx
   will output media type and URI scheme

   Note that 'key' and 'timestamp' are also available as global variables,
   so e.g. '(key,key.split(",")[0])' will yield the TLD.
   For a callback instead of printing, include -c cb-mod for a module to
     and within which to call init() once and the cbf function for each index line
'''
import json,sys

index = cbf = None
while len(sys.argv)>2 and sys.argv[1][0]=='-':
  if sys.argv[1] == '-i':
    sys.argv.pop(1)
    index=str(int(sys.argv.pop(1)))
  elif sys.argv[1] == '-c':
    sys.argv.pop(1)
    cbmodname = sys.argv.pop(1)
    from importlib import import_module
    cbm = import_module(cbmodname)
    cbf=cbm.cbf
    cbm.init()
  else:
    break

if len(sys.argv)>1 and sys.argv[1][0]=='-':
  print("""Reads index lines from stdin and extracts values from json dict part

  Usage: cdx2tsv.py [-i n] [-c callback-fn] fieldspecs...

  fieldspec is either a name or a quoted python tuple of a name and
    an expression with free variable f which will be evaluated with f
    having the field value.

  For example 
    cdx2tsv.py mime '(url,f.split(":",maxsplit=1)[0])' < xyzzy.cdx
  will output media type and URI scheme

   -i n will include n, typically the index number, as 1st column

   -c cbfile will apply cbfile:cbf to each list of values instead of printing
          [this is probably a better approach than using tuple args with
           lots of tricky lambdas]
  """,
        file=sys.stderr)
  exit(1)

fields=sys.argv[1:]

stash={} # This can be used to save an expensive field computation for re-use:
         # E.g. '(filename,(lambda g:(stash.__setitem__("T1",g) or g)[3].split(".")[1])(f.split("/",maxsplit=5)))'  '(filename,(stash["T1"])[4][0])

fields=[((lambda x,y:(x,eval("lambda f:%s"%y,globals())))(*(f[1:-1].split(',',maxsplit=1))) if f[0]=='(' else f) for f in fields]
for l in sys.stdin:
  (surt,stamp,jj)=l.rstrip().split(' ',maxsplit=2)
  key = surt.split(')')
  ja=json.loads(jj)
  args = ((ja.get(f,'NA') if isinstance(f,str) else
           (f[1](ja[f[0]] if f[0] in ja else 'NA'))) for f in (ja.keys()
                                                      if fields==["*"] else fields))
  if cbf:
    cbf(*args)
  else:
    if index:
      print(index,end='\t')
    print('\t'.join(args))

if cbf:
  cbm.finish()