#!/usr/bin/python3
'''Process output of lmh_warc [original 2-column version]
   Usage: <(uz ....warc.gz | fgrep $'\t'|sed "/GMT$/s/\([^ ]\)GMT$/\1 GMT/")
'''

# Assumes you have used grep -v $'\t' on input for speed
# Recommended to also sed '/GMT$/s/\([^ ]\)GMT$/\1 GMT/'
#  to fix a common 'bad' timestamp (~ .2% of inputs)

import email.utils
import sys
from urllib.parse import urlsplit, quote, unquote
from surt import surt

import re, codecs

WPAT = re.compile('(,www\\d*)+\\)')

# Thanks to https://stackoverflow.com/a/8776871
import locale
from functools import cmp_to_key

def percent_encode(ude):
  #print(ude.object,ude.object[ude.start:ude.end])
  return (''.join('%%%X'%c for c in ude.object[ude.start:ude.end]),
          ude.end)

codecs.register_error('percent',percent_encode)

# From RFC-3986:
# gen-delims    = ":" / "/" / "?" / "#" / "[" / "]" / "@"
# sub-delims    = "!" / "$" / "&" / "'" / "(" / ")"
#                / "*" / "+" / "," / ";" / "="
# But # _is_ escaped in Java surt results
#  and additionally " \ : < = > ? \ ^  _ ` { | } are not

# Note also that although quote already does _not_ quote - . / _ ~
#  they are included below as that's what we find in surt.surt 0.3.1

# Also, Java surt strips _all_ leading 'www\d*.',
#  where python3 surt only strips the first one.

def cdx_key(uristring):
  _surt = quote(unquote(surt(uristring),
                        errors='percent'),
                safe='''!"$&'()*+,-./:;<=>?@[\]^_`{|}~''' # '
                ).lower()
  return WPAT.sub(')',_surt)

def keyed(l):
  uri, cc_stamp, dateTime = l.split('\t',2)
  #print('ul',uri,file=sys.stderr)
  try:
    try:
      epoch = email.utils.parsedate_to_datetime(dateTime).timestamp()
    except OverflowError:
      epoch = 32535215999.0
    return ((cdx_key(uri), cc_stamp, uri), epoch)
  except (TypeError,IndexError,ValueError) as e:
    print(dateTime.rstrip(),e,sep='\t',file=sys.stderr)
    return

fstr = sys.argv[1]

with open(fstr,"r") as ff:
  # crucial that the following is done _after_ the file is opened
  #  with the default (utf-8) locale!
  locale.setlocale(locale.LC_ALL, "C")
  ctk=cmp_to_key(locale.strcoll)
  for key, ts in sorted((kk for l in ff if (kk:=keyed(l)) is not None),
                   key=lambda x:x[0]):
    print(key[0],key[1],key[2],ts,sep='\t')