#!/usr/bin/python3
'''Process output of lmh_warc [new 3-column version]
   Usage: <(uz ....warc.gz | fgrep $'\t'|sed "/GMT$/s/\([^ ]\)GMT$/\1 GMT/")
'''

# Assumes you have used grep -v $'\t' on input for speed
# Recommended to also sed '/GMT$/s/\([^ ]\)GMT$/\1 GMT/'
#  to fix a common 'bad' timestamp (~ .2% of inputs)

import email.utils
import sys
from urllib.parse import urlsplit, quote, unquote
import surt

import re, codecs

WPAT = re.compile('(,www\\d*)+\\)')

# Thanks to https://stackoverflow.com/a/8776871
import locale
from functools import cmp_to_key

def percent_encode(ude):
  #print(ude.object,ude.object[ude.start:ude.end])
  return (''.join('%%%X'%c for c in ude.object[ude.start:ude.end]),
          ude.end)

codecs.register_error('percent',percent_encode)

# From RFC-3986:
# gen-delims    = ":" / "/" / "?" / "#" / "[" / "]" / "@"
# sub-delims    = "!" / "$" / "&" / "'" / "(" / ")"
#                / "*" / "+" / "," / ";" / "="
# But # _is_ escaped in Java surt results
#  and additionally " \ : < = > ? \ ^  _ ` { | } are not

# Note also that although quote already does _not_ quote - . / _ ~
#  they are included below as that's what we find in surt.surt 0.3.1

# Also, Java surt strips _all_ leading 'www\d*.',
#  where python3 surt only strips the first one.

# And Java strips so-called option session-ids, but python doesn't

import surt.DefaultIAURLCanonicalizer, surt.GoogleURLCanonicalizer

def notDefaultCanon(hu,**options):
  if surt.GoogleURLCanonicalizer.DECIMAL_IP.match(hu.host):
    # Try to fix the incompatibility between Java and 
    #  Python surt handling of 'octal' numbers in numeric IPv4 addresses
    #  and it should!  See "After this line:
    # 
    # 15,225,107,143)" in .../azure/notes.txt
    try:
      bytestrs = hu.host.split(b'.')
      hu.host = b'.'.join(b'%d'%int(bs) for bs in bytestrs)
    except ValueError:
      pass
  # Either we don't hit any, or Java doesn't do path_strip_session_id
  options.setdefault('query_strip_session_id',False)
  return surt.DefaultIAURLCanonicalizer.canonicalize(hu,
                                                **options)

def cdx_key(uristring):
  _surt = quote(unquote(surt.surt(uristring,
                                  canonicalizer=notDefaultCanon),
                        errors='percent'),
                safe='''!"$&'()*+,-./:;<=>?@[\]^_`{|}~''' # '
                ).lower()
  return WPAT.sub(')',_surt)

def keyed(l):
  uri, cc_stamp, dateTime = l.split('\t',2)
  #print('ul',uri,file=sys.stderr)
  try:
    try:
      epoch = email.utils.parsedate_to_datetime(dateTime).timestamp()
    except OverflowError:
      epoch = 32535215999.0
    return ((cdx_key(uri), cc_stamp, uri), epoch)
  except (TypeError,IndexError,ValueError) as e:
    print(dateTime.rstrip(),e,sep='\t',file=sys.stderr)
    return

fstr = sys.argv[1]

with open(fstr,"r") as ff:
  # crucial that the following is done _after_ the file is opened
  #  with the default (utf-8) locale!
  locale.setlocale(locale.LC_ALL, "C")
  ctk=cmp_to_key(locale.strcoll)
  for key, ts in sorted((kk for l in ff if (kk:=keyed(l)) is not None),
                   key=lambda x:x[0]):
    print(key[0],key[1],key[2],ts,sep='\t')