#!/usr/bin/env python3
# cython: profile=False, language_level=3str
'''tabulate '''

import iwarc, re, sys
from collections import defaultdict

import typing

MDPAT: typing.Pattern[bytes] = re.compile(b'^WARC-Identified-Payload-Type: (.*?)\r?$',re.MULTILINE)
L1PAT: typing.Pattern[bytes] = re.compile(b'(.*)\r\n')

HCOUNTS: defaultdict[int] = defaultdict(int)
L1COUNTS: defaultdict[int] = defaultdict(int)
N: int = 0
WIN: int = 0


def tabml(wtype: int, buf: memoryview, part: int) -> None:
  global LMPAT
  global N, WIN, HCOUNTS, L1COUNTS
  global OFFSET

  lmi: int
  mm: typing.Match[bytes] | None
  mpat: bytes
  el1: int
  # HTTP headers
  if part == 1:
    mm=MDPAT.search(buf)
    if mm:
      N += 1
      mpat = mm[1]
      if mpat in [b'text/html',b'application/xhtml+xml',b'application/xml']:
        WIN += 1
        HCOUNTS[mpat.decode('utf8')]+=1
  else:
    # part = 4
    
    if (mm := L1PAT.match(buf)):
      L1COUNTS[(mm[1][:min(10,len(mm[1]))])] += 1

def main(inpath: str):
  global N, WIN, HCOUNTS, L1COUNTS
  iwarc.warc(inpath, tabml, [iwarc.RESP], parts = 5)
  print("%s out of %s html-ish responses"%(WIN,N))
  for k,v in sorted(HCOUNTS.items(),key=lambda x:x[1],reverse=True):
    print('%10d %s'%(v,k))
  print("Counts > 1 of the first 10 characters in the body")
  for k,v in sorted(L1COUNTS.items(),key=lambda x:x[1],reverse=True):
    if v == 1:
      break
    print('%10d %s'%(v,k))

if __name__ == '__main__':
  sys.exit(main(*sys.argv[1:]))