#!/usr/bin/env python
'''Extract request records from Common Crawl WARC-format files
given length, offset and file triples.
Input one triple on command line, or
triples from stdin as tab-delimited lines
or complete cdx index lines.

Note that if no output flag(s) is/are given, the whole WARC record will be output, more efficiently than would be the case if -whb is given.'''

import sys, argparse, regex

HACK_USAGE=regex.compile('\[-x\]\n\s*\[length\] \[offset\] \[filename\]')

class HackFormat(argparse.RawDescriptionHelpFormatter):
  def format_help(self):
    global FOO
    FOO=argparse.RawDescriptionHelpFormatter.format_help(self)
    return HACK_USAGE.sub('\n             [ ( -x | length offset filename ) ]',
                          FOO)
  
parser = argparse.ArgumentParser(
  description='''Extract records from warc files given length, offset and file triples.
Input one triple on command line, or
triples from stdin as tab-delimited lines
or complete cdx index lines.''',
  epilog='''Note that if no output flag(s) is/are given,
the whole WARC record will be output, more efficiently than
would be the case if all three flags were given.''',
  add_help=False,
  conflict_handler='resolve',
  formatter_class=HackFormat
  )
  
parser.add_argument('--help',help='Show help',action='help')
parser.add_argument('-d','--debug',help='Debug output',action='store_true')
parser.add_argument('-w','--warc',help='output WARC headers',
                    action='store_true')
parser.add_argument('-h','--headers',help='output HTTP headers',
                    action='store_true')
parser.add_argument('-b','--body',help='output HTTP body',
                    action='store_true')
parser.add_argument('-c','--cmd',help='pipes each result thru CMD')
sg=parser.add_mutually_exclusive_group()
sg.add_argument('-x','--index',
                    help='take lines of triples from a cdx index file as input',
                    action='store_true')
sg.add_argument('length',type=int,
                help='length in bytes of gzipped record',
                nargs='?')
parser.add_argument('offset',type=int,
                    help='start position in bytes of gzipped record',
                    nargs='?')
parser.add_argument('filename',
                    help='name of gzipped Common Crawl WARC-format file',
                    nargs='?',
                    type=argparse.FileType('rb',0))
# Hack the order of optional and positional in the help output
parser._action_groups.sort(key=lambda g:g.title)
#parser.print_help()
pa=parser.parse_args(sys.argv[1:])
# We have to enforce our own check..
print(pa,file=sys.stderr)
if pa.length is not None and (
  pa.offset is None or pa.filename is None):
  parser.error("length, offset and filename must all be supplied together")
  
buf=bytearray(1024*1024)