#!/usr/bin/env python
'''Extract request records from Common Crawl WARC-format files
given length, offset and file triples.
Input one triple on command line, or
triples from stdin as tab-delimited lines
or complete cdx index lines.

Note that if no output flag(s) is/are given, the whole WARC record will be output, more efficiently than would be the case if all three flags were given.'''

import argparse 

parser = argparse.ArgumentParser(
  description='''Extract records from warc files given length, offset and file triples.
Input one triple on command line, or
triples from stdin as tab-delimited lines
or complete cdx index lines.''',
  epilog='''[ -x | length option file]

Note that if no output flag(s) is/are given,
the whole WARC record will be output, more efficiently than
would be the case if all three flags were given.''',
  add_help=False,
  conflict_handler='resolve',
  formatter_class=argparse.RawDescriptionHelpFormatter
  )
  
parser.add_argument('--help',help='Show help',action='store_true')
parser.add_argument('-d','--debug',help='Debug output',action='store_true')
parser.add_argument('-w','--warc',help='output WARC headers',
                    action='store_true')
parser.add_argument('-h','--headers',help='output HTTP headers',
                    action='store_true')
parser.add_argument('-b','--body',help='output HTTP body',
                    action='store_true')
parser.add_argument('-e','--exec',help='pipes each result thru EXEC')
sg=parser.add_mutually_exclusive_group()
sg.add_argument('-x','--index',
                    help='take lines of triples from a cdx index file as input',
                    action='store_true')
tg=sg.add_argument_group('triple','explicit triple')
tg.add_argument('length',type=int,
                    help='length in bytes of gzipped record')
tg.add_argument('offset',type=int,
                    help='start position in bytes of gzipped record')
tg.add_argument('filename',
                    help='name of gzipped Common Crawl WARC-format file')

parser.print_help()