--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/ix.py	Thu Apr 15 19:22:27 2021 +0000
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+'''Extract request records from Common Crawl WARC-format files
+given length, offset and file triples.
+Input one triple on command line, or
+triples from stdin as tab-delimited lines
+or complete cdx index lines.
+
+Note that if no output flag(s) is/are given, the whole WARC record will be output, more efficiently than would be the case if all three flags were given.'''
+
+import argparse 
+
+parser = argparse.ArgumentParser(
+  description='''Extract records from warc files given length, offset and file triples.
+Input one triple on command line, or
+triples from stdin as tab-delimited lines
+or complete cdx index lines.''',
+  epilog='''[ -x | length option file]
+
+Note that if no output flag(s) is/are given,
+the whole WARC record will be output, more efficiently than
+would be the case if all three flags were given.''',
+  add_help=False,
+  conflict_handler='resolve',
+  formatter_class=argparse.RawDescriptionHelpFormatter
+  )
+  
+parser.add_argument('--help',help='Show help',action='store_true')
+parser.add_argument('-d','--debug',help='Debug output',action='store_true')
+parser.add_argument('-w','--warc',help='output WARC headers',
+                    action='store_true')
+parser.add_argument('-h','--headers',help='output HTTP headers',
+                    action='store_true')
+parser.add_argument('-b','--body',help='output HTTP body',
+                    action='store_true')
+parser.add_argument('-e','--exec',help='pipes each result thru EXEC')
+sg=parser.add_mutually_exclusive_group()
+sg.add_argument('-x','--index',
+                    help='take lines of triples from a cdx index file as input',
+                    action='store_true')
+tg=sg.add_argument_group('triple','explicit triple')
+tg.add_argument('length',type=int,
+                    help='length in bytes of gzipped record')
+tg.add_argument('offset',type=int,
+                    help='start position in bytes of gzipped record')
+tg.add_argument('filename',
+                    help='name of gzipped Common Crawl WARC-format file')
+
+parser.print_help()