Mercurial > hg > cc > cirrus_home

annotate bin/ix.py @ 98:1a4c5fdc2923

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
help format hacking done
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Fri, 16 Apr 2021 13:15:23 +0000
parents 2b880f2ce894
children d48537c4cbae
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/usr/bin/env python
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 '''Extract request records from Common Crawl WARC-format files
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 given length, offset and file triples.
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 Input one triple on command line, or
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 triples from stdin as tab-delimited lines
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 or complete cdx index lines.
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7
98
1a4c5fdc2923 help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 97
diff changeset
8 Note that if no output flag(s) is/are given, the whole WARC record will be output, more efficiently than would be the case if -whb is given.'''
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
10 import sys, argparse, regex
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
11
98
1a4c5fdc2923 help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 97
diff changeset
12 HACK_USAGE=regex.compile('\[-x\]\n\s*\[length\] \[offset\] \[filename\]')
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
14 class HackFormat(argparse.RawDescriptionHelpFormatter):
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
15 def format_help(self):
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
16 global FOO
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
17 FOO=argparse.RawDescriptionHelpFormatter.format_help(self)
98
1a4c5fdc2923 help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 97
diff changeset
18 return HACK_USAGE.sub('\n [ ( -x | length offset filename ) ]',
1a4c5fdc2923 help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 97
diff changeset
19 FOO)
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
20
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21 parser = argparse.ArgumentParser(
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22 description='''Extract records from warc files given length, offset and file triples.
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23 Input one triple on command line, or
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 triples from stdin as tab-delimited lines
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25 or complete cdx index lines.''',
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
26 epilog='''Note that if no output flag(s) is/are given,
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 the whole WARC record will be output, more efficiently than
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28 would be the case if all three flags were given.''',
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
29 add_help=False,
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
30 conflict_handler='resolve',
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
31 formatter_class=HackFormat
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
32 )
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
33
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
34 parser.add_argument('--help',help='Show help',action='help')
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
35 parser.add_argument('-d','--debug',help='Debug output',action='store_true')
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
36 parser.add_argument('-w','--warc',help='output WARC headers',
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
37 action='store_true')
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
38 parser.add_argument('-h','--headers',help='output HTTP headers',
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
39 action='store_true')
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
40 parser.add_argument('-b','--body',help='output HTTP body',
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
41 action='store_true')
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
42 parser.add_argument('-c','--cmd',help='pipes each result thru CMD')
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
43 sg=parser.add_mutually_exclusive_group()
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
44 sg.add_argument('-x','--index',
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
45 help='take lines of triples from a cdx index file as input',
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
46 action='store_true')
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
47 sg.add_argument('length',type=int,
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
48 help='length in bytes of gzipped record',
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
49 nargs='?')
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
50 parser.add_argument('offset',type=int,
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
51 help='start position in bytes of gzipped record',
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
52 nargs='?')
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
53 parser.add_argument('filename',
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
54 help='name of gzipped Common Crawl WARC-format file',
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
55 nargs='?')
98
1a4c5fdc2923 help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 97
diff changeset
56 # Hack the order of optional and positional in the help output
1a4c5fdc2923 help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 97
diff changeset
57 parser._action_groups.sort(key=lambda g:g.title)
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
58 #parser.print_help()
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
59 pa=parser.parse_args(sys.argv[1:])
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
60 # We have to enforce our own check..
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
61 print(pa)
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
62