Mercurial > hg > cc > cirrus_home
annotate bin/ix.py @ 106:815b33c3254a
working with -x and rich directory structure
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Tue, 20 Apr 2021 12:26:09 +0000 |
| parents | baf56ff538f8 |
| children | 007f35b9df9c |
| rev | line source |
|---|---|
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
1 #!/usr/bin/env python3 |
|
94
d60073ec798a
just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
2 '''Extract request records from Common Crawl WARC-format files |
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
3 given length, offset and filename triples. |
|
94
d60073ec798a
just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
4 Input one triple on command line, or |
|
d60073ec798a
just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
5 triples from stdin as tab-delimited lines |
|
d60073ec798a
just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
6 or complete cdx index lines. |
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
7 In all cases by 'filename' is meant crawlid/segmentid/filename |
|
94
d60073ec798a
just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
8 |
|
98
1a4c5fdc2923
help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
97
diff
changeset
|
9 Note that if no output flag(s) is/are given, the whole WARC record will be output, more efficiently than would be the case if -whb is given.''' |
|
94
d60073ec798a
just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
10 |
|
97
2b880f2ce894
basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
94
diff
changeset
|
11 import sys, argparse, regex |
|
2b880f2ce894
basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
94
diff
changeset
|
12 |
|
98
1a4c5fdc2923
help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
97
diff
changeset
|
13 HACK_USAGE=regex.compile('\[-x\]\n\s*\[length\] \[offset\] \[filename\]') |
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
14 FPAT="/beegfs/common_crawl/%s/%s/orig/warc/%s" |
|
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
15 BINOUT=sys.stdout.buffer |
|
94
d60073ec798a
just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
16 |
|
97
2b880f2ce894
basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
94
diff
changeset
|
17 class HackFormat(argparse.RawDescriptionHelpFormatter): |
|
2b880f2ce894
basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
94
diff
changeset
|
18 def format_help(self): |
|
2b880f2ce894
basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
94
diff
changeset
|
19 global FOO |
|
2b880f2ce894
basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
94
diff
changeset
|
20 FOO=argparse.RawDescriptionHelpFormatter.format_help(self) |
|
98
1a4c5fdc2923
help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
97
diff
changeset
|
21 return HACK_USAGE.sub('\n [ ( -x | length offset filename ) ]', |
|
1a4c5fdc2923
help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
97
diff
changeset
|
22 FOO) |
| 100 | 23 |
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
24 def process(options,buf,filename,offset,length,whole): |
|
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
25 file=open(filename,'rb',0) |
| 100 | 26 if whole: |
| 27 file.seek(offset) | |
| 28 bv=memoryview(buf)[:length] | |
| 29 nb=file.readinto(bv) | |
| 30 if nb!=length: | |
| 104 | 31 print("losing",file.name,length,nb,file=sys.stderr) |
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
32 BINOUT.write(bv) |
| 104 | 33 file.close() |
| 100 | 34 |
| 35 def main(): | |
| 36 parser = argparse.ArgumentParser( | |
| 37 description='''Extract records from warc files given length, offset and file triples. | |
| 38 Input one triple on command line, or | |
| 39 triples from stdin as tab-delimited lines | |
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
40 or complete cdx index lines. |
|
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
41 In all cases by 'filename' is meant crawlid/segmentid/filename''', |
| 100 | 42 epilog='''Note that if no output flag(s) is/are given, |
| 43 the whole WARC record will be output, more efficiently than | |
| 44 would be the case if all three flags were given.''', | |
| 45 add_help=False, | |
| 46 conflict_handler='resolve', | |
| 47 formatter_class=HackFormat | |
| 48 ) | |
|
97
2b880f2ce894
basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
94
diff
changeset
|
49 |
| 100 | 50 parser.add_argument('--help',help='Show help',action='help') |
| 51 parser.add_argument('-d','--debug',help='Debug output',action='store_true') | |
| 52 parser.add_argument('-w','--warc',help='output WARC headers', | |
| 53 action='store_true') | |
| 54 parser.add_argument('-h','--headers',help='output HTTP headers', | |
| 55 action='store_true') | |
| 56 parser.add_argument('-b','--body',help='output HTTP body', | |
| 57 action='store_true') | |
| 58 parser.add_argument('-c','--cmd',help='pipes each result thru CMD') | |
| 59 sg=parser.add_mutually_exclusive_group() | |
| 60 sg.add_argument('-x','--index', | |
| 61 help='take lines of triples from a cdx index file as input', | |
| 62 action='store_true') | |
| 63 sg.add_argument('length',type=int, | |
| 64 help='length in bytes of gzipped record', | |
| 65 nargs='?') | |
| 66 parser.add_argument('offset',type=int, | |
| 67 help='start position in bytes of gzipped record', | |
| 68 nargs='?') | |
| 69 parser.add_argument('filename', | |
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
70 help='pathname of gzipped Common Crawl WARC-format file', |
|
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
71 nargs='?') |
| 100 | 72 # Hack the order of optional and positional in the help output |
| 73 parser._action_groups.sort(key=lambda g:g.title) | |
| 74 #parser.print_help() | |
| 75 pa=parser.parse_args(sys.argv[1:]) | |
|
106
815b33c3254a
working with -x and rich directory structure
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
105
diff
changeset
|
76 #print(pa,file=sys.stderr) |
| 100 | 77 if pa.length is not None: |
| 78 # We have to enforce our own check.. | |
| 79 if pa.offset is None or pa.filename is None: | |
| 80 parser.error("length, offset and filename must all be supplied together") | |
| 81 | |
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
82 buf=bytearray(2024*1024) |
| 100 | 83 |
| 104 | 84 whole=not (pa.warc or pa.headers or pa.body) |
| 100 | 85 if pa.length is not None: |
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
86 process(pa,buf,FPAT%tuple(pa.filename.split('/')),pa.offset,pa.length,whole) |
| 100 | 87 exit(0) |
| 104 | 88 if pa.index: |
| 89 CDX=regex.compile('length": "([0-9]*)", "offset": "([0-9]*)", "filename": "crawl-data/([^/]*)/segments/([^/]*)/warc/(.*\.gz)"') | |
| 90 for l in sys.stdin: | |
| 91 m=CDX.search(l) | |
| 92 if m is None: | |
| 93 print("index line problem: \"%s\""%l.lstrip(),file=sys.stderr) | |
| 94 exit(2) | |
|
105
baf56ff538f8
convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
104
diff
changeset
|
95 f=FPAT%(m[3:6]) |
|
106
815b33c3254a
working with -x and rich directory structure
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
105
diff
changeset
|
96 process(pa,buf,f,int(m[2]),int(m[1]),whole) |
| 104 | 97 exit(0) |
| 100 | 98 if __name__ == "__main__": |
| 99 main() |