Mercurial > hg > cc > cirrus_home

annotate bin/ix.py @ 112:bcea0410143c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
(none)
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 26 Apr 2021 17:17:38 +0000
parents 3119bca71181
children 6467024cd072
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
105
baf56ff538f8 convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 104
diff changeset
1 #!/usr/bin/env python3
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 '''Extract request records from Common Crawl WARC-format files
105
baf56ff538f8 convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 104
diff changeset
3 given length, offset and filename triples.
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 Input one triple on command line, or
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 triples from stdin as tab-delimited lines
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 or complete cdx index lines.
105
baf56ff538f8 convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 104
diff changeset
7 In all cases by 'filename' is meant crawlid/segmentid/filename
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8
98
1a4c5fdc2923 help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 97
diff changeset
9 Note that if no output flag(s) is/are given, the whole WARC record will be output, more efficiently than would be the case if -whb is given.'''
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10
108
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
11 import sys, argparse, regex, os, shutil, io, gzip, time
110
f148c2366faa back to IGzipFile
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 109
diff changeset
12 from isal import igzip
f148c2366faa back to IGzipFile
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 109
diff changeset
13 #from subprocess import Popen, PIPE
108
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
14 #import asyncio
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
15
98
1a4c5fdc2923 help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 97
diff changeset
16 HACK_USAGE=regex.compile('\[-x\]\n\s*\[length\] \[offset\] \[filename\]')
105
baf56ff538f8 convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 104
diff changeset
17 BINOUT=sys.stdout.buffer
107
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
18 FPAT="/%s/%s/orig/warc/%s"
94
d60073ec798a just strugging with argparse
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
20 class HackFormat(argparse.RawDescriptionHelpFormatter):
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
21 def format_help(self):
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
22 global FOO
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
23 FOO=argparse.RawDescriptionHelpFormatter.format_help(self)
98
1a4c5fdc2923 help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 97
diff changeset
24 return HACK_USAGE.sub('\n [ ( -x | length offset filename ) ]',
1a4c5fdc2923 help format hacking done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 97
diff changeset
25 FOO)
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
26
107
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
27 def process(options,buf,root,filename,offset,length,whole):
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
28 rfn=root+filename
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
29 if root!="/beegfs/common_crawl":
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
30 if not os.path.exists(rfn):
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
31 if not os.path.exists(os.path.dirname(rfn)):
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
32 os.makedirs(os.path.dirname(rfn))
108
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
33 with io.FileIO('/beegfs/common_crawl'+filename,'r') as infile, \
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
34 io.FileIO(rfn,'w') as outfile:
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
35 #shutil.copyfileobj(infile,outfile,128*1024*1024)
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
36 while True:
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
37 l=infile.readinto(buf)
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
38 if l==0:
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
39 break
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
40 outfile.write(memoryview(buf)[:l])
107
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
41 file=open(rfn,'rb',0)
111
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
42 file.seek(offset)
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
43 bv=memoryview(buf)[:length]
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
44 nb=file.readinto(bv)
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
45 file.close()
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
46 if nb!=length:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
47 print("losing",file.name,length,nb,file=sys.stderr)
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
48 if whole and options.zipped:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
49 BINOUT.write(bv)
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
50 return
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
51 gzip_chunk = io.BytesIO(bv)
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
52 uv=memoryview(buf)[length:]
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
53 with igzip.IGzipFile(fileobj=gzip_chunk) as gzip_fin:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
54 ll=0
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
55 while True:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
56 l=gzip_fin.readinto(uv)
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
57 if not l:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
58 break
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
59 ll+=l
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
60 cb=memoryview(uv)[:ll]
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
61 if whole:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
62 BINOUT.write(cb)
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
63 return
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
64 # only parts wanted
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
65 # Note that _unlike the above_ this strips the ^M from the output lines
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
66 # so we are _not_ idempotent
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
67 state=0
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
68 tr=None
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
69 with io.TextIOWrapper(io.BytesIO(cb),encoding='iso-8859-1',
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
70 newline='\r\n') as clear_text:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
71 for L in clear_text:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
72 if state==0:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
73 # WARC header
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
74 if L.startswith("Content-Length: "):
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
75 wl=int(L[16:].rstrip())
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
76 elif L.startswith("WARC-Truncated: "):
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
77 tr=L[16:].rstrip()
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
78 tr="EMPTY" if tr=="" else tr
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
79 elif L.startswith("\r"): # make us idempotent
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
80 if not (options.headers or options.body):
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
81 return
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
82 state=1
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
83 bl=None
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
84 if options.warc:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
85 # preserve the empty line
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
86 print()
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
87 continue
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
88 if options.warc:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
89 print(L.rstrip())
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
90 continue
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
91 if state==1:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
92 # HTTP header
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
93 wl -= len(L)
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
94 if L.startswith("Content-Length: "):
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
95 bl=int(L[16:].rstrip())
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
96 elif L=="" or L.startswith("\r"):
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
97 if not options.body:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
98 return
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
99 state=2
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
100 if options.headers:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
101 # preserve the empty line
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
102 print()
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
103 if bl is not None:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
104 if bl!=wl:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
105 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
106 (length,offset,filename,wl,bl,tr),file=sys.stderr)
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
107 continue
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
108 if options.headers:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
109 print(L.rstrip())
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
110 continue
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
111 # HTTP body
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
112 if options.body:
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
113 sys.stdout.flush()
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
114 BINOUT.write(cb[clear_text.tell():])
3119bca71181 warc and headers parts working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 110
diff changeset
115 return
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
116
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
117 def main():
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
118 parser = argparse.ArgumentParser(
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
119 description='''Extract records from warc files given length, offset and file triples.
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
120 Input one triple on command line, or
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
121 triples from stdin as tab-delimited lines
105
baf56ff538f8 convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 104
diff changeset
122 or complete cdx index lines.
baf56ff538f8 convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 104
diff changeset
123 In all cases by 'filename' is meant crawlid/segmentid/filename''',
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
124 epilog='''Note that if no output flag(s) is/are given,
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
125 the whole WARC record will be output, more efficiently than
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
126 would be the case if all three flags were given.''',
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
127 add_help=False,
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
128 conflict_handler='resolve',
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
129 formatter_class=HackFormat
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
130 )
97
2b880f2ce894 basic help format hacking works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 94
diff changeset
131
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
132 parser.add_argument('--help',help='Show help',action='help')
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
133 parser.add_argument('-d','--debug',help='Debug output',action='store_true')
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
134 parser.add_argument('-w','--warc',help='output WARC headers',
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
135 action='store_true')
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
136 parser.add_argument('-h','--headers',help='output HTTP headers',
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
137 action='store_true')
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
138 parser.add_argument('-b','--body',help='output HTTP body',
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
139 action='store_true')
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
140 parser.add_argument('-c','--cmd',help='pipes each result thru CMD')
107
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
141 parser.add_argument('-r','--root',nargs='?',
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
142 help='File path root, create a copy there if necessary',
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
143 default='/beegfs/common_crawl'),
108
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
144 parser.add_argument('-z','--zipped',
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
145 help="output raw gzipped record, ignored if any of -bhw supplied",
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
146 action='store_true')
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
147 sg=parser.add_mutually_exclusive_group()
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
148 sg.add_argument('-x','--index',
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
149 help='take lines of triples from a cdx index file as input',
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
150 action='store_true')
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
151 sg.add_argument('length',type=int,
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
152 help='length in bytes of gzipped record',
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
153 nargs='?')
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
154 parser.add_argument('offset',type=int,
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
155 help='start position in bytes of gzipped record',
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
156 nargs='?')
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
157 parser.add_argument('filename',
105
baf56ff538f8 convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 104
diff changeset
158 help='pathname of gzipped Common Crawl WARC-format file',
baf56ff538f8 convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 104
diff changeset
159 nargs='?')
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
160 # Hack the order of optional and positional in the help output
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
161 parser._action_groups.sort(key=lambda g:g.title)
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
162 #parser.print_help()
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
163 pa=parser.parse_args(sys.argv[1:])
106
815b33c3254a working with -x and rich directory structure
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 105
diff changeset
164 #print(pa,file=sys.stderr)
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
165 if pa.length is not None:
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
166 # We have to enforce our own check..
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
167 if pa.offset is None or pa.filename is None:
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
168 parser.error("length, offset and filename must all be supplied together")
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
169
108
9e5b117dc461 using Popen to run igzip (also not great)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 107
diff changeset
170 buf=bytearray(128*1024*1024)
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
171
104
61122560ae0c -x barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 100
diff changeset
172 whole=not (pa.warc or pa.headers or pa.body)
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
173 if pa.length is not None:
107
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
174 process(pa,buf,pa.root,FPAT%list(pa.filename.split('/')),
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
175 pa.offset,pa.length,whole)
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
176 exit(0)
104
61122560ae0c -x barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 100
diff changeset
177 if pa.index:
61122560ae0c -x barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 100
diff changeset
178 CDX=regex.compile('length": "([0-9]*)", "offset": "([0-9]*)", "filename": "crawl-data/([^/]*)/segments/([^/]*)/warc/(.*\.gz)"')
61122560ae0c -x barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 100
diff changeset
179 for l in sys.stdin:
61122560ae0c -x barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 100
diff changeset
180 m=CDX.search(l)
61122560ae0c -x barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 100
diff changeset
181 if m is None:
61122560ae0c -x barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 100
diff changeset
182 print("index line problem: \"%s\""%l.lstrip(),file=sys.stderr)
61122560ae0c -x barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 100
diff changeset
183 exit(2)
105
baf56ff538f8 convert to rich directory structure per 2019-35
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 104
diff changeset
184 f=FPAT%(m[3:6])
107
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
185 process(pa,buf,pa.root,f,
007f35b9df9c added support for copying to/using /dev/shm or /tmp
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 106
diff changeset
186 int(m[2]),int(m[1]),whole)
104
61122560ae0c -x barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 100
diff changeset
187 exit(0)
100
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
188 if __name__ == "__main__":
f7623dbd8eb5 bare minimum working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 99
diff changeset
189 main()