Mercurial > hg > cc > cirrus_work

annotate lib/python/cc/ix.py @ 396:10813bab9072 plus tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added test 9 for wat that pretty-prints json content
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 30 Jun 2026 16:50:56 +0100
parents 531a85706485
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/usr/bin/env python3
41
fa43c318749b correct comment
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 21
diff changeset
2 '''Extract response records from Common Crawl WARC-format files
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 given length, offset and filename triples.
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 Input one triple on command line, or
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 triples from stdin as tab-delimited lines
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 or complete cdx index lines.
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 In all cases by 'filename' is meant crawlid/segmentid/type/filename
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 Note that if no output flag(s) is/are given, the whole WARC record will be output, more efficiently than would be the case if -whb is given.'''
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 import sys, argparse, regex, os, shutil, io, gzip, time, shlex
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 from isal import igzip
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13 from subprocess import Popen, PIPE
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14 #import asyncio
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15
342
4759d6ac4625 update ROOT, fix deprecated patterns
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 298
diff changeset
16 HACK_USAGE=regex.compile(r'\[-x\]\n\s*\[length\] \[offset\] \[filename\]')
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17 BINOUT=sys.stdout.buffer
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18 FPAT="/%s/%s/orig/%s/%s"
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
20 CMD_PROC=None
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21 TMPFILENAME=None
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22
342
4759d6ac4625 update ROOT, fix deprecated patterns
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 298
diff changeset
23 ROOT="/work/dc007/shared/pod12/common_crawl"
4759d6ac4625 update ROOT, fix deprecated patterns
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 298
diff changeset
24
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25 class HackFormat(argparse.RawDescriptionHelpFormatter):
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26 def format_help(self):
234
36a9dc0d52d4 for debugging?
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 120
diff changeset
27 global FOO
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28 FOO=argparse.RawDescriptionHelpFormatter.format_help(self)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
29 return HACK_USAGE.sub('\n [ ( -x | length offset filename ) ]',
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
30 FOO)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
31
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
32 def process(options,buf,filename,offset,length,whole):
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
33 try:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
34 process0(options,buf,filename,offset,length,whole)
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
35 except Exception as e:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
36 if options.debug:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
37 import traceback
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
38 traceback.print_exc(file=sys.stderr)
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
39 else:
238
a4538bdfa92a fix error message
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 234
diff changeset
40 print("Process fail: %s, input:\n length=%s, offset=%s, file=%s"%(e,
a4538bdfa92a fix error message
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 234
diff changeset
41 length,
a4538bdfa92a fix error message
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 234
diff changeset
42 offset,
a4538bdfa92a fix error message
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 234
diff changeset
43 filename),
a4538bdfa92a fix error message
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 234
diff changeset
44 file=sys.stderr)
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
45 exit(3)
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
46
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
47 def process0(options,buf,filename,offset,length,whole):
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
48 global TMPFILENAME, TMPFILE
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
49 if options.save:
388
531a85706485 use USER in tmpfile creation,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 387
diff changeset
50 (tf,TMPFILENAME)=tempfile.mkstemp(dir="/tmp/"+(os.getenv('USER') or 'xyzzy'))
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
51 TMPFILE=open(tf,mode='wb')
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
52 if options.cmd and not options.process:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
53 launch(options.cmd)
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
54 process1(options,buf,filename,offset,length,whole)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
55 if options.save:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
56 TMPFILE.close()
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
57 if options.cmd:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
58 _output_subproc(bytes(TMPFILENAME,'utf-8'))
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
59 _output_subproc(b"\n")
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
60 else:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
61 BINOUT.write(bytes(TMPFILENAME,'utf-8'))
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
62 BINOUT.write(b"\n")
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
63 if options.cmd:
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
64 if not options.process:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
65 windup(filename,options,length)
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
66 if options.save:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
67 os.unlink(TMPFILENAME)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
68 TMPFILENAME=None
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
69 elif options.save:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
70 print("%s will need to be deleted"%TMPFILENAME,file=sys.stderr)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
71 TMPFILENAME=None
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
72
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
73 def launch(cmd):
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
74 global CMD_PROC, BINOUT
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
75 CMD_PROC=Popen(shlex.split(cmd),stdin=PIPE,bufsize=0)
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
76 BINOUT=CMD_PROC.stdin
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
77
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
78 def windup(length,offset,filename):
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
79 # Wind up subproc
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
80 BINOUT.close()
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
81 if CMD_PROC.wait()!=0: # could/should be async?
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
82 print("subproc of %s:%s:%s failed with %s"%(length,offset,filename,
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
83 CMD_PROC.returncode),
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
84 file=sys.stderr)
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
85
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
86 def _output_tmpfile(buf):
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
87 TMPFILE.write(buf)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
88
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
89 def _output_stdout(buf):
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
90 BINOUT.write(buf)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
91
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
92 def _output_subproc(buf):
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
93 toWrite=len(buf)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
94 while toWrite>0:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
95 toWrite -= BINOUT.write(buf)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
96
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
97 def process1(options,buf,filename,offset,length,whole):
342
4759d6ac4625 update ROOT, fix deprecated patterns
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 298
diff changeset
98 global ROOT
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
99 root=options.root
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
100 rfn=root+filename
342
4759d6ac4625 update ROOT, fix deprecated patterns
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 298
diff changeset
101 if root!=ROOT:
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
102 # Support using ramdisk or other local disk as a faster cached
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
103 if not os.path.exists(rfn):
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
104 if not os.path.exists(os.path.dirname(rfn)):
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
105 os.makedirs(os.path.dirname(rfn))
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
106 with io.FileIO('/beegfs/common_crawl'+filename,'r') as infile, \
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
107 io.FileIO(rfn,'w') as outfile:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
108 #shutil.copyfileobj(infile,outfile,128*1024*1024)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
109 while True:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
110 l=infile.readinto(buf)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
111 if l==0:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
112 break
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
113 outfile.write(memoryview(buf)[:l])
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
114 file=open(rfn,'rb',0)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
115 file.seek(offset)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
116 bv=memoryview(buf)[:length]
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
117 nb=file.readinto(bv)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
118 file.close()
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
119 if nb!=length:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
120 raise ValueError("Chunk read losing: %s, got %s expected %s at %s"%(file.name,
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
121 nb,length,offset))
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
122 if whole and options.zipped:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
123 _output(bv)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
124 return
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
125 gzip_chunk = io.BytesIO(bv)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
126 uv=memoryview(buf)[length:]
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
127 with igzip.IGzipFile(fileobj=gzip_chunk) as gzip_fin:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
128 ll=0
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
129 while True:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
130 l=gzip_fin.readinto(uv)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
131 if not l:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
132 break
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
133 ll+=l
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
134 cb=memoryview(uv)[:ll]
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
135 if whole:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
136 _output(cb)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
137 return
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
138 # Only output parts (0 = WARC header, 1 = HTTP header, 2 = body) that are wanted
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
139 state=0
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
140 tr=None # Was this record truncated?
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
141 bl=None # for HTTP Content-Length for the length of the body?
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
142 # Could we make this faster by working purely within the cb memoryview?
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
143 # It would be messy, but avoid copying huge amounts
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
144 # The outer loop would just be something like
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
145 # clbv=memoryview(bytearray(b"Content-Length: "))
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
146 # i=s=0
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
147 # while i<ll:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
148 # if cb[i]==10: # need to handle \r\n
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
149 # L=cb[s:i]
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
150 # s=i=i+1
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
151 # if L[:16]==clbv:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
152 # wl=int(L[16:])
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
153 # else:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
154 # i+=1
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
155 #
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
156 with io.BytesIO(cb) as clear_text:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
157 for L in clear_text:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
158 if state==0:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
159 # WARC header
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
160 if L.startswith(b"Content-Length: "):
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
161 wl=int(L[16:].rstrip())
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
162 elif L.startswith(b"WARC-Truncated: "):
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
163 tr=L[16:].rstrip()
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
164 tr="EMPTY" if tr=="" else tr
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
165 elif L==b"" or L.startswith(b"\r"): # for idempotency
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
166 # Blank line, WARC header is finished
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
167 if not (options.headers or options.body):
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
168 return
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
169 state=1
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
170 # Note we preserve the empty line
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
171 if options.warc:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
172 _output(L)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
173 continue
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
174 if state==1:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
175 # HTTP header
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
176 wl -= len(L)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
177 if not (L==b"" or L.startswith(b"\r")):
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
178 # Non-blank, it's a header
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
179 (h,_,v)=L.partition(b": ")
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
180 if bl is None and (h==b"Content-Length"):
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
181 bl=int(v)
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
182 if options.headers:
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
183 if isinstance(options.headers,dict):
386
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
184 if (h:=h.lower()) in options.headers:
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
185 # note
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
186 # a) that this is seriously inefficient as it involves a string
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
187 # copy for every header line;
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
188 # b) it will probably confuse users as they will always see
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
189 # lower-case, which is almost never in files, but at lease
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
190 # they will not have to do another lower() in order to look
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
191 # things up in the result
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
192 options.headers[h]=v
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
193 else:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
194 _output(L)
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
195 else:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
196 # Blank line, HTTP header is finished
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
197 if isinstance(options.headers,dict):
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
198 _output(bytes(str(options.headers),'utf-8'))
388
531a85706485 use USER in tmpfile creation,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 387
diff changeset
199 if not options.body:
531a85706485 use USER in tmpfile creation,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 387
diff changeset
200 _output(b'\n')
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
201 if not options.body:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
202 return
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
203 if options.headers:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
204 _output(L)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
205 state=2
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
206 # The above is just for sanity, because we do _not_
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
207 # continue with the outer loop,
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
208 # since we can now block-output the entire rest of the
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
209 # input buffer.
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
210 if bl is not None:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
211 if bl!=wl:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
212 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
213 (length,offset,filename,wl,bl,tr),file=sys.stderr)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
214 # HTTP body
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
215 balance=clear_text.tell()
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
216 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
217 # Output whatever is left
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
218 _output(cb[balance:balance+wl])
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
219 return
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
220
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
221 def main():
342
4759d6ac4625 update ROOT, fix deprecated patterns
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 298
diff changeset
222 global _output,TMPFILE,TMPFILENAME,tempfile,ROOT
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
223 parser = argparse.ArgumentParser(
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
224 description='''Extract records from warc files given length, offset and file triples.
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
225 Input one triple on command line, or
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
226 triples from stdin as tab-delimited lines
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
227 or complete cdx index lines.
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
228 In all cases by 'filename' is meant crawlid/segmentid/type/filename''',
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
229 epilog='''Note that if no output flag(s) is/are given,
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
230 the whole WARC record will be output, more efficiently than
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
231 would be the case if all three flags were given.''',
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
232 add_help=False,
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
233 conflict_handler='resolve',
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
234 formatter_class=HackFormat
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
235 )
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
236 fphelp=('format string for turning 4 filename components into a path, must contain %%s exactly 4 times,\ndefault is "%s"'%FPAT).replace('%s','%%s')
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
237 parser.add_argument('--help',help='Show help',action='help')
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
238 parser.add_argument('-d','--debug',help='Debug output',action='store_true')
386
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
239 parser.add_argument('-w','--warc',help='Output Warc headers',
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
240 action='store_true')
386
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
241 parser.add_argument('-h','--headers',help='Process HTTP headers: if no (,-separated) args, output as read. With args, collect into dict with named values. Keys in dict are lower-cased, per HTTP case-insensitivity.',
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
242 nargs='?',default=None,const=True)
386
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
243 parser.add_argument('-b','--body',help='Output HTTP body',
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
244 action='store_true')
386
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
245 parser.add_argument('-c','--cmd',help='Pipe each result thru CMD')
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
246 parser.add_argument('-p','--process',help='with -c, launches CMD only once',
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
247 action='store_true')
386
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
248 parser.add_argument('-m','--module.function',help='Call module.function with a stream'),
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
249 parser.add_argument('-s','--save',action='store_true',
386
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
250 help="Write to a temporary file and output the name")
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
251 parser.add_argument('-f','--fpath',
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
252 help=fphelp,
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
253 default=FPAT)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
254 parser.add_argument('-r','--root',nargs='?',
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
255 help='File path root, create a copy there if necessary',
342
4759d6ac4625 update ROOT, fix deprecated patterns
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 298
diff changeset
256 default=ROOT),
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
257 parser.add_argument('-z','--zipped',
386
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
258 help="Output raw gzipped record, ignored if any of -bhw supplied",
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
259 action='store_true')
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
260 sg=parser.add_mutually_exclusive_group()
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
261 sg.add_argument('-x','--index',
386
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
262 help='Take lines of triples from a cdx index file as input',
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
263 action='store_true')
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
264 sg.add_argument('length',type=int,
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
265 help='length in bytes of gzipped record',
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
266 nargs='?')
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
267 parser.add_argument('offset',type=int,
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
268 help='start position in bytes of gzipped record',
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
269 nargs='?')
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
270 parser.add_argument('filename',
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
271 help='pathname of gzipped Common Crawl WARC-format file',
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
272 nargs='?')
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
273 # Hack the order of optional and positional in the help output
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
274 parser._action_groups.sort(key=lambda g:g.title)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
275 #parser.print_help()
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
276 pa=parser.parse_args(sys.argv[1:])
387
4eda36618ca2 remove debug print
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 386
diff changeset
277 #print(pa,file=sys.stderr)
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
278 if pa.length is not None:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
279 # We have to enforce our own check..
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
280 if pa.offset is None or pa.filename is None:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
281 parser.error("length, offset and filename must all be supplied together")
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
282 if isinstance(pa.headers,str):
386
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
283 pa.headers=dict((bytes(k.lower(),'utf-8'),None) for k in pa.headers.split(','))
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
284 # N.B. ^^^^^
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
285 if pa.__getattribute__("module.function") is not None:
69264e6389e2 tweak some arg doc strings, give NotImpl warning for -m
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 342
diff changeset
286 parser.error("-m,--module.function not supported yet")
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
287 buf=bytearray(128*1024*1024)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
288
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
289 whole=not (pa.warc or pa.headers or pa.body)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
290 if pa.save:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
291 _output=_output_tmpfile
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
292 import tempfile
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
293 elif pa.cmd:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
294 _output = _output_subproc
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
295 else:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
296 _output = _output_stdout
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
297 if pa.cmd and pa.process:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
298 launch(pa.cmd)
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
299 # three different ways to process
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
300 if pa.index:
342
4759d6ac4625 update ROOT, fix deprecated patterns
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 298
diff changeset
301 CDX=regex.compile(r'length": "([0-9]*)", "offset": "([0-9]*)", "filename": "crawl-data/([^/]*)/segments/([^/]*)/(warc|crawldiagnostics|robotstxt)/(.*\.gz)"') # robotstxt works?
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
302 for l in sys.stdin:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
303 m=CDX.search(l)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
304 if m is None:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
305 if l.find('/robotstxt/')>-1:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
306 continue
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
307 print("index line problem: \"%s\""%l,file=sys.stderr,end='')
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
308 exit(2)
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
309 filename=pa.fpath%(m[3:7])
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
310 process(pa,buf,filename,
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
311 int(offset:=m[2]),int(length:=m[1]),whole)
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
312 elif pa.length is not None:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
313 print(pa.filename,file=sys.stderr)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
314 process(pa,buf,pa.fpath%tuple(pa.filename.split('/')),
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
315 pa.offset,pa.length,whole)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
316 else:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
317 print("Reading length, offset, filename tab-delimited triples from stdin...",
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
318 file=sys.stderr)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
319 for l in sys.stdin:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
320 try:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
321 (length,offset,filename)=l.rstrip().split('\t')
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
322 length=int(length)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
323 offset=int(offset)
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
324 except ValueError as e:
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
325 parser.error('Invalid input line: %s\n "%s"'%(e,l))
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
326 process(pa,buf,pa.fpath%tuple(filename.split('/')),
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
327 offset,length,whole)
21
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
328 # processing done one way or another
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
329 if pa.cmd and pa.process:
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
330 windup(length,offset,filename)
cbac7dfe2f24 interpolate process0, support permanent subproc
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 18
diff changeset
331 # if pa.save and pa.process, deleting temp files is down to cmd
18
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
332 if __name__ == "__main__":
046dbe557911 write to tmp file implemented
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
333 main()