|
138
|
1 #!/usr/bin/env python3
|
|
|
2 '''Stream a warc format file, invoking a callback on each part.
|
|
|
3 Callback can be limited by WARC-Type'''
|
|
|
4 import sys
|
|
|
5
|
|
|
6 def warc(callback,types=['response']):
|
|
|
7 nb=0
|
|
|
8 stream=open(sys.stdin.fileno(),'rb',0)
|
|
|
9 bufsize=128*1024*1024
|
|
|
10 buf=bytearray(128*1024*1024)
|
|
|
11 l=b'\r\n'
|
|
|
12 while True:
|
|
|
13 while l==b'\r\n':
|
|
|
14 l=stream.readline()
|
|
|
15 nb+=len(l)
|
|
|
16 if l!=b'WARC/1.0\r\n':
|
|
|
17 if l==0:
|
|
|
18 return
|
|
|
19 raise ValueError("Not a WARC file? At %s: %s[%s]"%(nb-len(l),
|
|
|
20 l.decode('latin-1'),len(l)))
|
|
|
21 wtype=None
|
|
|
22 length=None
|
|
|
23 while l!=b'\r\n':
|
|
|
24 l=stream.readline()
|
|
|
25 nb+=len(l)
|
|
|
26 if l.startswith(b'WARC-Type: '):
|
|
|
27 wtype = l[11:-2]
|
|
|
28 elif l.startswith(b'Content-Length: '):
|
|
|
29 length = int(l[16:])
|
|
|
30 bv=memoryview(buf)[:length]
|
|
|
31 ii=0
|
|
|
32 while True:
|
|
|
33 i=stream.readinto(bv)
|
|
|
34 ii+=i
|
|
|
35 if ii>=length:
|
|
|
36 break
|
|
|
37 bv=memoryview(buf)[ii:length]
|
|
|
38 if ii!=length:
|
|
|
39 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length))
|
|
|
40 nb+=length
|
|
|
41 if wtype in types:
|
|
|
42 callback(wtype,bv)
|
|
|
43 if whole and options.zipped:
|
|
|
44 _output(bv)
|
|
|
45 return
|
|
|
46 gzip_chunk = io.BytesIO(bv)
|
|
|
47 uv=memoryview(buf)[length:]
|
|
|
48 with igzip.IGzipFile(fileobj=gzip_chunk) as gzip_fin:
|
|
|
49 ll=0
|
|
|
50 while True:
|
|
|
51 l=gzip_fin.readinto(uv)
|
|
|
52 if not l:
|
|
|
53 break
|
|
|
54 ll+=l
|
|
|
55 cb=memoryview(uv)[:ll]
|
|
|
56 if whole:
|
|
|
57 _output(cb)
|
|
|
58 return
|
|
|
59 # Only output parts (0 = WARC header, 1 = HTTP header, 2 = body) that are wanted
|
|
|
60 state=0
|
|
|
61 tr=None # Was this record truncated?
|
|
|
62 bl=None # for HTTP Content-Length for the length of the body?
|
|
|
63 with io.BytesIO(cb) as clear_text:
|
|
|
64 for L in clear_text:
|
|
|
65 if state==0:
|
|
|
66 # WARC header
|
|
|
67 if L.startswith(b"Content-Length: "):
|
|
|
68 wl=int(L[16:].rstrip())
|
|
|
69 elif L.startswith(b"WARC-Truncated: "):
|
|
|
70 tr=L[16:].rstrip()
|
|
|
71 tr="EMPTY" if tr=="" else tr
|
|
|
72 elif L==b"" or L.startswith(b"\r"): # for idempotency
|
|
|
73 # Blank line, WARC header is finished
|
|
|
74 if not (options.headers or options.body):
|
|
|
75 return
|
|
|
76 state=1
|
|
|
77 # Note we preserve the empty line
|
|
|
78 if options.warc:
|
|
|
79 _output(L)
|
|
|
80 continue
|
|
|
81 if state==1:
|
|
|
82 # HTTP header
|
|
|
83 wl -= len(L)
|
|
|
84 if not (L==b"" or L.startswith(b"\r")):
|
|
|
85 # Non-blank, it's a header
|
|
|
86 if bl is None and L.startswith(b"Content-Length: "):
|
|
|
87 bl=int(L[16:].rstrip())
|
|
|
88 if options.headers:
|
|
|
89 _output(L)
|
|
|
90 else:
|
|
|
91 # Blank line, HTTP header is finished
|
|
|
92 if not options.body:
|
|
|
93 return
|
|
|
94 if options.headers:
|
|
|
95 _output(L)
|
|
|
96 state=2
|
|
|
97 # The above is just for sanity, because we do _not_
|
|
|
98 # continue with the outer loop,
|
|
|
99 # since we can now block-output the entire rest of the
|
|
|
100 # input buffer.
|
|
|
101 if bl is not None:
|
|
|
102 if bl!=wl:
|
|
|
103 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
|
|
|
104 (length,offset,filename,wl,bl,tr),file=sys.stderr)
|
|
|
105 # HTTP body
|
|
|
106 balance=clear_text.tell()
|
|
|
107 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
|
|
|
108 # Output whatever is left
|
|
|
109 _output(cb[balance:balance+wl])
|
|
|
110 return
|
|
|
111
|
|
|
112 OUT=open(sys.stdout.fileno(),'wb')
|
|
|
113
|
|
|
114 def showme(wtype,buf):
|
|
|
115 OUT.write(buf)
|
|
|
116
|
|
|
117 warc(showme,[b'metadata'])
|
