Mercurial > hg > cc > cirrus_work

comparison bin/warc.py @ 62:11cbaee8bbc8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Test 2 works with parts=1,2,3. Tests 3 and 4 work; Test 1 works with parts=1, gives correct output for warcinfo and metadata with parts=1,2,3.
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Fri, 14 Jul 2023 17:38:54 +0100
parents f182d09ad1cd
children 75f1d3bc60d9
comparison
equal deleted inserted replaced
61:f182d09ad1cd 62:11cbaee8bbc8
4 part''' 4 part'''
5 5
6 import sys,io 6 import sys,io
7 from isal import igzip 7 from isal import igzip
8 8
9 RESP = b'response'
10 REQ = b'request'
11 META = b'metadata'
12 INFO = b'warcinfo'
13
9 def warc(filename,callback,types=['response'],whole=False,parts=7,debug=False): 14 def warc(filename,callback,types=['response'],whole=False,parts=7,debug=False):
10 '''parts is a bit-mask: 15 '''parts is a bit-mask:
11 1 for warc header; 16 1 for warc header;
12 2 for req/resp HTTP header, warcinfo/metadata features; 17 2 for req/resp HTTP header, warcinfo/metadata features;
13 4 for req/resp body''' 18 4 for req/resp body'''
19 # should do some sanity checking wrt parts and types
14 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types] 20 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types]
15 nb=0 21 nb=0
16 if filename.endswith(".gz"): 22 if filename.endswith(".gz"):
17 stream=igzip.IGzipFile(filename=filename) 23 stream=igzip.IGzipFile(filename=filename)
18 else: 24 else:
50 tr=b"EMPTY" 56 tr=b"EMPTY"
51 else: 57 else:
52 tr=bytes(bufView[bp+16:eol-2]) 58 tr=bytes(bufView[bp+16:eol-2])
53 elif buf.startswith(b'WARC-Type: ',bp): 59 elif buf.startswith(b'WARC-Type: ',bp):
54 if buf.startswith(b's',bp+13): 60 if buf.startswith(b's',bp+13):
55 wtype = b'response' 61 wtype = RESP
56 elif buf.startswith(b'q',bp+13): 62 elif buf.startswith(b'q',bp+13):
57 wtype = b'request' 63 wtype = REQ
58 elif buf.startswith(b'm',bp+11): 64 elif buf.startswith(b'm',bp+11):
59 wtype = b'metadata' 65 wtype = META
60 elif buf.startswith(b'w',bp+11): 66 elif buf.startswith(b'w',bp+11):
61 wtype = b'warcinfo' 67 wtype = INFO
62 else: 68 else:
63 raise ValueError("Unknown WARC-Type: %s at %s"%( 69 raise ValueError("Unknown WARC-Type: %s at %s"%(
64 bytes(bufView[bp+11:eol-2]), 70 bytes(bufView[bp+11:eol-2]),
65 fpos-(bl-bp))) 71 fpos-(bl-bp)))
66 bp=eol 72 bp=eol
99 if nb<spaceToFill: 105 if nb<spaceToFill:
100 done=True 106 done=True
101 if wtype not in types: 107 if wtype not in types:
102 continue 108 continue
103 if (wtype in types): 109 if (wtype in types):
110 # Output whole or part 1 as required
104 if whole: 111 if whole:
105 bp+=length 112 bp+=length
106 OUT=callback(wtype,bufView[start_1:bp],7) 113 OUT=callback(wtype,bufView[start_1:bp],7)
107 continue 114 continue
108 elif (parts & 1): 115 elif (parts & 1):
109 OUT=callback(wtype,bufView[start_1:eol],1) 116 OUT=callback(wtype,bufView[start_1:eol],1)
110 if parts!=1: 117 if parts!=1:
111 bv=bufView[start_2:start_2+length] 118 while buf.startswith(b'\r\n',bp):
112 ii=0 119 bp+=2
113 while True and not stream.closed: 120 start_2=bp
114 if (i:=stream.readinto(bv))==0: 121 eob=bp+length
115 break 122 while buf.startswith(b'\r\n',eob-2):
116 ii+=i 123 eob-=2
117 if ii>=length: 124 bv=bufView[start_2:eob]
118 break 125 # Only output parts (2 = HTTP header, 4 = body) that are wanted
119 bv=memoryview(buf)[start_2+ii:start_2+length] 126 if parts & 2:
120 if ii!=length: 127 if wtype is META or wtype is INFO:
121 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length)) 128 # rest of the part
122 nb+=length 129 OUT=callback(wtype,bv,2)
123 if wtype in types: 130 if parts & 4:
124 if whole: 131 for L in rec_text:
125 callback(wtype,bufView[0:start_2+length],7) 132 if state==2:
126 continue 133 # HTTP header
127 # Only output parts (1 = WARC header, 2 = HTTP header, 4 = body) that are wanted 134 wl -= len(L)
128 bl=None # for HTTP Content-Length for the length of the body? 135 if not (L==b"" or L.startswith(b"\r")):
129 L_start=start_2 136 # Non-empty, it's (a continuation of) a header
130 state=2 137 if bl is None and L.startswith(b"Content-Length: "):
131 bv=memoryview(buf)[start_2:start_2+length] 138 bl=int(L[16:].rstrip())
132 with io.BytesIO(bv) as rec_text: 139 else:
133 for L in rec_text: 140 # Blank line, HTTP header is finished
134 if state==2: 141 if parts & 2:
135 # HTTP header 142 callback(wtype,bufView[start_2:start_2+L_start],2)
136 wl -= len(L) 143 state=4
137 if not (L==b"" or L.startswith(b"\r")): 144 # The above is just for sanity, because we do _not_
138 # Non-empty, it's (a continuation of) a header 145 # continue with the outer loop,
139 if bl is None and L.startswith(b"Content-Length: "): 146 # since we can now block-output the entire rest of the
140 bl=int(L[16:].rstrip()) 147 # input buffer.
141 else: 148 if bl is not None:
142 # Blank line, HTTP header is finished 149 if bl!=wl:
143 if parts & 2: 150 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
144 callback(wtype,bufView[start_2:start_2+L_start],2) 151 (length,offset,filename,wl,bl,tr),file=sys.stderr)
145 state=4 152 # HTTP body
146 # The above is just for sanity, because we do _not_ 153 balance=start_2+rec_text.tell()
147 # continue with the outer loop, 154 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
148 # since we can now block-output the entire rest of the 155 # Output whatever is left
149 # input buffer. 156 if parts & 4:
150 if bl is not None: 157 callback(wtype,bufView[balance:balance+wl],4)
151 if bl!=wl: 158 state=1
152 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
153 (length,offset,filename,wl,bl,tr),file=sys.stderr)
154 # HTTP body
155 balance=start_2+rec_text.tell()
156 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
157 # Output whatever is left
158 if parts & 4:
159 callback(wtype,bufView[balance:balance+wl],4)
160 state=1
161 159
162 L_start=rec_text.tell() 160 L_start=rec_text.tell()
163 bp+=length 161 bp+=length
164 #print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr) 162 #print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr)
165 #while not buf.startswith(b'\r\n',bp): 163 #while not buf.startswith(b'\r\n',bp):