Mercurial > hg > cc > cirrus_work

comparison bin/warc.py @ 61:f182d09ad1cd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
whole working
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Fri, 14 Jul 2023 12:08:09 +0100
parents 7b68c3ebc35a
children 11cbaee8bbc8
comparison
equal deleted inserted replaced
60:7b68c3ebc35a 61:f182d09ad1cd
5 5
6 import sys,io 6 import sys,io
7 from isal import igzip 7 from isal import igzip
8 8
9 def warc(filename,callback,types=['response'],whole=False,parts=7,debug=False): 9 def warc(filename,callback,types=['response'],whole=False,parts=7,debug=False):
10 '''parts is a bit-mask:
11 1 for warc header;
12 2 for req/resp HTTP header, warcinfo/metadata features;
13 4 for req/resp body'''
10 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types] 14 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types]
11 nb=0 15 nb=0
12 if filename.endswith(".gz"): 16 if filename.endswith(".gz"):
13 stream=igzip.IGzipFile(filename=filename) 17 stream=igzip.IGzipFile(filename=filename)
14 else: 18 else:
96 done=True 100 done=True
97 if wtype not in types: 101 if wtype not in types:
98 continue 102 continue
99 if (wtype in types): 103 if (wtype in types):
100 if whole: 104 if whole:
101 pass # buf[bp:(bp:=bp+ln)]=l @fixme 105 bp+=length
106 OUT=callback(wtype,bufView[start_1:bp],7)
107 continue
102 elif (parts & 1): 108 elif (parts & 1):
103 OUT=callback(wtype,bufView[start_1:eol],1) 109 OUT=callback(wtype,bufView[start_1:eol],1)
104 if parts!=1: 110 if parts!=1:
105 # everything from bv= goes here 111 bv=bufView[start_2:start_2+length]
106 pass 112 ii=0
113 while True and not stream.closed:
114 if (i:=stream.readinto(bv))==0:
115 break
116 ii+=i
117 if ii>=length:
118 break
119 bv=memoryview(buf)[start_2+ii:start_2+length]
120 if ii!=length:
121 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length))
122 nb+=length
123 if wtype in types:
124 if whole:
125 callback(wtype,bufView[0:start_2+length],7)
126 continue
127 # Only output parts (1 = WARC header, 2 = HTTP header, 4 = body) that are wanted
128 bl=None # for HTTP Content-Length for the length of the body?
129 L_start=start_2
130 state=2
131 bv=memoryview(buf)[start_2:start_2+length]
132 with io.BytesIO(bv) as rec_text:
133 for L in rec_text:
134 if state==2:
135 # HTTP header
136 wl -= len(L)
137 if not (L==b"" or L.startswith(b"\r")):
138 # Non-empty, it's (a continuation of) a header
139 if bl is None and L.startswith(b"Content-Length: "):
140 bl=int(L[16:].rstrip())
141 else:
142 # Blank line, HTTP header is finished
143 if parts & 2:
144 callback(wtype,bufView[start_2:start_2+L_start],2)
145 state=4
146 # The above is just for sanity, because we do _not_
147 # continue with the outer loop,
148 # since we can now block-output the entire rest of the
149 # input buffer.
150 if bl is not None:
151 if bl!=wl:
152 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
153 (length,offset,filename,wl,bl,tr),file=sys.stderr)
154 # HTTP body
155 balance=start_2+rec_text.tell()
156 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
157 # Output whatever is left
158 if parts & 4:
159 callback(wtype,bufView[balance:balance+wl],4)
160 state=1
161
162 L_start=rec_text.tell()
107 bp+=length 163 bp+=length
108 print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr) 164 #print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr)
109 #while not buf.startswith(b'\r\n',bp): 165 #while not buf.startswith(b'\r\n',bp):
110 continue 166 continue
111 bv=memoryview(buf)[start_2:start_2+length]
112 ii=0
113 while True and not stream.closed:
114 if (i:=stream.readinto(bv))==0:
115 break
116 ii+=i
117 if ii>=length:
118 break
119 bv=memoryview(buf)[start_2+ii:start_2+length]
120 if ii!=length:
121 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length))
122 nb+=length
123 if wtype in types:
124 if whole:
125 callback(wtype,bufView[0:start_2+length],7)
126 continue
127 # Only output parts (1 = WARC header, 2 = HTTP header, 4 = body) that are wanted
128 bl=None # for HTTP Content-Length for the length of the body?
129 L_start=start_2
130 state=2
131 bv=memoryview(buf)[start_2:start_2+length]
132 with io.BytesIO(bv) as rec_text:
133 for L in rec_text:
134 if state==2:
135 # HTTP header
136 wl -= len(L)
137 if not (L==b"" or L.startswith(b"\r")):
138 # Non-empty, it's (a continuation of) a header
139 if bl is None and L.startswith(b"Content-Length: "):
140 bl=int(L[16:].rstrip())
141 else:
142 # Blank line, HTTP header is finished
143 if parts & 2:
144 callback(wtype,bufView[start_2:start_2+L_start],2)
145 state=4
146 # The above is just for sanity, because we do _not_
147 # continue with the outer loop,
148 # since we can now block-output the entire rest of the
149 # input buffer.
150 if bl is not None:
151 if bl!=wl:
152 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
153 (length,offset,filename,wl,bl,tr),file=sys.stderr)
154 # HTTP body
155 balance=start_2+rec_text.tell()
156 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
157 # Output whatever is left
158 if parts & 4:
159 callback(wtype,bufView[balance:balance+wl],4)
160 state=1
161
162 L_start=rec_text.tell()