Mercurial > hg > cc > cirrus_work

annotate bin/warc.py @ 60:7b68c3ebc35a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
tests 1 & 2 now working
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Thu, 13 Jul 2023 14:02:02 +0100
parents 5d40d7511374
children f182d09ad1cd
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/usr/bin/env python3
46
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
2 '''Stream a warc format file, unzipping if necessary, invoking a
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
3 callback on each record. Callback can be limited by WARC-Type, record
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
4 part'''
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5
42
689a0e311cd2 make warc.py a library, separate out testing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 39
diff changeset
6 import sys,io
689a0e311cd2 make warc.py a library, separate out testing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 39
diff changeset
7 from isal import igzip
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8
42
689a0e311cd2 make warc.py a library, separate out testing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 39
diff changeset
9 def warc(filename,callback,types=['response'],whole=False,parts=7,debug=False):
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types]
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 nb=0
46
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
12 if filename.endswith(".gz"):
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
13 stream=igzip.IGzipFile(filename=filename)
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
14 else:
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
15 stream=open(filename,'rb',0)
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
16 bufSize=2*1024*1024
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
17 hdrMax=16*1024
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
18 buf=bytearray(bufSize)
59
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
19 bufView=memoryview(buf)
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
20 fpos=bl=stream.readinto(buf)
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
21 bp=0
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
22 done=False
50
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
23 while True:
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
24 while buf.startswith(b'\r\n',bp): # will Fail if buffer (nearly) empty
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
25 bp+=2
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
26 start_1=bp
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
27 if not buf.startswith(b'WARC/1.0\r\n',bp):
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
28 if done and bl-bp==0:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
29 # really done
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
30 return
51
c0b4359dd26a working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 50
diff changeset
31 raise ValueError("Not a WARC file? At %s: %s[%s]"%(bp,
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
32 buf[bp:min(bl,bp+20)].decode('latin-1'), bl-bp))
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
33 bp+=10
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
34 wtype=None
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
35 length=None
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
36 state=1
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
37 tr=None # Was this record truncated?
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
38 while not buf.startswith(b'\r\n',bp):
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
39 # there should always be enough in the buffer to complete this loop,
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
40 # because of the buffer update logic below
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
41 eol=buf.index(b'\r\n',bp)+2
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
42 if buf.startswith(b"Content-Length: ",bp):
59
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
43 length=wl=int(bufView[bp+16:eol-2])
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
44 elif buf.startswith(b"WARC-Truncated: ",bp):
59
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
45 if bp+16==eol-2:
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
46 tr=b"EMPTY"
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
47 else:
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
48 tr=bytes(bufView[bp+16:eol-2])
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
49 elif buf.startswith(b'WARC-Type: ',bp):
59
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
50 if buf.startswith(b's',bp+13):
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
51 wtype = b'response'
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
52 elif buf.startswith(b'q',bp+13):
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
53 wtype = b'request'
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
54 elif buf.startswith(b'm',bp+11):
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
55 wtype = b'metadata'
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
56 elif buf.startswith(b'w',bp+11):
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
57 wtype = b'warcinfo'
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
58 else:
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
59 raise ValueError("Unknown WARC-Type: %s at %s"%(
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
60 bytes(bufView[bp+11:eol-2]),
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
61 fpos-(bl-bp)))
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
62 bp=eol
51
c0b4359dd26a working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 50
diff changeset
63 bp=eol+2
57
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
64 if done:
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
65 if (bp+length)>bl:
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
66 raise ValueError("Done but need more! %s + %s > %s"%(bp,
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
67 length,bl))
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
68 elif (bp+(length+hdrMax))>bl:
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
69 # Need more data
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
70 if wtype in types:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
71 # we need to keep from start_1 to bl
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
72 keepFrom=start_1
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
73 keepLen=bl-keepFrom
59
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
74 buf[0:keepLen]=bufView[keepFrom:bl]
60
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
75 eol=eol-start_1
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
76 start_1=0
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
77 bp=eol+2
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
78 else:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
79 # we can skip the rest of this part
60
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
80 if (bp+length)<=bl:
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
81 # we have at least some bytes from the next part
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
82 keepLen=bl-(bp+length)
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
83 buf[0:keepLen]=bufView[bl-keepLen:bl]
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
84 else:
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
85 # we don't have all of the bytes from the current part
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
86 # so can skip the rest of it
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
87 keepLen=0
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
88 fpos=stream.seek(fpos+(bp+length-bl))
7b68c3ebc35a tests 1 & 2 now working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 59
diff changeset
89 bp=0
57
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
90 spaceToFill=bufSize-keepLen
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
91 with memoryview(buf)[keepLen:bufSize] as xBuf:
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
92 nb=stream.readinto(xBuf)
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
93 fpos+=nb
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
94 bl=keepLen+nb
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
95 if nb<spaceToFill:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
96 done=True
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
97 if wtype not in types:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
98 continue
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
99 if (wtype in types):
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
100 if whole:
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
101 pass # buf[bp:(bp:=bp+ln)]=l @fixme
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
102 elif (parts & 1):
59
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
103 OUT=callback(wtype,bufView[start_1:eol],1)
50
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
104 if parts!=1:
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
105 # everything from bv= goes here
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
106 pass
58
299e3d0f2310 but skip at eobp is not working (with test 2)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 57
diff changeset
107 bp+=length
57
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
108 print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr)
50
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
109 #while not buf.startswith(b'\r\n',bp):
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
110 continue
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
111 bv=memoryview(buf)[start_2:start_2+length]
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
112 ii=0
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
113 while True and not stream.closed:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
114 if (i:=stream.readinto(bv))==0:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
115 break
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
116 ii+=i
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
117 if ii>=length:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
118 break
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
119 bv=memoryview(buf)[start_2+ii:start_2+length]
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
120 if ii!=length:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
121 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length))
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
122 nb+=length
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
123 if wtype in types:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
124 if whole:
59
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
125 callback(wtype,bufView[0:start_2+length],7)
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
126 continue
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
127 # Only output parts (1 = WARC header, 2 = HTTP header, 4 = body) that are wanted
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
128 bl=None # for HTTP Content-Length for the length of the body?
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
129 L_start=start_2
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
130 state=2
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
131 bv=memoryview(buf)[start_2:start_2+length]
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
132 with io.BytesIO(bv) as rec_text:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
133 for L in rec_text:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
134 if state==2:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
135 # HTTP header
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
136 wl -= len(L)
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
137 if not (L==b"" or L.startswith(b"\r")):
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
138 # Non-empty, it's (a continuation of) a header
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
139 if bl is None and L.startswith(b"Content-Length: "):
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
140 bl=int(L[16:].rstrip())
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
141 else:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
142 # Blank line, HTTP header is finished
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
143 if parts & 2:
59
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
144 callback(wtype,bufView[start_2:start_2+L_start],2)
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
145 state=4
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
146 # The above is just for sanity, because we do _not_
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
147 # continue with the outer loop,
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
148 # since we can now block-output the entire rest of the
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
149 # input buffer.
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
150 if bl is not None:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
151 if bl!=wl:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
152 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
153 (length,offset,filename,wl,bl,tr),file=sys.stderr)
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
154 # HTTP body
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
155 balance=start_2+rec_text.tell()
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
156 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
157 # Output whatever is left
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
158 if parts & 4:
59
5d40d7511374 avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 58
diff changeset
159 callback(wtype,bufView[balance:balance+wl],4)
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
160 state=1
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
161
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
162 L_start=rec_text.tell()