Mercurial > hg > cc > cirrus_work

annotate bin/warc.py @ 58:299e3d0f2310

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
but skip at eobp is not working (with test 2)
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 12 Jul 2023 19:07:56 +0100
parents 61b0a1582af8
children 5d40d7511374
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/usr/bin/env python3
46
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
2 '''Stream a warc format file, unzipping if necessary, invoking a
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
3 callback on each record. Callback can be limited by WARC-Type, record
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
4 part'''
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5
42
689a0e311cd2 make warc.py a library, separate out testing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 39
diff changeset
6 import sys,io
689a0e311cd2 make warc.py a library, separate out testing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 39
diff changeset
7 from isal import igzip
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8
42
689a0e311cd2 make warc.py a library, separate out testing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 39
diff changeset
9 def warc(filename,callback,types=['response'],whole=False,parts=7,debug=False):
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types]
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 nb=0
46
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
12 if filename.endswith(".gz"):
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
13 stream=igzip.IGzipFile(filename=filename)
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
14 else:
44d3a4f4ea51 support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 43
diff changeset
15 stream=open(filename,'rb',0)
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
16 bufSize=2*1024*1024
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
17 hdrMax=16*1024
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
18 buf=bytearray(bufSize)
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
19 #with memoryview(buf)[:hdrMax] as hdrBuf:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
20 fpos=bl=stream.readinto(buf)
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
21 bp=0
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
22 done=False
50
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
23 while True:
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
24 while buf.startswith(b'\r\n',bp): # will Fail if buffer (nearly) empty
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
25 bp+=2
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
26 start_1=bp
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
27 if not buf.startswith(b'WARC/1.0\r\n',bp):
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
28 if done and bl-bp==0:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
29 # really done
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
30 return
51
c0b4359dd26a working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 50
diff changeset
31 raise ValueError("Not a WARC file? At %s: %s[%s]"%(bp,
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
32 buf[bp:min(bl,bp+20)].decode('latin-1'), bl-bp))
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
33 bp+=10
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
34 wtype=None
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
35 length=None
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
36 state=1
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
37 tr=None # Was this record truncated?
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
38 while not buf.startswith(b'\r\n',bp):
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
39 # there should always be enough in the buffer to complete this loop,
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
40 # because of the buffer update logic below
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
41 eol=buf.index(b'\r\n',bp)+2
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
42 if buf.startswith(b"Content-Length: ",bp):
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
43 length=wl=int(buf[bp+16:eol-2])
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
44 elif buf.startswith(b"WARC-Truncated: ",bp):
57
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
45 tr=bytes(buf[bp+16:eol-2])
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
46 tr=b"EMPTY" if tr==b"" else tr
48
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
47 elif buf.startswith(b'WARC-Type: ',bp):
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
48 wtype = bytes(buf[bp+11:eol-2])
d0d2fd9830d6 starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 46
diff changeset
49 bp=eol
51
c0b4359dd26a working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 50
diff changeset
50 bp=eol+2
57
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
51 if done:
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
52 if (bp+length)>bl:
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
53 raise ValueError("Done but need more! %s + %s > %s"%(bp,
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
54 length,bl))
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
55 elif (bp+(length+hdrMax))>bl:
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
56 # Need more data
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
57 if wtype in types:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
58 # we need to keep from start_1 to bl
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
59 keepFrom=start_1
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
60 keepLen=bl-keepFrom
57
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
61 buf[0:keepLen]=buf[keepFrom:bl]
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
62 else:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
63 # we can skip the rest of this part
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
64 keepLen=0
57
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
65 fpos=stream.seek(fpos+(bp+length-bl))
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
66 spaceToFill=bufSize-keepLen
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
67 with memoryview(buf)[keepLen:bufSize] as xBuf:
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
68 nb=stream.readinto(xBuf)
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
69 fpos+=nb
57
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
70 eol=eol-start_1
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
71 start_1=0
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
72 bp=eol+2
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
73 bl=keepLen+nb
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
74 if nb<spaceToFill:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
75 done=True
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
76 if wtype not in types:
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
77 continue
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
78 if (wtype in types):
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
79 if whole:
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
80 pass # buf[bp:(bp:=bp+ln)]=l @fixme
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
81 elif (parts & 1):
56
f8c8f79b2532 rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 54
diff changeset
82 OUT=callback(wtype,buf[start_1:eol],1)
50
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
83 if parts!=1:
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
84 # everything from bv= goes here
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
85 pass
58
299e3d0f2310 but skip at eobp is not working (with test 2)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 57
diff changeset
86 bp+=length
57
61b0a1582af8 works with all types, part=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 56
diff changeset
87 print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr)
50
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
88 #while not buf.startswith(b'\r\n',bp):
55943918794e a bit better
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 49
diff changeset
89 continue
39
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
90 bv=memoryview(buf)[start_2:start_2+length]
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
91 ii=0
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
92 while True and not stream.closed:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
93 if (i:=stream.readinto(bv))==0:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
94 break
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
95 ii+=i
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
96 if ii>=length:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
97 break
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
98 bv=memoryview(buf)[start_2+ii:start_2+length]
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
99 if ii!=length:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
100 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length))
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
101 nb+=length
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
102 if wtype in types:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
103 if whole:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
104 callback(wtype,buf[0:start_2+length],7)
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
105 continue
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
106 # Only output parts (1 = WARC header, 2 = HTTP header, 4 = body) that are wanted
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
107 bl=None # for HTTP Content-Length for the length of the body?
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
108 L_start=start_2
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
109 state=2
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
110 bv=memoryview(buf)[start_2:start_2+length]
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
111 with io.BytesIO(bv) as rec_text:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
112 for L in rec_text:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
113 if state==2:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
114 # HTTP header
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
115 wl -= len(L)
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
116 if not (L==b"" or L.startswith(b"\r")):
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
117 # Non-empty, it's (a continuation of) a header
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
118 if bl is None and L.startswith(b"Content-Length: "):
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
119 bl=int(L[16:].rstrip())
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
120 else:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
121 # Blank line, HTTP header is finished
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
122 if parts & 2:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
123 callback(wtype,buf[start_2:start_2+L_start],2)
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
124 state=4
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
125 # The above is just for sanity, because we do _not_
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
126 # continue with the outer loop,
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
127 # since we can now block-output the entire rest of the
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
128 # input buffer.
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
129 if bl is not None:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
130 if bl!=wl:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
131 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
132 (length,offset,filename,wl,bl,tr),file=sys.stderr)
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
133 # HTTP body
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
134 balance=start_2+rec_text.tell()
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
135 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
136 # Output whatever is left
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
137 if parts & 4:
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
138 callback(wtype,buf[balance:balance+wl],4)
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
139 state=1
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
140
8661062a50b1 moved from home bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
141 L_start=rec_text.tell()