Mercurial > hg > cc > cirrus_work
annotate lib/python/cc/warc.py @ 290:52c9d1875608
simple refill working?
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Wed, 09 Apr 2025 12:57:50 +0100 |
| parents | f17aef7ba4a7 |
| children | 70da637d1402 |
| rev | line source |
|---|---|
| 39 | 1 #!/usr/bin/env python3 |
|
46
44d3a4f4ea51
support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
43
diff
changeset
|
2 '''Stream a warc format file, unzipping if necessary, invoking a |
|
44d3a4f4ea51
support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
43
diff
changeset
|
3 callback on each record. Callback can be limited by WARC-Type, record |
|
44d3a4f4ea51
support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
43
diff
changeset
|
4 part''' |
| 39 | 5 |
|
164
4315a36b1672
refactor to provide for buffer overflow fix
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
120
diff
changeset
|
6 import sys, io |
|
42
689a0e311cd2
make warc.py a library, separate out testing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
39
diff
changeset
|
7 from isal import igzip |
| 289 | 8 import cython, typing, gzip |
| 39 | 9 |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
10 # see warc.pxd for function signatures |
|
283
6739e08d19ff
type decls, cythonize works
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
164
diff
changeset
|
11 |
| 289 | 12 INFO: int = 0 |
| 13 RESP: int = 1 | |
| 14 REQ: int = 2 | |
| 15 META: int = 3 | |
|
62
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
16 |
| 289 | 17 BUFSIZE: int = 16 * 1024 * 1024 |
|
288
d3fc7b5c73d0
park that, try fixed large buffer and large-enough min to ensure we always have a whole record in view
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
287
diff
changeset
|
18 BUFMIN: int = 3 * 512 * 1024 # 1.5MiB, will need to be increased |
|
d3fc7b5c73d0
park that, try fixed large buffer and large-enough min to ensure we always have a whole record in view
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
287
diff
changeset
|
19 # to 5.5MiB for CC-MAIN-2025-13 (Mar) and thereafter |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
20 |
|
288
d3fc7b5c73d0
park that, try fixed large buffer and large-enough min to ensure we always have a whole record in view
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
287
diff
changeset
|
21 HDRMAX: int = 0 # will grow |
| 290 | 22 RECORDMAX: int = 0 # will grow |
|
164
4315a36b1672
refactor to provide for buffer overflow fix
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
120
diff
changeset
|
23 |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
24 def warc(filename: str, |
|
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
25 callback: typing.Callable[[bytes, typing.ByteString, int], typing.BinaryIO], |
| 289 | 26 types: typing.List[int] = [RESP], whole: bool = False, parts: int = 7, |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
27 debug: bool = False): |
| 61 | 28 '''parts is a bit-mask: |
| 29 1 for warc header; | |
| 30 2 for req/resp HTTP header, warcinfo/metadata features; | |
| 31 4 for req/resp body''' | |
|
288
d3fc7b5c73d0
park that, try fixed large buffer and large-enough min to ensure we always have a whole record in view
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
287
diff
changeset
|
32 # Not currently trying to depend on this, but I believe that |
| 289 | 33 # warcinfo: record-headers+1bl+crawl-headers+2bl |
| 34 # request: record-headers+1bl+HTTP-headers+3bl | |
| 35 # response: record-headers+1bl+HTTP-headers+[1bl or 2bl]+HTTP-body+1bl | |
| 36 # metadata: record-headers+1bl+metadata-headers+3bl | |
| 37 global BUFSIZE, HDRMAX, BUFMIN, RECORDMAX | |
| 38 _out: typing.BinaryIO | |
|
62
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
39 # should do some sanity checking wrt parts and types |
| 289 | 40 stream: typing.BinaryIO |
| 41 with gzip.open(filename, 'r') as fh: | |
| 42 try: | |
| 43 fh.read(1) | |
| 44 except gzip.BadGzipFile: | |
| 45 stream = open(filename,'rb',0) | |
| 46 else: | |
| 47 stream = igzip.IGzipFile(filename=filename) | |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
48 buf: char[::1] = bytearray(BUFSIZE) |
| 289 | 49 bufView: char[::1] = memoryview(buf) |
| 290 | 50 fpos: int = 0 |
| 51 bp: int = 0 | |
| 52 bl: int = stream.readinto(buf) | |
| 53 n: int = 0 | |
| 289 | 54 done: bool = bl < BUFSIZE |
| 290 | 55 while buf.startswith(b'\r\n',bp): |
| 56 bp+=2 | |
| 289 | 57 while not (done and bl == bp): |
| 290 | 58 start_1: int = bp |
|
48
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
59 if not buf.startswith(b'WARC/1.0\r\n',bp): |
|
67
b8d4a5ede7a3
fix eof bug, expand error messages
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
66
diff
changeset
|
60 raise ValueError("Not a WARC file? In %s at %s of %s (%s): %s[%s]"%(filename, |
|
b8d4a5ede7a3
fix eof bug, expand error messages
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
66
diff
changeset
|
61 bp,bl,fpos, |
|
b8d4a5ede7a3
fix eof bug, expand error messages
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
66
diff
changeset
|
62 (buf[bp:min(bl,bp+20)] if bp<bl else buf[bl-20:bl]).decode('latin-1'), |
|
b8d4a5ede7a3
fix eof bug, expand error messages
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
66
diff
changeset
|
63 bl-bp)) |
| 290 | 64 bp += 10 |
| 65 n += 1 | |
| 66 wtype: int = -1 | |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
67 length: int = 0 |
|
288
d3fc7b5c73d0
park that, try fixed large buffer and large-enough min to ensure we always have a whole record in view
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
287
diff
changeset
|
68 tr: bytes = b'' # Was this record truncated? |
|
48
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
69 while not buf.startswith(b'\r\n',bp): |
|
56
f8c8f79b2532
rework completely to refill as much as possible only when necessary,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
54
diff
changeset
|
70 # there should always be enough in the buffer to complete this loop, |
| 289 | 71 # because of the buffer update logic at the end |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
72 eol = buf.index(b'\r\n', bp) |
| 289 | 73 if buf.startswith(b"Content-Length: ",bp): |
| 290 | 74 length=wl=int(bufView[bp+16:eol]) |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
75 if buf.startswith(b"WARC-Truncated: ",bp): |
|
59
5d40d7511374
avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
58
diff
changeset
|
76 if bp+16==eol-2: |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
77 tr = b"EMPTY" |
|
59
5d40d7511374
avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
58
diff
changeset
|
78 else: |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
79 tr = bytes(bufView[bp+16:eol-2]) |
|
48
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
80 elif buf.startswith(b'WARC-Type: ',bp): |
|
59
5d40d7511374
avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
58
diff
changeset
|
81 if buf.startswith(b's',bp+13): |
|
62
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
82 wtype = RESP |
|
59
5d40d7511374
avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
58
diff
changeset
|
83 elif buf.startswith(b'q',bp+13): |
|
62
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
84 wtype = REQ |
|
59
5d40d7511374
avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
58
diff
changeset
|
85 elif buf.startswith(b'm',bp+11): |
|
62
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
86 wtype = META |
|
59
5d40d7511374
avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
58
diff
changeset
|
87 elif buf.startswith(b'w',bp+11): |
|
62
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
88 wtype = INFO |
|
59
5d40d7511374
avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
58
diff
changeset
|
89 else: |
|
67
b8d4a5ede7a3
fix eof bug, expand error messages
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
66
diff
changeset
|
90 raise ValueError("Unknown WARC-Type: %s in %s at %s"%( |
|
b8d4a5ede7a3
fix eof bug, expand error messages
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
66
diff
changeset
|
91 bytes(bufView[bp+11:eol-2]),filename, |
|
59
5d40d7511374
avoid slicing buf by using memoryview to save copying
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
58
diff
changeset
|
92 fpos-(bl-bp))) |
| 290 | 93 bp=eol+2 |
| 289 | 94 # record header done |
|
288
d3fc7b5c73d0
park that, try fixed large buffer and large-enough min to ensure we always have a whole record in view
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
287
diff
changeset
|
95 if (hl:=(bp - start_1)) > HDRMAX: |
|
d3fc7b5c73d0
park that, try fixed large buffer and large-enough min to ensure we always have a whole record in view
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
287
diff
changeset
|
96 HDRMAX = hl |
| 289 | 97 #if done: |
| 98 # if (bp+length)>bl: | |
| 99 # raise ValueError("Done but need more! %s + %s > %s in %s"%(bp, | |
| 100 # length,bl,filename)) | |
| 39 | 101 if (wtype in types): |
|
62
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
102 # Output whole or part 1 as required |
| 39 | 103 if whole: |
| 61 | 104 bp+=length |
| 289 | 105 _out=callback(wtype,bufView[start_1:bp],7) |
| 61 | 106 continue |
| 39 | 107 elif (parts & 1): |
| 289 | 108 _out=callback(wtype,bufView[start_1:eol],1) |
| 290 | 109 bp = eol |
| 110 while buf.startswith(b'\r\n',bp): | |
| 111 bp+=2 | |
| 50 | 112 if parts!=1: |
|
62
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
113 start_2=bp |
|
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
114 eob=bp+length |
|
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
115 while buf.startswith(b'\r\n',eob-2): |
|
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
116 eob-=2 |
|
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
117 # Only output parts (2 = HTTP header, 4 = body) that are wanted |
|
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
118 if parts & 2: |
| 289 | 119 if wtype == META or wtype == INFO: |
|
62
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
120 # rest of the part |
| 289 | 121 _out=callback(wtype,bufView[start_2:eob],2) |
|
66
75f1d3bc60d9
part 2 is now working for all types
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
62
diff
changeset
|
122 else: |
|
75f1d3bc60d9
part 2 is now working for all types
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
62
diff
changeset
|
123 # request and response have http headers |
|
75f1d3bc60d9
part 2 is now working for all types
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
62
diff
changeset
|
124 eo2=buf.index(b'\r\n\r\n',start_2) |
| 289 | 125 _out=callback(wtype,bufView[start_2:eo2+2],2) |
|
62
11cbaee8bbc8
Test 2 works with parts=1,2,3.
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
61
diff
changeset
|
126 if parts & 4: |
|
287
fe78af4ea7c5
in the midst of trying to rethink the refill logic
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
283
diff
changeset
|
127 raise ValueError("Not implemented: body part (4): %s"%parts) |
| 289 | 128 bp += length |
| 129 rl: int | |
| 130 if (rl := (bp - start_1)) > RECORDMAX: | |
| 131 RECORDMAX = rl | |
| 132 keepLen: int | |
| 133 if (not done) and (keepLen := bl - bp) < BUFMIN: | |
| 134 # we need to shift and read more | |
| 135 buf[0:keepLen]=bufView[bp:bl] | |
| 136 with memoryview(buf)[keepLen:BUFSIZE] as xBuf: | |
| 137 nb=stream.readinto(xBuf) | |
| 138 bl = keepLen+nb | |
| 139 done = bl < BUFSIZE | |
| 140 bp = 0 | |
| 290 | 141 while buf.startswith(b'\r\n',bp): |
| 142 bp+=2 | |
| 61 | 143 #print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr) |
| 290 | 144 print('%d records, max record: %d, max header: %d'%(n, RECORDMAX, HDRMAX), |
| 145 file=sys.stderr) |