Mercurial > hg > cc > cirrus_work
changeset 354:90f541382b0e plus
use gzip_plus to get access to gzip block offsets
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Mon, 16 Mar 2026 10:43:45 +0000 |
| parents | 6b1137bdbd0d |
| children | a0d476144685 |
| files | lib/python/cc/test_warc.py lib/python/cc/warc.py |
| diffstat | 2 files changed, 16 insertions(+), 6 deletions(-) [+] |
line wrap: on
line diff
--- a/lib/python/cc/test_warc.py Mon Mar 16 10:43:01 2026 +0000 +++ b/lib/python/cc/test_warc.py Mon Mar 16 10:43:45 2026 +0000 @@ -1,4 +1,4 @@ -import warc,sys +import warc_plus,sys OUT=open(sys.stdout.fileno(),'wb')
--- a/lib/python/cc/warc.py Mon Mar 16 10:43:01 2026 +0000 +++ b/lib/python/cc/warc.py Mon Mar 16 10:43:45 2026 +0000 @@ -5,8 +5,9 @@ part''' import sys, io -from isal import igzip -import cython, typing, gzip +#from isal import igzip +import cython, typing, gzip_plus +from collections import deque INFO: int = 0 RESP: int = 1 @@ -21,10 +22,16 @@ HDRMAX: int = 0 # will grow RECORDMAX: int = 0 # will grow +Q: queue = None + +def enq(b: int, e: int): + global Q + Q.append((b,e)) + def warc(filename: str, callback: typing.Callable[[bytes, typing.ByteString, int], typing.BinaryIO], types: typing.List[int] = [RESP], whole: bool = False, parts: int = 7, - debug: bool = False): + debug: bool = False, block = False): '''parts is a bit-mask: 1 for warc header; 2 for req/resp HTTP header, warcinfo/metadata features; @@ -34,7 +41,7 @@ # request: record-headers+1bl+HTTP-headers+3bl # response: record-headers+1bl+HTTP-headers+[1bl or 2bl]+HTTP-body+1bl # metadata: record-headers+1bl+metadata-headers+3bl - global BUFSIZE, HDRMAX, BUFMIN, RECORDMAX + global BUFSIZE, HDRMAX, BUFMIN, RECORDMAX, Q _out: typing.BinaryIO # should do some sanity checking wrt parts and types stream: typing.BinaryIO @@ -44,7 +51,10 @@ except gzip.BadGzipFile: stream = open(filename, 'rb', 0) else: - stream = igzip.IGzipFile(filename = filename) + if block: + Q = deque() + stream = gzip_plus.GzipFile(filename = filename, + eob_callback = enq if block else None) buf: char[::1] = bytearray(BUFSIZE) bufView: char[::1] = memoryview(buf) fpos: int = 0