Mercurial > hg > cc > cirrus_work
changeset 349:a8a5f07211c4 trim
Now using isal_zlib.decompressobj
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Wed, 11 Mar 2026 18:42:21 +0000 |
| parents | ceacc13bf0e7 |
| children | 4ead0a390a29 |
| files | lib/python/cc/warc.py |
| diffstat | 1 files changed, 13 insertions(+), 5 deletions(-) [+] |
line wrap: on
line diff
--- a/lib/python/cc/warc.py Wed Mar 11 18:20:05 2026 +0000 +++ b/lib/python/cc/warc.py Wed Mar 11 18:42:21 2026 +0000 @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# cython: profile=False, language_level=3str '''Stream a warc format file, unzipping if necessary, invoking a callback on each record. Callback can be limited by WARC-Type, record part''' @@ -166,8 +167,8 @@ if gzip._read_gzip_header(fp) is None: return (b"",0) bp=fp.tell() - # Use a zlib raw deflate compressor - do = zlib.decompressobj(wbits=-zlib.MAX_WBITS) + # Use a isal's zlib raw deflate compressor + do = isal_zlib.decompressobj(wbits=-zlib.MAX_WBITS) # Read all the data except the header decompressed = do.decompress(data[bp:]) if not do.eof or (uu:=len(do.unused_data)) < 8: @@ -177,14 +178,14 @@ crc, length = struct.unpack("<II", do.unused_data[:8]) bp += 8 if crc != zlib.crc32(decompressed): - raise BadGzipFile("CRC check failed") + raise gzip.BadGzipFile("CRC check failed") if length != (len(decompressed) & 0xffffffff): - raise BadGzipFile("Incorrect length of data produced") + raise gzip.BadGzipFile("Incorrect length of data produced") while bp < bl and data[bp] == 0: bp += 1 return (decompressed, bp) -if __name__ == "__main__": +def dotest(): f=open(sys.argv[1],"rb") buf = bytearray(BUFSIZE) bufView = memoryview(buf) @@ -211,3 +212,10 @@ bp = 0 print("EOF", file = sys.stderr) f.close() + +if __name__ == "__main__": + dotest() + + + +
