Mercurial > hg > cc > cirrus_work
changeset 343:9ecf193718c1 trim
process exactly one block at a time, maybe?
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Tue, 10 Mar 2026 21:18:10 +0000 |
| parents | 4759d6ac4625 |
| children | 279793350225 |
| files | lib/python/cc/warc.py |
| diffstat | 1 files changed, 29 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/lib/python/cc/warc.py Tue Mar 10 21:17:17 2026 +0000 +++ b/lib/python/cc/warc.py Tue Mar 10 21:18:10 2026 +0000 @@ -153,3 +153,32 @@ #print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr) print('%d records, max record: %d, max header: %d'%(n, RECORDMAX, HDRMAX), file=sys.stderr) + +def decompOneBlock(fp: io.BytesIO): + """Decompress one block of a gzip compressed stream in one shot. + Return the decompressed string and the stream repositioned + at the start of the next block. + """ + data = fp.getbuffer() + if _read_gzip_header(fp) is None: + return b"" + # Use a zlib raw deflate compressor + do = zlib.decompressobj(wbits=-zlib.MAX_WBITS) + # Read all the data except the header + decompressed = do.decompress(data[fp.tell():]) + if not do.eof or len(do.unused_data) < 8: + raise EOFError("Compressed file ended before the end-of-stream " + "marker was reached") + crc, length = struct.unpack("<II", do.unused_data[:8]) + if crc != zlib.crc32(decompressed): + raise BadGzipFile("CRC check failed") + if length != (len(decompressed) & 0xffffffff): + raise BadGzipFile("Incorrect length of data produced") + eob = 8 + unused = len(do.unused_data) - 8 + while unused > 0 and do.unused_data[eob] == 0: + eob += 1 + unused -= 1 + if unused > 0: + fp.seek(-unused, 2) + return decompressed
