Mercurial > hg > cc > cirrus_work
changeset 389:2c04af3ca99a plus
implement part with 4-bit set
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Mon, 22 Jun 2026 12:33:40 +0100 |
| parents | 531a85706485 |
| children | af49ea848756 |
| files | lib/python/cc/iwarc.py |
| diffstat | 1 files changed, 4 insertions(+), 70 deletions(-) [+] |
line wrap: on
line diff
--- a/lib/python/cc/iwarc.py Sat Jun 20 16:39:46 2026 +0100 +++ b/lib/python/cc/iwarc.py Mon Jun 22 12:33:40 2026 +0100 @@ -28,7 +28,7 @@ '''parts is a bit-mask: 1 for warc header; 2 for req/resp HTTP header, warcinfo/metadata features; - 4 for req/resp body''' + 4 for resp body''' # Not currently trying to depend on this, but I believe that # warcinfo: record-headers+1bl+crawl-headers+2bl # request: record-headers+1bl+HTTP-headers+3bl @@ -126,8 +126,9 @@ else: # rest of the part _out = callback(wtype, bufView[start_2:eob], 2) - if parts & 4: - raise ValueError("Not implemented: body part (4): %s"%parts) + if (wtype == RESP) and parts & 4: + _out = callback(wtype, bufView[eo2 + 4:eob], 4) + #raise ValueError("Not implemented: body part (4): %s"%parts) #bp += length #if buf[bp] != 13: # # Why does this sometimes happen, e.g. when doing @@ -154,73 +155,6 @@ print('%d records, max record: %d, max header: %d'%(n, RECORDMAX, HDRMAX), file = sys.stderr) -import zlib, gzip, struct -from isal import isal_zlib - -def decompOneBlock(data: bytes, bl: int, bp: int = 0) -> tuple[bytes, int]: - """Decompress one block of a gzip compressed stream in one shot. - Return the decompressed string and the stream repositioned - at the start of the next block. - """ - fp: io.BytesIO = io.BytesIO(data) - fp.seek(bp) - if gzip._read_gzip_header(fp) is None: - return (b"",0) - bp: int = fp.tell() - # Use a isal's zlib raw deflate compressor - do: isal_zlib.Decompress = isal_zlib.decompressobj(wbits = -zlib.MAX_WBITS) - # Read all the data except the header - decompressed: bytes = do.decompress(data[bp:]) - if not do.eof or (uu:=len(do.unused_data)) < 8: - raise EOFError("Compressed file ended before the end-of-stream " - "marker was reached") - bp = bl - uu - crc: int - length: int - crc, length = struct.unpack("<II", do.unused_data[:8]) - bp += 8 - if crc != zlib.crc32(decompressed): - raise gzip.BadGzipFile("CRC check failed") - if length != (len(decompressed) & 0xffffffff): - raise gzip.BadGzipFile("Incorrect length of data produced") - while bp < bl and data[bp] == 0: - bp += 1 - return (decompressed, bp) - -def dotest(): - f: io.BinaryIO = open(sys.argv[1], "rb") - buf: bytes = bytearray(BUFSIZE) - bufView: bytes = memoryview(buf) - xBuf: bytes - bl: int = f.readinto(buf) - offset: int = 0 - bp: int = 0 - done: bool = bl < BUFSIZE - unc: bytes - keepLen: int - print(0, file = sys.stderr, end = " ") - while True: - (unc, bp) = decompOneBlock(bufView, bl, bp) - if unc == b"": - break - print(unc[21:29],file = sys.stderr) - print(offset + bp, file = sys.stderr, end = " ") - if (not done) and (keepLen := bl - bp) < BUFMIN: - # we need to shift and read more - offset += bp - buf[0:keepLen]=bufView[bp:bl] - with memoryview(buf)[keepLen:BUFSIZE] as xBuf: - nb: int = f.readinto(xBuf) - bl = keepLen+nb - if (done := (bl < BUFSIZE)): - bufView = bufView[0:bl] - bp = 0 - print("EOF", file = sys.stderr) - f.close() - -if __name__ == "__main__": - dotest() -
