changeset 389:2c04af3ca99a plus

implement part with 4-bit set
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 22 Jun 2026 12:33:40 +0100
parents 531a85706485
children af49ea848756
files lib/python/cc/iwarc.py
diffstat 1 files changed, 4 insertions(+), 70 deletions(-) [+]
line wrap: on
line diff
--- a/lib/python/cc/iwarc.py	Sat Jun 20 16:39:46 2026 +0100
+++ b/lib/python/cc/iwarc.py	Mon Jun 22 12:33:40 2026 +0100
@@ -28,7 +28,7 @@
   '''parts is a bit-mask:
      1 for warc header;
      2 for req/resp HTTP header, warcinfo/metadata features;
-     4 for req/resp body'''
+     4 for resp body'''
   # Not currently trying to depend on this, but I believe that
   #   warcinfo: record-headers+1bl+crawl-headers+2bl
   #   request: record-headers+1bl+HTTP-headers+3bl
@@ -126,8 +126,9 @@
             else:
               # rest of the part
               _out = callback(wtype, bufView[start_2:eob], 2)
-          if parts & 4:
-            raise ValueError("Not implemented: body part (4): %s"%parts)
+          if (wtype == RESP) and parts & 4:
+            _out = callback(wtype, bufView[eo2 + 4:eob], 4)
+            #raise ValueError("Not implemented: body part (4): %s"%parts)
     #bp += length
     #if buf[bp] != 13:
     #  # Why does this sometimes happen, e.g. when doing
@@ -154,73 +155,6 @@
   print('%d records, max record: %d, max header: %d'%(n, RECORDMAX, HDRMAX),
         file = sys.stderr)
 
-import zlib, gzip, struct
-from isal import isal_zlib
-
-def decompOneBlock(data: bytes, bl: int, bp: int = 0) -> tuple[bytes, int]:
-    """Decompress one block of a gzip compressed stream in one shot.
-    Return the decompressed string and the stream repositioned
-      at the start of the next block.
-    """
-    fp: io.BytesIO = io.BytesIO(data)
-    fp.seek(bp)
-    if gzip._read_gzip_header(fp) is None:
-        return (b"",0)
-    bp: int = fp.tell()
-    # Use a isal's zlib raw deflate compressor
-    do: isal_zlib.Decompress = isal_zlib.decompressobj(wbits = -zlib.MAX_WBITS)
-    # Read all the data except the header
-    decompressed: bytes = do.decompress(data[bp:])
-    if not do.eof or (uu:=len(do.unused_data)) < 8:
-        raise EOFError("Compressed file ended before the end-of-stream "
-                       "marker was reached")
-    bp = bl - uu
-    crc: int
-    length: int
-    crc, length = struct.unpack("<II", do.unused_data[:8])
-    bp += 8
-    if crc != zlib.crc32(decompressed):
-        raise gzip.BadGzipFile("CRC check failed")
-    if length != (len(decompressed) & 0xffffffff):
-        raise gzip.BadGzipFile("Incorrect length of data produced")
-    while bp < bl and data[bp] == 0:
-        bp += 1
-    return (decompressed, bp)
-
-def dotest():
-  f: io.BinaryIO = open(sys.argv[1], "rb")
-  buf: bytes = bytearray(BUFSIZE)
-  bufView: bytes = memoryview(buf)
-  xBuf: bytes
-  bl: int = f.readinto(buf)
-  offset: int = 0
-  bp: int = 0
-  done: bool = bl < BUFSIZE 
-  unc: bytes
-  keepLen: int
-  print(0, file = sys.stderr, end = " ")
-  while True:
-    (unc, bp) = decompOneBlock(bufView, bl, bp)
-    if unc == b"":
-      break
-    print(unc[21:29],file = sys.stderr)
-    print(offset + bp, file = sys.stderr, end = " ")
-    if (not done) and (keepLen := bl - bp) < BUFMIN:
-      # we need to shift and read more
-      offset += bp
-      buf[0:keepLen]=bufView[bp:bl]
-      with memoryview(buf)[keepLen:BUFSIZE] as xBuf:
-        nb: int = f.readinto(xBuf)
-      bl = keepLen+nb
-      if (done := (bl < BUFSIZE)):
-        bufView = bufView[0:bl]
-      bp = 0
-  print("EOF", file = sys.stderr)
-  f.close()
-
-if __name__ == "__main__":
-  dotest()
 
 
 
-