# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1782128020 -3600
# Node ID 2c04af3ca99a4ca8ca745c4eed0ab397220050d2
# Parent  531a857064854eb4908586be84ccbd2bc6dbd611
implement part with 4-bit set

diff -r 531a85706485 -r 2c04af3ca99a lib/python/cc/iwarc.py
--- a/lib/python/cc/iwarc.py	Sat Jun 20 16:39:46 2026 +0100
+++ b/lib/python/cc/iwarc.py	Mon Jun 22 12:33:40 2026 +0100
@@ -28,7 +28,7 @@
   '''parts is a bit-mask:
      1 for warc header;
      2 for req/resp HTTP header, warcinfo/metadata features;
-     4 for req/resp body'''
+     4 for resp body'''
   # Not currently trying to depend on this, but I believe that
   #   warcinfo: record-headers+1bl+crawl-headers+2bl
   #   request: record-headers+1bl+HTTP-headers+3bl
@@ -126,8 +126,9 @@
             else:
               # rest of the part
               _out = callback(wtype, bufView[start_2:eob], 2)
-          if parts & 4:
-            raise ValueError("Not implemented: body part (4): %s"%parts)
+          if (wtype == RESP) and parts & 4:
+            _out = callback(wtype, bufView[eo2 + 4:eob], 4)
+            #raise ValueError("Not implemented: body part (4): %s"%parts)
     #bp += length
     #if buf[bp] != 13:
     #  # Why does this sometimes happen, e.g. when doing
@@ -154,73 +155,6 @@
   print('%d records, max record: %d, max header: %d'%(n, RECORDMAX, HDRMAX),
         file = sys.stderr)
 
-import zlib, gzip, struct
-from isal import isal_zlib
-
-def decompOneBlock(data: bytes, bl: int, bp: int = 0) -> tuple[bytes, int]:
-    """Decompress one block of a gzip compressed stream in one shot.
-    Return the decompressed string and the stream repositioned
-      at the start of the next block.
-    """
-    fp: io.BytesIO = io.BytesIO(data)
-    fp.seek(bp)
-    if gzip._read_gzip_header(fp) is None:
-        return (b"",0)
-    bp: int = fp.tell()
-    # Use a isal's zlib raw deflate compressor
-    do: isal_zlib.Decompress = isal_zlib.decompressobj(wbits = -zlib.MAX_WBITS)
-    # Read all the data except the header
-    decompressed: bytes = do.decompress(data[bp:])
-    if not do.eof or (uu:=len(do.unused_data)) < 8:
-        raise EOFError("Compressed file ended before the end-of-stream "
-                       "marker was reached")
-    bp = bl - uu
-    crc: int
-    length: int
-    crc, length = struct.unpack("<II", do.unused_data[:8])
-    bp += 8
-    if crc != zlib.crc32(decompressed):
-        raise gzip.BadGzipFile("CRC check failed")
-    if length != (len(decompressed) & 0xffffffff):
-        raise gzip.BadGzipFile("Incorrect length of data produced")
-    while bp < bl and data[bp] == 0:
-        bp += 1
-    return (decompressed, bp)
-
-def dotest():
-  f: io.BinaryIO = open(sys.argv[1], "rb")
-  buf: bytes = bytearray(BUFSIZE)
-  bufView: bytes = memoryview(buf)
-  xBuf: bytes
-  bl: int = f.readinto(buf)
-  offset: int = 0
-  bp: int = 0
-  done: bool = bl < BUFSIZE 
-  unc: bytes
-  keepLen: int
-  print(0, file = sys.stderr, end = " ")
-  while True:
-    (unc, bp) = decompOneBlock(bufView, bl, bp)
-    if unc == b"":
-      break
-    print(unc[21:29],file = sys.stderr)
-    print(offset + bp, file = sys.stderr, end = " ")
-    if (not done) and (keepLen := bl - bp) < BUFMIN:
-      # we need to shift and read more
-      offset += bp
-      buf[0:keepLen]=bufView[bp:bl]
-      with memoryview(buf)[keepLen:BUFSIZE] as xBuf:
-        nb: int = f.readinto(xBuf)
-      bl = keepLen+nb
-      if (done := (bl < BUFSIZE)):
-        bufView = bufView[0:bl]
-      bp = 0
-  print("EOF", file = sys.stderr)
-  f.close()
-
-if __name__ == "__main__":
-  dotest()
 
 
 
-