# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1773245315 0
# Node ID 8c9e7578ed30e54292e07d3e71c19e463fcbb166
# Parent  279793350225fcab48be6692cfeca5437094b3fe
refactor slightly, read whole warc file as test, seems to be working, still using zlib.decompressobj

diff -r 279793350225 -r 8c9e7578ed30 lib/python/cc/warc.py
--- a/lib/python/cc/warc.py	Wed Mar 11 13:24:48 2026 +0000
+++ b/lib/python/cc/warc.py	Wed Mar 11 16:08:35 2026 +0000
@@ -58,7 +58,6 @@
   while not (done and bp >= bl):
     start_1: int = bp
     if not buf.startswith(b'WARC/1.0\r\n',bp):
-      breakpoint()
       raise ValueError("Not a WARC file? In %s at %s of %s (%s): %s[%s]"%(filename,
                                                                    bp,bl,fpos,
          (buf[bp:min(bl,bp+20)] if bp<bl else buf[bl-20:bl]).decode('latin-1'),
@@ -155,32 +154,59 @@
         file=sys.stderr)
 
 import zlib, gzip, struct
+from isal import isal_zlib
 
-def decompOneBlock(fp: io.BytesIO):
+def decompOneBlock(data: memoryview, bl: int, bp: int = 0):
     """Decompress one block of a gzip compressed stream in one shot.
     Return the decompressed string and the stream repositioned
       at the start of the next block.
     """
-    data = fp.getbuffer()
+    fp: io.BytesIO = io.BytesIO(data)
+    fp.seek(bp)
     if gzip._read_gzip_header(fp) is None:
-        return b""
+        return (b"",0)
+    bp=fp.tell()
     # Use a zlib raw deflate compressor
     do = zlib.decompressobj(wbits=-zlib.MAX_WBITS)
     # Read all the data except the header
-    decompressed = do.decompress(data[fp.tell():])
-    if not do.eof or len(do.unused_data) < 8:
+    decompressed = do.decompress(data[bp:])
+    if not do.eof or (uu:=len(do.unused_data)) < 8:
         raise EOFError("Compressed file ended before the end-of-stream "
                        "marker was reached")
+    bp = bl - uu
     crc, length = struct.unpack("<II", do.unused_data[:8])
+    bp += 8
     if crc != zlib.crc32(decompressed):
         raise BadGzipFile("CRC check failed")
     if length != (len(decompressed) & 0xffffffff):
         raise BadGzipFile("Incorrect length of data produced")
-    eob = 8
-    unused = len(do.unused_data) - 8
-    while unused > 0 and do.unused_data[eob] == 0:
-        eob += 1
-        unused -= 1
-    if unused > 0:
-        fp.seek(-unused, 2)
-    return decompressed
+    while bp < bl and data[bp] == 0:
+        bp += 1
+    return (decompressed, bp)
+
+if __name__ == "__main__":
+  f=open(sys.argv[1],"rb")
+  buf = bytearray(BUFSIZE)
+  bufView = memoryview(buf)
+  bl: int = f.readinto(buf)
+  offset: int = 0
+  bp: int = 0
+  done: bool = bl < BUFSIZE 
+  print(0, file=sys.stderr)
+  while True:
+    (unc, bp) = decompOneBlock(bufView, bl, bp)
+    if unc == b"":
+      break
+    print(offset + bp,unc[21:29],file=sys.stderr)
+    if (not done) and (keepLen := bl - bp) < BUFMIN:
+      # we need to shift and read more
+      print("shifting",file=sys.stderr)
+      offset += bp
+      buf[0:keepLen]=bufView[bp:bl]
+      with memoryview(buf)[keepLen:BUFSIZE] as xBuf:
+        nb=f.readinto(xBuf)
+      bl = keepLen+nb
+      done = bl < BUFSIZE 
+      bp = 0
+  fp.close()
+  f.close()