# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1773177490 0
# Node ID 9ecf193718c11a22e7d300340f66ee814199ec2e
# Parent  4759d6ac46251abab0362d99f0b82e6ea7dc4b00
process exactly one block at a time, maybe?

diff -r 4759d6ac4625 -r 9ecf193718c1 lib/python/cc/warc.py
--- a/lib/python/cc/warc.py	Tue Mar 10 21:17:17 2026 +0000
+++ b/lib/python/cc/warc.py	Tue Mar 10 21:18:10 2026 +0000
@@ -153,3 +153,32 @@
     #print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr)
   print('%d records, max record: %d, max header: %d'%(n, RECORDMAX, HDRMAX),
         file=sys.stderr)
+
+def decompOneBlock(fp: io.BytesIO):
+    """Decompress one block of a gzip compressed stream in one shot.
+    Return the decompressed string and the stream repositioned
+      at the start of the next block.
+    """
+    data = fp.getbuffer()
+    if _read_gzip_header(fp) is None:
+        return b""
+    # Use a zlib raw deflate compressor
+    do = zlib.decompressobj(wbits=-zlib.MAX_WBITS)
+    # Read all the data except the header
+    decompressed = do.decompress(data[fp.tell():])
+    if not do.eof or len(do.unused_data) < 8:
+        raise EOFError("Compressed file ended before the end-of-stream "
+                       "marker was reached")
+    crc, length = struct.unpack("<II", do.unused_data[:8])
+    if crc != zlib.crc32(decompressed):
+        raise BadGzipFile("CRC check failed")
+    if length != (len(decompressed) & 0xffffffff):
+        raise BadGzipFile("Incorrect length of data produced")
+    eob = 8
+    unused = len(do.unused_data) - 8
+    while unused > 0 and do.unused_data[eob] == 0:
+        eob += 1
+        unused -= 1
+    if unused > 0:
+        fp.seek(-unused, 2)
+    return decompressed