# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1773339601 0
# Node ID 444ddf7045025d6ee47a64e48950355c9cd20fdc
# Parent  4ead0a390a29ce9b712f74a7b1c1e46b3b92c0ef
switch to plain GzipFile for comparison purposes,
lots of syntax tidying

diff -r 4ead0a390a29 -r 444ddf704502 lib/python/cc/warc.py
--- a/lib/python/cc/warc.py	Thu Mar 12 18:08:54 2026 +0000
+++ b/lib/python/cc/warc.py	Thu Mar 12 18:20:01 2026 +0000
@@ -5,11 +5,9 @@
 part'''
 
 import sys, io
-from isal import igzip
+#from isal import igzip
 import cython, typing, gzip
 
-# see warc.pxd for function signatures
-
 INFO: int =  0
 RESP: int = 1
 REQ: int =  2
@@ -44,25 +42,27 @@
     try:
       fh.read(1)
     except gzip.BadGzipFile:
-      stream = open(filename,'rb',0)
+      stream = open(filename, 'rb', 0)
     else:
-      stream = igzip.IGzipFile(filename=filename)
+      stream = gzip.GzipFile(filename = filename)
   buf: char[::1] = bytearray(BUFSIZE)
   bufView: char[::1] = memoryview(buf)
   fpos: int = 0
   bp: int = 0
   bl: int = stream.readinto(buf)
   n: int = 0
+  eob: int
+  eo2: int
   done: bool = bl < BUFSIZE 
-  while buf.startswith(b'\r\n',bp):
-    bp+=2
+  while buf.startswith(b'\r\n', bp):
+    bp += 2
   while not (done and bp >= bl):
     start_1: int = bp
-    if not buf.startswith(b'WARC/1.0\r\n',bp):
+    if not buf.startswith(b'WARC/1.0\r\n', bp):
       raise ValueError("Not a WARC file? In %s at %s of %s (%s): %s[%s]"%(filename,
-                                                                   bp,bl,fpos,
-         (buf[bp:min(bl,bp+20)] if bp<bl else buf[bl-20:bl]).decode('latin-1'),
-                                                                     bl-bp))
+                                                                   bp, bl, fpos,
+         (buf[bp:min(bl, bp + 20)] if bp < bl else buf[bl-20:bl]).decode('latin-1'),
+                                                                     bl - bp))
     bp += 10
     n += 1
     wtype: int = -1
@@ -72,31 +72,31 @@
       # there should always be enough in the buffer to complete this loop,
       #  because of the buffer update logic at the end
       eol = buf.index(b'\r\n', bp)
-      if buf.startswith(b"Content-Length: ",bp):
-        length=wl=int(bufView[bp+16:eol])
-      if buf.startswith(b"WARC-Truncated: ",bp):
-        if bp+16==eol-2:
+      if buf.startswith(b"Content-Length: ", bp):
+        length = wl = int(bufView[bp+16:eol])
+      if buf.startswith(b"WARC-Truncated: ", bp):
+        if bp + 16 == eol - 2:
           tr = b"EMPTY"
         else:
-          tr = bytes(bufView[bp+16:eol-2])
-      elif buf.startswith(b'WARC-Type: ',bp):
-        if buf.startswith(b's',bp+13):
+          tr = bytes(bufView[bp + 16:eol - 2])
+      elif buf.startswith(b'WARC-Type: ', bp):
+        if buf.startswith(b's', bp + 13):
           wtype = RESP
-        elif buf.startswith(b'q',bp+13):
+        elif buf.startswith(b'q', bp + 13):
           wtype = REQ
-        elif buf.startswith(b'm',bp+11):
+        elif buf.startswith(b'm', bp + 11):
           wtype = META
-        elif buf.startswith(b'w',bp+11):
+        elif buf.startswith(b'w', bp + 11):
           wtype = INFO
-        elif buf.startswith(b'v',bp+13):
+        elif buf.startswith(b'v', bp + 13):
           wtype = REVISIT
         else:
           raise ValueError("Unknown WARC-Type: %s in %s at %s"%(
-                             bytes(bufView[bp+11:eol-2]),filename,
-                             fpos-(bl-bp)))
+                             bytes(bufView[bp + 11:eol - 2]), filename,
+                             fpos - (bl - bp)))
       bp=eol+2
     # record header done
-    if (hl:=(bp - start_1)) > HDRMAX:
+    if (hl := (bp - start_1)) > HDRMAX:
       HDRMAX = hl
     #if done:
     #  if (bp+length)>bl:
@@ -105,27 +105,27 @@
     if (wtype in types):
       # Output whole or part 1 as required
       if whole:
-        _out=callback(wtype,bufView[start_1:bp+length],7)
+        _out = callback(wtype,bufView[start_1:bp+length], 7)
       else:
         if (parts & 1):
-          bp = eol+2
-          _out=callback(wtype,bufView[start_1:bp],1)
-        if parts!=1:
-          while buf.startswith(b'\r\n',bp):
-            bp+=2
-          start_2=bp
-          eob=bp+length
-          while buf.startswith(b'\r\n',eob-2):
-            eob-=2
+          bp = eol + 2
+          _out = callback(wtype, bufView[start_1:bp], 1)
+        if parts != 1:
+          while buf.startswith(b'\r\n', bp):
+            bp += 2
+          start_2: int = bp
+          eob = bp + length
+          while buf.startswith(b'\r\n', eob - 2):
+            eob -= 2
           # Only output parts (2 = HTTP header, 4 = body) that are wanted
           if parts & 2:
             if wtype == RESP or wtype == REQ :
               # request and response have http headers
-              eo2=buf.index(b'\r\n\r\n',start_2)
-              _out=callback(wtype,bufView[start_2:eo2+2],2)
+              eo2 = buf.index(b'\r\n\r\n', start_2)
+              _out = callback(wtype, bufView[start_2:eo2 + 2], 2)
             else:
               # rest of the part
-              _out=callback(wtype,bufView[start_2:eob],2)
+              _out = callback(wtype, bufView[start_2:eob], 2)
           if parts & 4:
             raise ValueError("Not implemented: body part (4): %s"%parts)
     #bp += length
@@ -142,22 +142,22 @@
     keepLen: int
     if (not done) and (keepLen := bl - bp) < BUFMIN:
       # we need to shift and read more
-      buf[0:keepLen]=bufView[bp:bl]
+      buf[0:keepLen] = bufView[bp:bl]
       with memoryview(buf)[keepLen:BUFSIZE] as xBuf:
-        nb=stream.readinto(xBuf)
+        nb = stream.readinto(xBuf)
       bl = keepLen+nb
       done = bl < BUFSIZE 
       bp = 0
-    while buf.startswith(b'\r\n',bp):
+    while buf.startswith(b'\r\n', bp):
       bp+=2
     #print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr)
   print('%d records, max record: %d, max header: %d'%(n, RECORDMAX, HDRMAX),
-        file=sys.stderr)
+        file = sys.stderr)
 
 import zlib, gzip, struct
 from isal import isal_zlib
 
-def decompOneBlock(data: memoryview, bl: int, bp: int = 0):
+def decompOneBlock(data: char[::1], bl: int, bp: int = 0) -> tuple(bytes, int):
     """Decompress one block of a gzip compressed stream in one shot.
     Return the decompressed string and the stream repositioned
       at the start of the next block.
@@ -166,15 +166,17 @@
     fp.seek(bp)
     if gzip._read_gzip_header(fp) is None:
         return (b"",0)
-    bp=fp.tell()
+    bp: int = fp.tell()
     # Use a isal's zlib raw deflate compressor
-    do = isal_zlib.decompressobj(wbits=-zlib.MAX_WBITS)
+    do: isal_zlib.Decompress = isal_zlib.decompressobj(wbits = -zlib.MAX_WBITS)
     # Read all the data except the header
-    decompressed = do.decompress(data[bp:])
+    decompressed: bytes = do.decompress(data[bp:])
     if not do.eof or (uu:=len(do.unused_data)) < 8:
         raise EOFError("Compressed file ended before the end-of-stream "
                        "marker was reached")
     bp = bl - uu
+    crc: int
+    length: int
     crc, length = struct.unpack("<II", do.unused_data[:8])
     bp += 8
     if crc != zlib.crc32(decompressed):
@@ -186,13 +188,16 @@
     return (decompressed, bp)
 
 def dotest():
-  f=open(sys.argv[1],"rb")
-  buf = bytearray(BUFSIZE)
-  bufView = memoryview(buf)
+  f: io.BinaryIO = open(sys.argv[1], "rb")
+  buf: char[::1] = bytearray(BUFSIZE)
+  bufView: char[::1] = memoryview(buf)
+  xBuf: char[::1]
   bl: int = f.readinto(buf)
   offset: int = 0
   bp: int = 0
   done: bool = bl < BUFSIZE 
+  unc: bytes
+  keepLen: int
   print(0, file = sys.stderr, end = " ")
   while True:
     (unc, bp) = decompOneBlock(bufView, bl, bp)
@@ -205,7 +210,7 @@
       offset += bp
       buf[0:keepLen]=bufView[bp:bl]
       with memoryview(buf)[keepLen:BUFSIZE] as xBuf:
-        nb=f.readinto(xBuf)
+        nb: int = f.readinto(xBuf)
       bl = keepLen+nb
       if (done := (bl < BUFSIZE)):
         bufView = bufView[0:bl]