changeset 349:a8a5f07211c4 trim

Now using isal_zlib.decompressobj
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 11 Mar 2026 18:42:21 +0000
parents ceacc13bf0e7
children 4ead0a390a29
files lib/python/cc/warc.py
diffstat 1 files changed, 13 insertions(+), 5 deletions(-) [+]
line wrap: on
line diff
--- a/lib/python/cc/warc.py	Wed Mar 11 18:20:05 2026 +0000
+++ b/lib/python/cc/warc.py	Wed Mar 11 18:42:21 2026 +0000
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# cython: profile=False, language_level=3str
 '''Stream a warc format file, unzipping if necessary, invoking a
 callback on each record.  Callback can be limited by WARC-Type, record
 part'''
@@ -166,8 +167,8 @@
     if gzip._read_gzip_header(fp) is None:
         return (b"",0)
     bp=fp.tell()
-    # Use a zlib raw deflate compressor
-    do = zlib.decompressobj(wbits=-zlib.MAX_WBITS)
+    # Use a isal's zlib raw deflate compressor
+    do = isal_zlib.decompressobj(wbits=-zlib.MAX_WBITS)
     # Read all the data except the header
     decompressed = do.decompress(data[bp:])
     if not do.eof or (uu:=len(do.unused_data)) < 8:
@@ -177,14 +178,14 @@
     crc, length = struct.unpack("<II", do.unused_data[:8])
     bp += 8
     if crc != zlib.crc32(decompressed):
-        raise BadGzipFile("CRC check failed")
+        raise gzip.BadGzipFile("CRC check failed")
     if length != (len(decompressed) & 0xffffffff):
-        raise BadGzipFile("Incorrect length of data produced")
+        raise gzip.BadGzipFile("Incorrect length of data produced")
     while bp < bl and data[bp] == 0:
         bp += 1
     return (decompressed, bp)
 
-if __name__ == "__main__":
+def dotest():
   f=open(sys.argv[1],"rb")
   buf = bytearray(BUFSIZE)
   bufView = memoryview(buf)
@@ -211,3 +212,10 @@
       bp = 0
   print("EOF", file = sys.stderr)
   f.close()
+
+if __name__ == "__main__":
+  dotest()
+
+
+
+