Mercurial > hg > cc > cirrus_work

changeset 372:a83d0863a5cf plus

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bigger buffer to accommodate new bigger max size
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 20 Apr 2026 18:19:17 +0100
parents 875724011bc3
children cf25927ef965
files lib/python/cc/warc.py
diffstat 1 files changed, 12 insertions(+), 7 deletions(-) [+]
line wrap: on
line diff
--- a/lib/python/cc/warc.py	Mon Apr 20 18:18:32 2026 +0100
+++ b/lib/python/cc/warc.py	Mon Apr 20 18:19:17 2026 +0100
@@ -16,7 +16,7 @@
 REVISIT: int = 4
 
 BUFSIZE: int = 16 * 1024 * 1024
-BUFMIN: int = 3 * 1024 * 1024 # 1.5MiB, will need to be increased
+BUFMIN: int = 5.5 * 1024 * 1024 # was 3MiB, increased
                              # to 5.5MiB for CC-MAIN-2025-13 (Mar) and thereafter
 
 HDRMAX: int = 0  # will grow
@@ -64,6 +64,7 @@
   n: int = 0
   eob: int
   eo2: int
+  brange: tuple[int, int]
   done: bool = bl < BUFSIZE 
   while buf.startswith(b'\r\n', bp):
     bp += 2
@@ -113,7 +114,8 @@
     #  if (bp+length)>bl:
     #    raise ValueError("Done but need more! %s + %s > %s in %s"%(bp,
     #                     length,bl,filename))
-    brange = Q.popleft()
+    if block:
+      brange = Q.popleft()
     if (wtype in types):
       # Output whole or part 1 as required
       if whole:
@@ -154,7 +156,10 @@
     #  python3 ~/lib/python/cc/test_warc.py 4 /beegfs/common_crawl/CC-MAIN-2019-35/1566027313501.0/orig/crawldiagnostics/CC-MAIN-20190817222907-20190818004907-00000.warc.gz
     #  at a point where bp+length is 11018, looking at >\n\r\n
     #  bp += 1 [doesn't work]
-    bp = buf.index(b'\r\n',bp+length)
+    try:
+      bp = buf.index(b'\r\n',bp+length)
+    except ValueError:
+      breakpoint()
     # check if refill needed
     rl: int
     if (rl := (bp - start_1)) > RECORDMAX:
@@ -177,7 +182,7 @@
 import zlib, gzip, struct
 from isal import isal_zlib
 
-def decompOneBlock(data: char[::1], bl: int, bp: int = 0) -> tuple[bytes, int]:
+def decompOneBlock(data: bytes, bl: int, bp: int = 0) -> tuple[bytes, int]:
     """Decompress one block of a gzip compressed stream in one shot.
     Return the decompressed string and the stream repositioned
       at the start of the next block.
@@ -209,9 +214,9 @@
 
 def dotest():
   f: io.BinaryIO = open(sys.argv[1], "rb")
-  buf: char[::1] = bytearray(BUFSIZE)
-  bufView: char[::1] = memoryview(buf)
-  xBuf: char[::1]
+  buf: bytes = bytearray(BUFSIZE)
+  bufView: bytes = memoryview(buf)
+  xBuf: bytes
   bl: int = f.readinto(buf)
   offset: int = 0
   bp: int = 0