Mercurial > hg > cc > cirrus_work
changeset 351:444ddf704502 trim
switch to plain GzipFile for comparison purposes,
lots of syntax tidying
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Thu, 12 Mar 2026 18:20:01 +0000 |
| parents | 4ead0a390a29 |
| children | 4497f2ad3a6f |
| files | lib/python/cc/warc.py |
| diffstat | 1 files changed, 55 insertions(+), 50 deletions(-) [+] |
line wrap: on
line diff
--- a/lib/python/cc/warc.py Thu Mar 12 18:08:54 2026 +0000 +++ b/lib/python/cc/warc.py Thu Mar 12 18:20:01 2026 +0000 @@ -5,11 +5,9 @@ part''' import sys, io -from isal import igzip +#from isal import igzip import cython, typing, gzip -# see warc.pxd for function signatures - INFO: int = 0 RESP: int = 1 REQ: int = 2 @@ -44,25 +42,27 @@ try: fh.read(1) except gzip.BadGzipFile: - stream = open(filename,'rb',0) + stream = open(filename, 'rb', 0) else: - stream = igzip.IGzipFile(filename=filename) + stream = gzip.GzipFile(filename = filename) buf: char[::1] = bytearray(BUFSIZE) bufView: char[::1] = memoryview(buf) fpos: int = 0 bp: int = 0 bl: int = stream.readinto(buf) n: int = 0 + eob: int + eo2: int done: bool = bl < BUFSIZE - while buf.startswith(b'\r\n',bp): - bp+=2 + while buf.startswith(b'\r\n', bp): + bp += 2 while not (done and bp >= bl): start_1: int = bp - if not buf.startswith(b'WARC/1.0\r\n',bp): + if not buf.startswith(b'WARC/1.0\r\n', bp): raise ValueError("Not a WARC file? In %s at %s of %s (%s): %s[%s]"%(filename, - bp,bl,fpos, - (buf[bp:min(bl,bp+20)] if bp<bl else buf[bl-20:bl]).decode('latin-1'), - bl-bp)) + bp, bl, fpos, + (buf[bp:min(bl, bp + 20)] if bp < bl else buf[bl-20:bl]).decode('latin-1'), + bl - bp)) bp += 10 n += 1 wtype: int = -1 @@ -72,31 +72,31 @@ # there should always be enough in the buffer to complete this loop, # because of the buffer update logic at the end eol = buf.index(b'\r\n', bp) - if buf.startswith(b"Content-Length: ",bp): - length=wl=int(bufView[bp+16:eol]) - if buf.startswith(b"WARC-Truncated: ",bp): - if bp+16==eol-2: + if buf.startswith(b"Content-Length: ", bp): + length = wl = int(bufView[bp+16:eol]) + if buf.startswith(b"WARC-Truncated: ", bp): + if bp + 16 == eol - 2: tr = b"EMPTY" else: - tr = bytes(bufView[bp+16:eol-2]) - elif buf.startswith(b'WARC-Type: ',bp): - if buf.startswith(b's',bp+13): + tr = bytes(bufView[bp + 16:eol - 2]) + elif buf.startswith(b'WARC-Type: ', bp): + if buf.startswith(b's', bp + 13): wtype = RESP - elif buf.startswith(b'q',bp+13): + elif buf.startswith(b'q', bp + 13): wtype = REQ - elif buf.startswith(b'm',bp+11): + elif buf.startswith(b'm', bp + 11): wtype = META - elif buf.startswith(b'w',bp+11): + elif buf.startswith(b'w', bp + 11): wtype = INFO - elif buf.startswith(b'v',bp+13): + elif buf.startswith(b'v', bp + 13): wtype = REVISIT else: raise ValueError("Unknown WARC-Type: %s in %s at %s"%( - bytes(bufView[bp+11:eol-2]),filename, - fpos-(bl-bp))) + bytes(bufView[bp + 11:eol - 2]), filename, + fpos - (bl - bp))) bp=eol+2 # record header done - if (hl:=(bp - start_1)) > HDRMAX: + if (hl := (bp - start_1)) > HDRMAX: HDRMAX = hl #if done: # if (bp+length)>bl: @@ -105,27 +105,27 @@ if (wtype in types): # Output whole or part 1 as required if whole: - _out=callback(wtype,bufView[start_1:bp+length],7) + _out = callback(wtype,bufView[start_1:bp+length], 7) else: if (parts & 1): - bp = eol+2 - _out=callback(wtype,bufView[start_1:bp],1) - if parts!=1: - while buf.startswith(b'\r\n',bp): - bp+=2 - start_2=bp - eob=bp+length - while buf.startswith(b'\r\n',eob-2): - eob-=2 + bp = eol + 2 + _out = callback(wtype, bufView[start_1:bp], 1) + if parts != 1: + while buf.startswith(b'\r\n', bp): + bp += 2 + start_2: int = bp + eob = bp + length + while buf.startswith(b'\r\n', eob - 2): + eob -= 2 # Only output parts (2 = HTTP header, 4 = body) that are wanted if parts & 2: if wtype == RESP or wtype == REQ : # request and response have http headers - eo2=buf.index(b'\r\n\r\n',start_2) - _out=callback(wtype,bufView[start_2:eo2+2],2) + eo2 = buf.index(b'\r\n\r\n', start_2) + _out = callback(wtype, bufView[start_2:eo2 + 2], 2) else: # rest of the part - _out=callback(wtype,bufView[start_2:eob],2) + _out = callback(wtype, bufView[start_2:eob], 2) if parts & 4: raise ValueError("Not implemented: body part (4): %s"%parts) #bp += length @@ -142,22 +142,22 @@ keepLen: int if (not done) and (keepLen := bl - bp) < BUFMIN: # we need to shift and read more - buf[0:keepLen]=bufView[bp:bl] + buf[0:keepLen] = bufView[bp:bl] with memoryview(buf)[keepLen:BUFSIZE] as xBuf: - nb=stream.readinto(xBuf) + nb = stream.readinto(xBuf) bl = keepLen+nb done = bl < BUFSIZE bp = 0 - while buf.startswith(b'\r\n',bp): + while buf.startswith(b'\r\n', bp): bp+=2 #print('end of loop',wtype,start_1,bp,eol,length,bl,file=sys.stderr) print('%d records, max record: %d, max header: %d'%(n, RECORDMAX, HDRMAX), - file=sys.stderr) + file = sys.stderr) import zlib, gzip, struct from isal import isal_zlib -def decompOneBlock(data: memoryview, bl: int, bp: int = 0): +def decompOneBlock(data: char[::1], bl: int, bp: int = 0) -> tuple(bytes, int): """Decompress one block of a gzip compressed stream in one shot. Return the decompressed string and the stream repositioned at the start of the next block. @@ -166,15 +166,17 @@ fp.seek(bp) if gzip._read_gzip_header(fp) is None: return (b"",0) - bp=fp.tell() + bp: int = fp.tell() # Use a isal's zlib raw deflate compressor - do = isal_zlib.decompressobj(wbits=-zlib.MAX_WBITS) + do: isal_zlib.Decompress = isal_zlib.decompressobj(wbits = -zlib.MAX_WBITS) # Read all the data except the header - decompressed = do.decompress(data[bp:]) + decompressed: bytes = do.decompress(data[bp:]) if not do.eof or (uu:=len(do.unused_data)) < 8: raise EOFError("Compressed file ended before the end-of-stream " "marker was reached") bp = bl - uu + crc: int + length: int crc, length = struct.unpack("<II", do.unused_data[:8]) bp += 8 if crc != zlib.crc32(decompressed): @@ -186,13 +188,16 @@ return (decompressed, bp) def dotest(): - f=open(sys.argv[1],"rb") - buf = bytearray(BUFSIZE) - bufView = memoryview(buf) + f: io.BinaryIO = open(sys.argv[1], "rb") + buf: char[::1] = bytearray(BUFSIZE) + bufView: char[::1] = memoryview(buf) + xBuf: char[::1] bl: int = f.readinto(buf) offset: int = 0 bp: int = 0 done: bool = bl < BUFSIZE + unc: bytes + keepLen: int print(0, file = sys.stderr, end = " ") while True: (unc, bp) = decompOneBlock(bufView, bl, bp) @@ -205,7 +210,7 @@ offset += bp buf[0:keepLen]=bufView[bp:bl] with memoryview(buf)[keepLen:BUFSIZE] as xBuf: - nb=f.readinto(xBuf) + nb: int = f.readinto(xBuf) bl = keepLen+nb if (done := (bl < BUFSIZE)): bufView = bufView[0:bl]