Mercurial > hg > cc > cirrus_work
changeset 357:2874da240b73 plus
simple test of block ranges appears to work
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Mon, 16 Mar 2026 13:18:51 +0000 |
| parents | 0367405a02fc |
| children | 22a952de4b91 |
| files | lib/python/cc/test_warc.py lib/python/cc/warc.py |
| diffstat | 2 files changed, 18 insertions(+), 5 deletions(-) [+] |
line wrap: on
line diff
--- a/lib/python/cc/test_warc.py Mon Mar 16 12:25:29 2026 +0000 +++ b/lib/python/cc/test_warc.py Mon Mar 16 13:18:51 2026 +0000 @@ -11,11 +11,15 @@ tt=int(sys.argv.pop(1)) -def showme(wtype,buf,part): +def showme(wtype,buf,part,brange = None): # This should exactly reproduce a complete warc file if called # as per version 1 below if debug: OUT.write(b"----start %d-----\n"%part) + if brange: + OUT.write(buf[0:10]) # print the WARC1.0 line + OUT.write(b"WARC-Range: %d %d\n"%brange) + buf = buf[10:] OUT.write(buf) if buf[-1]!=10: OUT.write(b'\r\n')
--- a/lib/python/cc/warc.py Mon Mar 16 12:25:29 2026 +0000 +++ b/lib/python/cc/warc.py Mon Mar 16 13:18:51 2026 +0000 @@ -112,14 +112,19 @@ # if (bp+length)>bl: # raise ValueError("Done but need more! %s + %s > %s in %s"%(bp, # length,bl,filename)) + brange = Q.popleft() if (wtype in types): # Output whole or part 1 as required if whole: - _out = callback(wtype,bufView[start_1:bp+length], 7) + args = (wtype,bufView[start_1:bp+length], 7, brange) if block \ + else (wtype,bufView[start_1:bp+length], 7) + _out = callback(*args) else: if (parts & 1): bp = eol + 2 - _out = callback(wtype, bufView[start_1:bp], 1) + args = (wtype, bufView[start_1:bp], 1, brange) if block \ + else (wtype, bufView[start_1:bp], 1) + _out = callback(*args) if parts != 1: while buf.startswith(b'\r\n', bp): bp += 2 @@ -132,10 +137,14 @@ if wtype == RESP or wtype == REQ : # request and response have http headers eo2 = buf.index(b'\r\n\r\n', start_2) - _out = callback(wtype, bufView[start_2:eo2 + 2], 2) + args = (wtype, bufView[start_2:eo2 + 2], 2, brange) if block \ + else (wtype, bufView[start_2:eo2 + 2], 2) + _out = callback(*args) else: # rest of the part - _out = callback(wtype, bufView[start_2:eob], 2) + args = (wtype, bufView[start_2:eob], 2, brange) if block \ + else (wtype, bufView[start_2:eob], 2) + _out = callback(*args) if parts & 4: raise ValueError("Not implemented: body part (4): %s"%parts) #bp += length