Mercurial > hg > cc > cirrus_work
annotate bin/warc.py @ 53:0dc144bd027c
made 1 mean 1, still losing after a while
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Fri, 07 Jul 2023 19:04:16 +0100 |
| parents | c0b4359dd26a |
| children | 9c63039a9b6d |
| rev | line source |
|---|---|
| 39 | 1 #!/usr/bin/env python3 |
|
46
44d3a4f4ea51
support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
43
diff
changeset
|
2 '''Stream a warc format file, unzipping if necessary, invoking a |
|
44d3a4f4ea51
support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
43
diff
changeset
|
3 callback on each record. Callback can be limited by WARC-Type, record |
|
44d3a4f4ea51
support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
43
diff
changeset
|
4 part''' |
| 39 | 5 |
|
42
689a0e311cd2
make warc.py a library, separate out testing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
39
diff
changeset
|
6 import sys,io |
|
689a0e311cd2
make warc.py a library, separate out testing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
39
diff
changeset
|
7 from isal import igzip |
| 39 | 8 |
|
42
689a0e311cd2
make warc.py a library, separate out testing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
39
diff
changeset
|
9 def warc(filename,callback,types=['response'],whole=False,parts=7,debug=False): |
| 39 | 10 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types] |
| 11 nb=0 | |
|
46
44d3a4f4ea51
support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
43
diff
changeset
|
12 if filename.endswith(".gz"): |
|
44d3a4f4ea51
support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
43
diff
changeset
|
13 stream=igzip.IGzipFile(filename=filename) |
|
44d3a4f4ea51
support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
43
diff
changeset
|
14 else: |
|
44d3a4f4ea51
support on-board unzipping, reduce buffer size to 2MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
43
diff
changeset
|
15 stream=open(filename,'rb',0) |
|
48
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
16 bufSize=2*1024*1024 |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
17 hdrMax=16*1024 |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
18 buf=bytearray(bufSize) |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
19 hdrBuf=memoryview(buf)[:hdrMax] |
|
51
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
20 fpos=bl=stream.readinto(hdrBuf) |
| 50 | 21 while True: |
| 39 | 22 bp=0 |
|
48
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
23 while buf.startswith(b'\r\n',bp): |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
24 bp+=2 |
|
53
0dc144bd027c
made 1 mean 1, still losing after a while
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
51
diff
changeset
|
25 bob=bp |
|
48
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
26 if not buf.startswith(b'WARC/1.0\r\n',bp): |
|
51
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
27 raise ValueError("Not a WARC file? At %s: %s[%s]"%(bp, |
|
48
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
28 buf[bp:min(bl,bp+20)].decode('latin-1'), bl-bp)) |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
29 bp+=10 |
| 39 | 30 wtype=None |
| 31 length=None | |
| 32 state=1 | |
| 50 | 33 done=False |
| 39 | 34 tr=None # Was this record truncated? |
|
48
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
35 while not buf.startswith(b'\r\n',bp): |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
36 eol=buf.index(b'\r\n',bp)+2 |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
37 if buf.startswith(b"Content-Length: ",bp): |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
38 length=wl=int(buf[bp+16:eol-2]) |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
39 elif buf.startswith(b"WARC-Truncated: ",bp): |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
40 tr=l[bp+16:eol-2] |
| 39 | 41 tr="EMPTY" if tr=="" else tr |
|
48
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
42 elif buf.startswith(b'WARC-Type: ',bp): |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
43 wtype = bytes(buf[bp+11:eol-2]) |
|
d0d2fd9830d6
starting on conversion to direct-querying of buffer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
46
diff
changeset
|
44 bp=eol |
|
51
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
45 bp=eol+2 |
| 39 | 46 if (wtype in types): |
| 47 if whole: | |
|
49
699ef141af10
just barely working for 1, need to rethink buffering
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
48
diff
changeset
|
48 pass # buf[bp:(bp:=bp+ln)]=l |
| 39 | 49 elif (parts & 1): |
| 50 | 50 print('cb') |
|
53
0dc144bd027c
made 1 mean 1, still losing after a while
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
51
diff
changeset
|
51 OUT=callback(wtype,buf[bob:eol],1) |
| 50 | 52 if parts!=1: |
| 53 # everything from bv= goes here | |
|
51
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
54 # need to read more if eol+length>hdrMax |
| 50 | 55 pass |
|
51
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
56 print(wtype,bob,eol,length) |
|
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
57 jumpTo=bp+length |
|
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
58 buf[0:hdrMax-jumpTo]=buf[jumpTo:hdrMax] |
|
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
59 stream.seek(fpos:=fpos+jumpTo) |
| 50 | 60 if done: |
|
51
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
61 continue |
|
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
62 n=stream.readinto(memoryview(buf)[hdrMax-jumpTo:hdrMax]) |
|
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
63 print('read',n) |
|
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
64 if n<jumpTo or n==0: |
|
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
65 print('done',n,jumpTo) |
| 50 | 66 done=True |
|
51
c0b4359dd26a
working better, gets confused by 3-part response
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
50
diff
changeset
|
67 bp=0 |
| 50 | 68 #while not buf.startswith(b'\r\n',bp): |
| 69 OUT.write(b"=====\n") | |
| 70 OUT.write(buf[0:100]) | |
| 71 continue | |
| 72 return | |
| 39 | 73 bv=memoryview(buf)[start_2:start_2+length] |
| 74 ii=0 | |
| 75 while True and not stream.closed: | |
| 76 if (i:=stream.readinto(bv))==0: | |
| 77 break | |
| 78 ii+=i | |
| 79 if ii>=length: | |
| 80 break | |
| 81 bv=memoryview(buf)[start_2+ii:start_2+length] | |
| 82 if ii!=length: | |
| 83 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length)) | |
| 84 nb+=length | |
| 85 if wtype in types: | |
| 86 if whole: | |
| 87 callback(wtype,buf[0:start_2+length],7) | |
| 88 continue | |
| 89 # Only output parts (1 = WARC header, 2 = HTTP header, 4 = body) that are wanted | |
| 90 bl=None # for HTTP Content-Length for the length of the body? | |
| 91 L_start=start_2 | |
| 92 state=2 | |
| 93 bv=memoryview(buf)[start_2:start_2+length] | |
| 94 with io.BytesIO(bv) as rec_text: | |
| 95 for L in rec_text: | |
| 96 if state==2: | |
| 97 # HTTP header | |
| 98 wl -= len(L) | |
| 99 if not (L==b"" or L.startswith(b"\r")): | |
| 100 # Non-empty, it's (a continuation of) a header | |
| 101 if bl is None and L.startswith(b"Content-Length: "): | |
| 102 bl=int(L[16:].rstrip()) | |
| 103 else: | |
| 104 # Blank line, HTTP header is finished | |
| 105 if parts & 2: | |
| 106 callback(wtype,buf[start_2:start_2+L_start],2) | |
| 107 state=4 | |
| 108 # The above is just for sanity, because we do _not_ | |
| 109 # continue with the outer loop, | |
| 110 # since we can now block-output the entire rest of the | |
| 111 # input buffer. | |
| 112 if bl is not None: | |
| 113 if bl!=wl: | |
| 114 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\ | |
| 115 (length,offset,filename,wl,bl,tr),file=sys.stderr) | |
| 116 # HTTP body | |
| 117 balance=start_2+rec_text.tell() | |
| 118 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr) | |
| 119 # Output whatever is left | |
| 120 if parts & 4: | |
| 121 callback(wtype,buf[balance:balance+wl],4) | |
| 122 state=1 | |
| 123 | |
| 124 L_start=rec_text.tell() |