Mercurial > hg > cc > cirrus_home

comparison bin/warc.py @ 176:97137f5bbe0f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
working, about to move to work tree
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 05 Jul 2023 14:50:00 +0100
parents d123ef7fdb82
children
comparison
equal deleted inserted replaced
175:d123ef7fdb82 176:97137f5bbe0f
4 import sys,os,io 4 import sys,os,io
5 5
6 if (debug:=(sys.argv[1]=='-d')): 6 if (debug:=(sys.argv[1]=='-d')):
7 sys.argv.pop(1) 7 sys.argv.pop(1)
8 8
9 def warc(callback,types=['response'],parts=7): 9 def warc(callback,types=['response'],whole=False,parts=7):
10 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types] 10 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types]
11 nb=0 11 nb=0
12 stream=open(sys.argv[1],'rb',0) 12 stream=open(sys.argv[1],'rb',0)
13 bufsize=128*1024*1024 13 bufsize=128*1024*1024
14 buf=bytearray(128*1024*1024) 14 buf=bytearray(128*1024*1024)
15 l=b'\r\n' 15 l=b'\r\n'
16 while True: 16 while not stream.closed:
17 bp=0 17 bp=0
18 while l==b'\r\n': 18 while l==b'\r\n':
19 l=stream.readline() 19 l=stream.readline()
20 nb+=(ln:=len(l)) 20 nb+=(ln:=len(l))
21 if ln==0:
22 break
21 if l!=b'WARC/1.0\r\n': 23 if l!=b'WARC/1.0\r\n':
22 raise ValueError("Not a WARC file? At %s: %s[%s]"%(nb-len(l), 24 raise ValueError("Not a WARC file? At %s: %s[%s]"%(nb-len(l),
23 l.decode('latin-1'),len(l))) 25 l.decode('latin-1'),len(l)))
24 wtype=None 26 wtype=None
25 length=None 27 length=None
26 state=1 28 state=1
27 tr=None # Was this record truncated? 29 tr=None # Was this record truncated?
28 while l!=b'\r\n': 30 while l!=b'\r\n':
31 # WARC header
29 if parts & 1: 32 if parts & 1:
30 buf[bp:(bp:=bp+ln)]=l 33 buf[bp:(bp:=bp+ln)]=l
31 l=stream.readline() 34 l=stream.readline()
32 nb+=(ln:=len(l)) 35 nb+=(ln:=len(l))
33 # WARC header
34 if l.startswith(b"Content-Length: "): 36 if l.startswith(b"Content-Length: "):
35 length=wl=int(l[16:].rstrip()) 37 length=wl=int(l[16:].rstrip())
36 elif l.startswith(b"WARC-Truncated: "): 38 elif l.startswith(b"WARC-Truncated: "):
37 tr=l[16:].rstrip() 39 tr=l[16:].rstrip()
38 tr="EMPTY" if tr=="" else tr 40 tr="EMPTY" if tr=="" else tr
39 elif l.startswith(b'WARC-Type: '): 41 elif l.startswith(b'WARC-Type: '):
40 wtype = l[11:-2] 42 wtype = l[11:-2]
41 start_2=bp 43 start_2=bp
42 if (wtype in types) and (parts & 1): 44 if (wtype in types):
43 if parts!=1: 45 if whole:
44 buf[bp:(bp:=bp+ln)]=l 46 buf[bp:(bp:=bp+ln)]=l
47 elif (parts & 1):
48 callback(wtype,buf[:start_2],1)
49 if parts==1:
50 start_2=0
51 else:
45 start_2=bp 52 start_2=bp
46 if parts!=7:
47 callback(wtype,buf[:start_2],1)
48 else: 53 else:
49 start_2=0 54 start_2=0
50 bv=memoryview(buf)[start_2:start_2+length] 55 bv=memoryview(buf)[start_2:start_2+length]
51 ii=0 56 ii=0
52 while True and not stream.closed: 57 while True and not stream.closed:
57 break 62 break
58 bv=memoryview(buf)[start_2+ii:start_2+length] 63 bv=memoryview(buf)[start_2+ii:start_2+length]
59 if ii!=length: 64 if ii!=length:
60 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length)) 65 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length))
61 nb+=length 66 nb+=length
62 bv=memoryview(buf)[start_2:start_2+length]
63 if wtype in types: 67 if wtype in types:
64 if parts==7: 68 if whole:
65 callback(wtype,memoryview(buf)[0:start_2+length],7) 69 callback(wtype,buf[0:start_2+length],7)
66 continue 70 continue
67 # Only output parts (1 = WARC header, 2 = HTTP header, 4 = body) that are wanted 71 # Only output parts (1 = WARC header, 2 = HTTP header, 4 = body) that are wanted
68 bl=None # for HTTP Content-Length for the length of the body? 72 bl=None # for HTTP Content-Length for the length of the body?
69 L_start=0 73 L_start=start_2
70 state=2 74 state=2
75 bv=memoryview(buf)[start_2:start_2+length]
71 with io.BytesIO(bv) as rec_text: 76 with io.BytesIO(bv) as rec_text:
72 for L in rec_text: 77 for L in rec_text:
73 if state==2: 78 if state==2:
74 # HTTP header 79 # HTTP header
75 wl -= len(L) 80 wl -= len(L)
78 if bl is None and L.startswith(b"Content-Length: "): 83 if bl is None and L.startswith(b"Content-Length: "):
79 bl=int(L[16:].rstrip()) 84 bl=int(L[16:].rstrip())
80 else: 85 else:
81 # Blank line, HTTP header is finished 86 # Blank line, HTTP header is finished
82 if parts & 2: 87 if parts & 2:
83 callback(wtype,bv[start_2:L_start],2) 88 callback(wtype,buf[start_2:start_2+L_start],2)
84 state=4 89 state=4
85 # The above is just for sanity, because we do _not_ 90 # The above is just for sanity, because we do _not_
86 # continue with the outer loop, 91 # continue with the outer loop,
87 # since we can now block-output the entire rest of the 92 # since we can now block-output the entire rest of the
88 # input buffer. 93 # input buffer.
89 if bl is not None: 94 if bl is not None:
90 if bl!=wl: 95 if bl!=wl:
91 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\ 96 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
92 (length,offset,filename,wl,bl,tr),file=sys.stderr) 97 (length,offset,filename,wl,bl,tr),file=sys.stderr)
93 # HTTP body 98 # HTTP body
94 balance=rec_text.tell() 99 balance=start_2+rec_text.tell()
95 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr) 100 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
96 # Output whatever is left 101 # Output whatever is left
97 if parts & 4: 102 if parts & 4:
98 callback(wtype,bv[balance:balance+wl],4) 103 callback(wtype,buf[balance:balance+wl],4)
99 state=1 104 state=1
100 105
101 L_start=rec_text.tell() 106 L_start=rec_text.tell()
102 OUT=open(sys.stdout.fileno(),'wb') 107 OUT=open(sys.stdout.fileno(),'wb')
103 108
104 import re 109 import re
110 TUPAT=re.compile(b'^WARC-Target-URI: (.*?)\r',re.MULTILINE)
105 LMPAT=re.compile(b'^Last-Modified: (.*?)\r',re.MULTILINE) 111 LMPAT=re.compile(b'^Last-Modified: (.*?)\r',re.MULTILINE)
106 112
107 def showmeLMH(wtype,buf,part=2): 113 def showmeLMH(wtype,buf,part):
108 m=LMPAT.search(buf.tobytes(order='A')) 114 global URI
109 if m: 115 if part==1:
110 OUT.write(m[1]) 116 if (m:=TUPAT.search(buf)):
111 OUT.write(b'\n') 117 URI=m[1]
118 else:
119 raise ValueError(b"No target URI in %s ??"%buf)
120 else:
121 m=LMPAT.search(buf)
122 OUT.write(URI)
123 if m:
124 OUT.write(b'\t')
125 OUT.write(m[1])
126 OUT.write(b'\n')
112 127
113 def showme(wtype,buf,part): 128 def showme(wtype,buf,part):
114 if debug: 129 if debug:
115 breakpoint() 130 breakpoint()
116 OUT.write(b"%d\n%b"%(part,buf)) 131 OUT.write(b"%d\n%b"%(part,buf))
117 132
118 #warc(showmeLMH,[b'response'],2) 133 warc(showmeLMH,[b'response'],parts=3)
119 134
120 #warc(showme,[b'response','warcinfo','request','metadata'],int(sys.argv[2])) 135 #warc(showme,[b'response','warcinfo','request','metadata'],int(sys.argv[2]))
121 136
122 warc(showme,[b'response'],int(sys.argv[2])) 137 #warc(showme,[b'response'],parts=int(sys.argv[2]))
138 #warc(showme,[b'response'],whole=True)