|
39
|
1 #!/usr/bin/env python3
|
|
|
2 '''Stream a warc format file, invoking a callback on each record.
|
|
|
3 Callback can be limited by WARC-Type, record part'''
|
|
|
4 import sys,os,io
|
|
|
5
|
|
|
6 if (debug:=(sys.argv[1]=='-d')):
|
|
|
7 sys.argv.pop(1)
|
|
|
8
|
|
|
9 def warc(callback,types=['response'],whole=False,parts=7):
|
|
|
10 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types]
|
|
|
11 nb=0
|
|
|
12 stream=open(sys.argv[1],'rb',0)
|
|
|
13 bufsize=128*1024*1024
|
|
|
14 buf=bytearray(128*1024*1024)
|
|
|
15 l=b'\r\n'
|
|
|
16 while not stream.closed:
|
|
|
17 bp=0
|
|
|
18 while l==b'\r\n':
|
|
|
19 l=stream.readline()
|
|
|
20 nb+=(ln:=len(l))
|
|
|
21 if ln==0:
|
|
|
22 break
|
|
|
23 if l!=b'WARC/1.0\r\n':
|
|
|
24 raise ValueError("Not a WARC file? At %s: %s[%s]"%(nb-len(l),
|
|
|
25 l.decode('latin-1'),len(l)))
|
|
|
26 wtype=None
|
|
|
27 length=None
|
|
|
28 state=1
|
|
|
29 tr=None # Was this record truncated?
|
|
|
30 while l!=b'\r\n':
|
|
|
31 # WARC header
|
|
|
32 if parts & 1:
|
|
|
33 buf[bp:(bp:=bp+ln)]=l
|
|
|
34 l=stream.readline()
|
|
|
35 nb+=(ln:=len(l))
|
|
|
36 if l.startswith(b"Content-Length: "):
|
|
|
37 length=wl=int(l[16:].rstrip())
|
|
|
38 elif l.startswith(b"WARC-Truncated: "):
|
|
|
39 tr=l[16:].rstrip()
|
|
|
40 tr="EMPTY" if tr=="" else tr
|
|
|
41 elif l.startswith(b'WARC-Type: '):
|
|
|
42 wtype = l[11:-2]
|
|
|
43 start_2=bp
|
|
|
44 if (wtype in types):
|
|
|
45 if whole:
|
|
|
46 buf[bp:(bp:=bp+ln)]=l
|
|
|
47 elif (parts & 1):
|
|
|
48 callback(wtype,buf[:start_2],1)
|
|
|
49 if parts==1:
|
|
|
50 start_2=0
|
|
|
51 else:
|
|
|
52 start_2=bp
|
|
|
53 else:
|
|
|
54 start_2=0
|
|
|
55 bv=memoryview(buf)[start_2:start_2+length]
|
|
|
56 ii=0
|
|
|
57 while True and not stream.closed:
|
|
|
58 if (i:=stream.readinto(bv))==0:
|
|
|
59 break
|
|
|
60 ii+=i
|
|
|
61 if ii>=length:
|
|
|
62 break
|
|
|
63 bv=memoryview(buf)[start_2+ii:start_2+length]
|
|
|
64 if ii!=length:
|
|
|
65 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length))
|
|
|
66 nb+=length
|
|
|
67 if wtype in types:
|
|
|
68 if whole:
|
|
|
69 callback(wtype,buf[0:start_2+length],7)
|
|
|
70 continue
|
|
|
71 # Only output parts (1 = WARC header, 2 = HTTP header, 4 = body) that are wanted
|
|
|
72 bl=None # for HTTP Content-Length for the length of the body?
|
|
|
73 L_start=start_2
|
|
|
74 state=2
|
|
|
75 bv=memoryview(buf)[start_2:start_2+length]
|
|
|
76 with io.BytesIO(bv) as rec_text:
|
|
|
77 for L in rec_text:
|
|
|
78 if state==2:
|
|
|
79 # HTTP header
|
|
|
80 wl -= len(L)
|
|
|
81 if not (L==b"" or L.startswith(b"\r")):
|
|
|
82 # Non-empty, it's (a continuation of) a header
|
|
|
83 if bl is None and L.startswith(b"Content-Length: "):
|
|
|
84 bl=int(L[16:].rstrip())
|
|
|
85 else:
|
|
|
86 # Blank line, HTTP header is finished
|
|
|
87 if parts & 2:
|
|
|
88 callback(wtype,buf[start_2:start_2+L_start],2)
|
|
|
89 state=4
|
|
|
90 # The above is just for sanity, because we do _not_
|
|
|
91 # continue with the outer loop,
|
|
|
92 # since we can now block-output the entire rest of the
|
|
|
93 # input buffer.
|
|
|
94 if bl is not None:
|
|
|
95 if bl!=wl:
|
|
|
96 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
|
|
|
97 (length,offset,filename,wl,bl,tr),file=sys.stderr)
|
|
|
98 # HTTP body
|
|
|
99 balance=start_2+rec_text.tell()
|
|
|
100 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
|
|
|
101 # Output whatever is left
|
|
|
102 if parts & 4:
|
|
|
103 callback(wtype,buf[balance:balance+wl],4)
|
|
|
104 state=1
|
|
|
105
|
|
|
106 L_start=rec_text.tell()
|
|
|
107 OUT=open(sys.stdout.fileno(),'wb')
|
|
|
108
|
|
|
109 import re
|
|
|
110 TUPAT=re.compile(b'^WARC-Target-URI: (.*?)\r',re.MULTILINE)
|
|
|
111 LMPAT=re.compile(b'^Last-Modified: (.*?)\r',re.MULTILINE)
|
|
|
112
|
|
|
113 def showmeLMH(wtype,buf,part):
|
|
|
114 global URI
|
|
|
115 if part==1:
|
|
|
116 if (m:=TUPAT.search(buf)):
|
|
|
117 URI=m[1]
|
|
|
118 else:
|
|
|
119 raise ValueError(b"No target URI in %s ??"%buf)
|
|
|
120 else:
|
|
|
121 m=LMPAT.search(buf)
|
|
|
122 OUT.write(URI)
|
|
|
123 if m:
|
|
|
124 OUT.write(b'\t')
|
|
|
125 OUT.write(m[1])
|
|
|
126 OUT.write(b'\n')
|
|
|
127
|
|
|
128 def showme(wtype,buf,part):
|
|
|
129 if debug:
|
|
|
130 breakpoint()
|
|
|
131 OUT.write(b"%d\n%b"%(part,buf))
|
|
|
132
|
|
|
133 warc(showmeLMH,[b'response'],parts=3)
|
|
|
134
|
|
|
135 #warc(showme,[b'response','warcinfo','request','metadata'],int(sys.argv[2]))
|
|
|
136
|
|
|
137 #warc(showme,[b'response'],parts=int(sys.argv[2]))
|
|
|
138 #warc(showme,[b'response'],whole=True)
|
