Mercurial > hg > cc > cirrus_home

annotate bin/warc.py @ 175:d123ef7fdb82

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
working on implementing types and parts: 1, 2, 4 working, 3 not
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 03 Jul 2023 18:16:14 +0100
parents e96d444b0f84
children 97137f5bbe0f
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/usr/bin/env python3
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
2 '''Stream a warc format file, invoking a callback on each record.
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
3 Callback can be limited by WARC-Type, record part'''
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
4 import sys,os,io
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
6 if (debug:=(sys.argv[1]=='-d')):
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
7 sys.argv.pop(1)
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
8
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
9 def warc(callback,types=['response'],parts=7):
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
10 types=[(t if isinstance(t,bytes) else bytes(t,'utf8')) for t in types]
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 nb=0
139
e96d444b0f84 fixed bug(s) wrt large payload files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 138
diff changeset
12 stream=open(sys.argv[1],'rb',0)
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13 bufsize=128*1024*1024
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14 buf=bytearray(128*1024*1024)
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15 l=b'\r\n'
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16 while True:
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
17 bp=0
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18 while l==b'\r\n':
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19 l=stream.readline()
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
20 nb+=(ln:=len(l))
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21 if l!=b'WARC/1.0\r\n':
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22 raise ValueError("Not a WARC file? At %s: %s[%s]"%(nb-len(l),
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23 l.decode('latin-1'),len(l)))
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 wtype=None
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25 length=None
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
26 state=1
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
27 tr=None # Was this record truncated?
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28 while l!=b'\r\n':
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
29 if parts & 1:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
30 buf[bp:(bp:=bp+ln)]=l
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
31 l=stream.readline()
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
32 nb+=(ln:=len(l))
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
33 # WARC header
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
34 if l.startswith(b"Content-Length: "):
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
35 length=wl=int(l[16:].rstrip())
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
36 elif l.startswith(b"WARC-Truncated: "):
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
37 tr=l[16:].rstrip()
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
38 tr="EMPTY" if tr=="" else tr
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
39 elif l.startswith(b'WARC-Type: '):
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
40 wtype = l[11:-2]
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
41 start_2=bp
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
42 if (wtype in types) and (parts & 1):
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
43 if parts!=1:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
44 buf[bp:(bp:=bp+ln)]=l
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
45 start_2=bp
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
46 if parts!=7:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
47 callback(wtype,buf[:start_2],1)
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
48 else:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
49 start_2=0
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
50 bv=memoryview(buf)[start_2:start_2+length]
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
51 ii=0
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
52 while True and not stream.closed:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
53 if (i:=stream.readinto(bv))==0:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
54 break
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
55 ii+=i
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
56 if ii>=length:
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
57 break
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
58 bv=memoryview(buf)[start_2+ii:start_2+length]
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
59 if ii!=length:
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
60 raise ValueError("Chunk read losing, from %s got %s expected %s"%(nb,ii,length))
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
61 nb+=length
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
62 bv=memoryview(buf)[start_2:start_2+length]
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
63 if wtype in types:
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
64 if parts==7:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
65 callback(wtype,memoryview(buf)[0:start_2+length],7)
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
66 continue
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
67 # Only output parts (1 = WARC header, 2 = HTTP header, 4 = body) that are wanted
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
68 bl=None # for HTTP Content-Length for the length of the body?
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
69 L_start=0
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
70 state=2
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
71 with io.BytesIO(bv) as rec_text:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
72 for L in rec_text:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
73 if state==2:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
74 # HTTP header
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
75 wl -= len(L)
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
76 if not (L==b"" or L.startswith(b"\r")):
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
77 # Non-empty, it's (a continuation of) a header
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
78 if bl is None and L.startswith(b"Content-Length: "):
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
79 bl=int(L[16:].rstrip())
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
80 else:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
81 # Blank line, HTTP header is finished
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
82 if parts & 2:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
83 callback(wtype,bv[start_2:L_start],2)
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
84 state=4
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
85 # The above is just for sanity, because we do _not_
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
86 # continue with the outer loop,
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
87 # since we can now block-output the entire rest of the
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
88 # input buffer.
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
89 if bl is not None:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
90 if bl!=wl:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
91 print("length mismatch: %s %s %s here: %s given: %s trunc: %s"%\
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
92 (length,offset,filename,wl,bl,tr),file=sys.stderr)
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
93 # HTTP body
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
94 balance=rec_text.tell()
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
95 #print(balance,bl,wl,ll,ll-balance,file=sys.stderr)
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
96 # Output whatever is left
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
97 if parts & 4:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
98 callback(wtype,bv[balance:balance+wl],4)
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
99 state=1
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
100
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
101 L_start=rec_text.tell()
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
102 OUT=open(sys.stdout.fileno(),'wb')
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
103
139
e96d444b0f84 fixed bug(s) wrt large payload files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 138
diff changeset
104 import re
e96d444b0f84 fixed bug(s) wrt large payload files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 138
diff changeset
105 LMPAT=re.compile(b'^Last-Modified: (.*?)\r',re.MULTILINE)
e96d444b0f84 fixed bug(s) wrt large payload files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 138
diff changeset
106
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
107 def showmeLMH(wtype,buf,part=2):
139
e96d444b0f84 fixed bug(s) wrt large payload files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 138
diff changeset
108 m=LMPAT.search(buf.tobytes(order='A'))
e96d444b0f84 fixed bug(s) wrt large payload files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 138
diff changeset
109 if m:
e96d444b0f84 fixed bug(s) wrt large payload files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 138
diff changeset
110 OUT.write(m[1])
e96d444b0f84 fixed bug(s) wrt large payload files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 138
diff changeset
111 OUT.write(b'\n')
e96d444b0f84 fixed bug(s) wrt large payload files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 138
diff changeset
112
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
113 def showme(wtype,buf,part):
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
114 if debug:
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
115 breakpoint()
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
116 OUT.write(b"%d\n%b"%(part,buf))
138
9ea12f7b304b just barely working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
117
175
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
118 #warc(showmeLMH,[b'response'],2)
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
119
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
120 #warc(showme,[b'response','warcinfo','request','metadata'],int(sys.argv[2]))
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
121
d123ef7fdb82 working on implementing types and parts:
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 139
diff changeset
122 warc(showme,[b'response'],int(sys.argv[2]))