Mercurial > hg > cc > cirrus_work
view bin/lmh_warc.py @ 90:c1a70532444c
flip loops
author | Henry S. Thompson <ht@inf.ed.ac.uk> |
---|---|
date | Thu, 31 Aug 2023 14:14:21 +0100 |
parents | 120d90b47d74 |
children | 86df63d251cf |
line wrap: on
line source
#!/usr/bin/env python3 import re,warc,sys TUPAT=re.compile(b'^WARC-Target-URI: (.*?)\r',re.MULTILINE) DPAT=re.compile(b'^WARC-Date: (.*?)\r',re.MULTILINE) LMPAT=re.compile(b'^Last-Modified: (.*?)\r',re.MULTILINE) DTAB=bytearray(range(256)) DDEL=b'TZ-:' OUT=open(sys.stdout.fileno(),'wb') def showmeLMH(wtype,buf,part): global URI, DATE if part==1: if (m:=TUPAT.search(buf)): URI=m[1] else: raise ValueError(b"No target URI in %s ??"%buf) if (md:=DPAT.search(buf)): DATE=md[1] else: raise ValueError(b"No date in %s ??"%buf) else: mm=LMPAT.search(buf) OUT.write(URI) if mm: OUT.write(b'\t') OUT.write(DATE.translate(DTAB,DDEL)) OUT.write(b'\t') OUT.write(mm[1]) OUT.write(b'\n') warc.warc(sys.argv[1],showmeLMH,[b'response'],parts=3)