view bin/test_warc.py @ 45:212da3fe3b19

make test 1 idempotent
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 05 Jul 2023 19:32:02 +0100
parents 689a0e311cd2
children e20c64917805
line wrap: on
line source

import warc,sys

OUT=open(sys.stdout.fileno(),'wb')

if (debug:=(sys.argv[1]=='-d')):
  sys.argv.pop(1)

def showme(wtype,buf,part):
  # This should exactly reproduce a complete warc file if called
  #  as per version 1 below
  if debug:
    OUT.write(b"%d\n%b"%(part,buf))
  else:
    OUT.write(buf)
    if buf[-1]!=b'\n':
      OUT.write(b'\r\n')
    if part==7:
      OUT.write(b'\r\n') # to match complete file formatting

# 1 warc.warc(sys.argv[1],showme,[b'response','warcinfo','request','metadata'],whole=True)

# 2
warc.warc(sys.argv[1],showme,[b'response'],parts=int(sys.argv[2]),debug=debug)