# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1782128032 -3600
# Node ID af49ea848756a63e4daf658eb87076b34dad4521
# Parent  2c04af3ca99a4ca8ca745c4eed0ab397220050d2
sic

diff -r 2c04af3ca99a -r af49ea848756 lib/python/cc/test_iwarc.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/python/cc/test_iwarc.py	Mon Jun 22 12:33:52 2026 +0100
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# cython: profile=False, language_level=3str
+'''Test various parameter settings for complete warc file streaming callback.
+   
+
+Usage: test_iwarc.py [-d] test-number filename warc-types part-mask 
+ -d turns on debugging output
+ filename gives a path to a .warc or .warc.gz WARC file
+ warc-types is list of integers corresponding to WARC record types
+    to be extracted and printed to stdout:
+        warcinfo: 0
+        response: 1 
+        request:  2
+        metadata: 3
+        [revisit: 4 only appears in .../crawldiagnostics/... files]
+      
+ part-mask is a bit-mask controling what part of each record is printed:
+   1: WARC headers
+   2: req/resp HTTP header, warcinfo/metadata features
+   3: resp body'''
+
+import iwarc,sys
+
+OUT=open(sys.stdout.fileno(),'wb')
+
+if (debug:=(sys.argv[1]=='-d')):
+  sys.argv.pop(1)
+
+tt=int(sys.argv.pop(1))
+
+def showme(wtype,buf,part):
+  # This should exactly reproduce a complete warc file if called
+  #  as per test no. 4 below, sub-parts one at a time if no. 1.
+  #  See comments
+  if debug:
+    OUT.write(b"----start %d-----\n"%part)
+  OUT.write(buf)
+  if buf and buf[-1]!=10:
+    OUT.write(b'\r\n')
+  if part==7:
+    OUT.write(b'\r\n') # to match complete file formatting
+  if debug:
+    OUT.write(b"----end %d-----\n"%part)
+  return OUT
+
+if tt==1:
+  # all parts
+  iwarc.warc(sys.argv[1],showme,[1,2,3,0],debug=debug)
+elif tt==2:
+  # just the initial warcinfo record, WARC (1), CC (2), both (3)
+  iwarc.warc(sys.argv[1],showme,[0],parts=int(sys.argv[2]),debug=debug)
+elif tt==3:
+  # just the initial warcinfo record
+  iwarc.warc(sys.argv[1],showme,[0],whole=True,debug=debug)
+elif tt==4:
+  # the whole thing
+  iwarc.warc(sys.argv[1],showme,[1,2,3,0,4],whole=True,debug=debug)
+elif tt==5:
+  # response records
+  iwarc.warc(sys.argv[1],showme,[1],parts=int(sys.argv[2]),debug=debug)
+elif tt==6:
+  # revisit records
+  iwarc.warc(sys.argv[1],showme,[4],parts=int(sys.argv[2]),debug=debug)
+elif tt==7:
+  # metadata records
+  iwarc.warc(sys.argv[1],showme,[3],parts=int(sys.argv[2]),debug=debug)
+elif tt==8:
+  # request records
+  iwarc.warc(sys.argv[1],showme,[2],parts=int(sys.argv[2]),debug=debug)
+