# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1782834656 -3600
# Node ID 10813bab9072a4583b51b0eed238e2339009db27
# Parent  54304add8adf8ca902d1070eec0da874df87f76d
added test 9 for wat that pretty-prints json content

diff -r 54304add8adf -r 10813bab9072 lib/python/cc/test_hwarc.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/python/cc/test_hwarc.py	Tue Jun 30 16:50:56 2026 +0100
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# cython: profile=False, language_level=3str
+'''Test various parameter settings for complete warc file streaming callback.
+   
+
+Usage: test_iwarc.py [-d] test-number filename warc-types part-mask 
+ -d turns on debugging output
+ filename gives a path to a .warc or .warc.gz WARC file
+ warc-types is list of integers corresponding to WARC record types
+    to be extracted and printed to stdout:
+        warcinfo: 0
+        response: 1 
+        request:  2
+        metadata: 3
+        [revisit: 4 only appears in .../crawldiagnostics/... files]
+      
+ part-mask is a bit-mask controling what part of each record is printed:
+   1: WARC headers
+   2: req/resp HTTP header, warcinfo/metadata features
+   3: resp body'''
+
+import iwarc,sys
+
+global JSON
+JSON = False
+
+OUT=open(sys.stdout.fileno(),'wb')
+
+if (debug:=(sys.argv[1]=='-d')):
+  sys.argv.pop(1)
+
+tt=int(sys.argv.pop(1))
+
+def showme(wtype,buf,part):
+  # This should exactly reproduce a complete warc file if called
+  #  as per test no. 4 below, sub-parts one at a time if no. 1.
+  #  See comments
+  global JSON, debug
+  if debug:
+    OUT.write(b"----start %d-----\n"%part)
+  if JSON:
+    OUT.write(json.dumps(json.loads(bytearray(buf)), sort_keys = True, indent = 2).encode())
+  else:
+    OUT.write(buf)
+  if buf and buf[-1]!=10:
+    OUT.write(b'\r\n')
+  if part==7:
+    OUT.write(b'\r\n') # to match complete file formatting
+  if debug:
+    OUT.write(b"----end %d-----\n"%part)
+  return OUT
+
+if tt==1:
+  # all parts
+  iwarc.warc(sys.argv[1],showme,[1,2,3,0],debug=debug)
+elif tt==2:
+  # just the initial warcinfo record, WARC (1), CC (2), both (3)
+  iwarc.warc(sys.argv[1],showme,[0],parts=int(sys.argv[2]),debug=debug)
+elif tt==3:
+  # just the initial warcinfo record
+  iwarc.warc(sys.argv[1],showme,[0],whole=True,debug=debug)
+elif tt==4:
+  # the whole thing
+  iwarc.warc(sys.argv[1],showme,[1,2,3,0,4],whole=True,debug=debug)
+elif tt==5:
+  # response records
+  iwarc.warc(sys.argv[1],showme,[1],parts=int(sys.argv[2]),debug=debug)
+elif tt==6:
+  # revisit records
+  iwarc.warc(sys.argv[1],showme,[4],parts=int(sys.argv[2]),debug=debug)
+elif tt==7:
+  # metadata records
+  iwarc.warc(sys.argv[1],showme,[3],parts=int(sys.argv[2]),debug=debug)
+elif tt==8:
+  # request records
+  iwarc.warc(sys.argv[1],showme,[2],parts=int(sys.argv[2]),debug=debug)
+elif tt==9:
+  # wat metadata
+  JSON = True
+  import json
+  iwarc.warc(sys.argv[1],showme,[3],parts=2,debug=debug)
+
+