changeset 318:2502b81838da trim

for tracking progress (of w2c?)
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 13 Jan 2026 15:22:28 +0000
parents 9450b557ccac
children 439cbfd6cbda
files bin/wprog.py bin/wprog.sh
diffstat 2 files changed, 36 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/wprog.py	Tue Jan 13 15:22:28 2026 +0000
@@ -0,0 +1,9 @@
+#!/usr/bin/python3
+import sys, datetime
+(s,d1,d2) = sys.argv[1:4]
+dd2 = datetime.datetime.strptime(d2[4:22],"%b %d %I:%M:%S %p")
+dd1 = datetime.datetime.strptime(d1[4:18],"%b %d %H:%M:%S")
+d = (dd2 - dd1).seconds
+h = d / 3600
+print("%.2f %d:%0.2d:%0.2d %.2f"%(a := float(s),
+	h, (m := (d % 3600)) / 60, m % 60, a / h))
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/wprog.sh	Tue Jan 13 15:22:28 2026 +0000
@@ -0,0 +1,27 @@
+#!/usr/bin/bash
+# Usage wprog.sh seg start end=$(date) ty='warc' 
+seg="$1"
+shift
+start="$1"
+shift
+end=$(date)
+ty='warc'
+while [ "$1" ]
+do
+ case "$1" in
+  [A-Z]*)
+    end="$1"
+    shift
+    ;;
+   warc|robotstxt|crawldiagnostics)
+    ty=$1
+    shift
+    ;;
+   *)
+    echo "Hunh?" "$@" 1>&2
+    echo "Usage: wprog.sh start end=$(date) ty='warc'" 1>&2
+    exit 1
+    ;;
+ esac
+done
+wprog.py $(fgrep -ch records x?/$seg/$ty/w* | btot -s | cut -f 2) "$start" "$end"