--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/hist	Sat Feb 28 15:52:47 2026 +0000
@@ -0,0 +1,88 @@
+#!/bin/sh
+# Usage: hist file bins [-max n] [-min n] [-ll n] [-t title] [-ntile n]
+file=$1
+shift
+nb=$1
+shift
+date=`date`
+while
+  case "x$1" in
+   x-max) shift; max=$1 ;;
+   x-ll) shift; ll=$1 ;;
+   x-min) shift; min=$1 ;;
+   x-t) shift; title=$1 ;;
+   x-ntile) shift; ntile=$1 ;;
+   x) break ;;
+   *) echo hunh 1>&2; exit 1 ;;
+  esac
+ do
+  shift
+done
+if [ \( -z "$max" -o -z "$min" \) ]
+ then
+  rfile=/tmp/hist$$
+  sort -n $file > $rfile
+fi
+perl -e "
+(\$max,\$min,\$nb,\$ll,\$title,\$ntile)=(${max:-`tail -1 /tmp/hist$$`},
+                    ${min:-`head -1 /tmp/hist$$`}
+                   ,$nb,${ll:-80},\"${title:-$file}\",${ntile:-0});"'
+$#bins=$nb;
+$range=$max-$min;
+while (<>) {
+  if ($_<$min) {
+    $minn+=1;
+  }
+  elsif ($_>$max) {
+    $maxn+=1;
+  }
+  else {
+    $n+=1;
+    $i=(($_-$min)/$range)*$nb;
+    if ($i==$nb) {$bins[$i-1]+=1} else {$bins[int($i)]+=1};
+  };
+};
+$start=$min+$range/(2*$nb);
+$step=$range/$nb;
+foreach $i (0..$nb-1) {
+  $mb=$bins[$i] if ($bins[$i]>$mb);
+};
+$bscale=$mb/($ll-16);
+$bscale=1 if ($bscale<1);
+print "$title ",`date`,"\n";
+print "    n     min    max    width  maxcnt bscale";
+if ($ntile) {
+  print "  ntiles  tilew";
+  $tile=$tstep=($n/$ntile);
+  $ntile--;
+}
+print "\n";
+printf("%7d%7.3f %7.3f%7.3f%6d  %7.3f",$n,$min,$max,$step,$mb,$bscale);
+printf("%5d   %7.2f",$ntile+1,$tstep) if ($ntile);
+print "\n\n";
+if ($minn) {printf("<%7.3f%7d\n",$min,$minn)};
+foreach $i (0..$nb-1) {
+  $n=$bins[$i];
+  if ($ntile) {
+    $cum+=$n;
+    if ($cum>=$tile) {
+      print "-"; $tile+=$tstep; $ntile--;
+    }
+    else {
+      print " ";
+    };
+  }
+  else {
+    print " ";
+  };
+  printf("%7.3f%7d ",$start+($i*$step),$bins[$i]);
+  $nx=$bins[$i]/$bscale;
+  $nx=1 if (($nx<1) && ($bins[$i]>0));
+  print "*" x $nx; print "\n";
+};
+if ($maxn) {printf(">%7.3f%7d\n",$max,$maxn)};' ${rfile:-$file}
+if [ "$rfile" ]
+ then
+  rm $rfile
+fi
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/ix.py	Sat Feb 28 15:52:47 2026 +0000
@@ -0,0 +1,1 @@
+/work/dc007/dc007/hst/lib/python/cc/ix.py
\ No newline at end of file

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/qpdf_check	Sat Feb 28 15:52:47 2026 +0000
@@ -0,0 +1,2 @@
+#!/bin/bash
+qpdf --warning-exit-0 --suppress-recovery --check "$@" 2> >(egrep -v '^WARNING:' 1>&2) >/dev/null

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/sing	Sat Feb 28 15:52:47 2026 +0000
@@ -0,0 +1,7 @@
+#!/bin/bash
+# For running in HST's environment for common crawl projects
+if [ ! -x /usr/bin/singularity ]
+then
+ module load singularity
+fi
+singularity run -B /work/dc007 /work/dc007/dc007/shared/cc "$@"

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/uniq_merge.py	Sat Feb 28 15:52:47 2026 +0000
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# Merge counts by key from the output of "uniq -c" (or sus) and sort in descending order
+# An alternative to sus when the scale is too big for the initial sort, or if uniq -c already does a lot
+#  of the work
+# Usage: ... | uniq -c | uniq-merge.py [-c]
+# If -c, remove commas from count field
+import sys
+from collections import defaultdict
+s=defaultdict(int)
+if len(sys.argv)==2 and sys.argv[1]=='-c':
+    for l in sys.stdin:
+        try:
+            (i,d)=l.split(maxsplit=1)
+        except ValueError:
+            sys.stderr.write("bogus input: %s"%l)
+            continue
+        s[d]+=int(i.replace(',',''))
+else:
+    for l in sys.stdin:
+        try:
+            (i,d)=l.split(maxsplit=1)
+        except ValueError:
+            sys.stderr.write("bogus input: %s"%l)
+            continue
+        s[d]+=int(i)
+ss=sorted(s.items(),key=lambda j:j[1],reverse=True)
+fmt='%'+str(len(str(ss[0][1]))+1)+'d\t%s'
+for (d,n) in ss:
+ sys.stdout.write(fmt%(n,d))