--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/w2c.sh	Thu Apr 23 10:51:36 2026 +0100
@@ -0,0 +1,29 @@
+#!/usr/bin/bash
+# Queue an array of N nTask-task w2c.py jobs, one segment per task
+# Usage: w2c.sh sI sJ N threadsPerTask nInputs time [warc|crawldiagnostics|robotstxt]
+# (sJ - sI) + 1, the number of segments, should be divisible by N, the number of jobs,
+#  to give the number of tasks per job
+export sI=$1
+sJ=$2
+nS=$(((sJ - sI) + 1))
+export N=$3
+nTasks=$((nS / N))
+ar="$sI-$sJ:$nTasks"
+export threadsPerTask=$4
+export nInputs=$5
+time=$6
+export ty=${7-warc}
+
+c=$(((nInputs * threadsPerTask) + 1))
+echo sbatch --output=slurm_${sI}-${sJ}x5-%j.out --time=$time --ntasks=$nTasks --array=$ar -c $c $HOME/bin/runme.sh -m $sI $sJ $PWD -d -t $threadsPerTask \
+ -b "export ty=$ty" -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh:$W/hst/lib/python
+ export CCC=/work/dc007/shared/pod12/common_crawl
+ j=$((i - 1)) ; tp=$((xarg + task))
+ mkdir -p ./x${j}/$tp/$ty
+ ~/lib/python/cc/w2c.py 2025-33 $tp ./x${j} $ty "$arg" 2> ./x${j}/$tp/$ty/w2c_errs'
+sbatch --output=slurm_${sI}-${sJ}x5-%j.out --time=$time --ntasks=$nTasks --array=$ar -c $c $HOME/bin/runme.sh -m $sI $sJ $PWD -d -t $threadsPerTask \
+ -b "export ty=$ty" -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh:$W/hst/lib/python
+ export CCC=/work/dc007/shared/pod12/common_crawl
+ j=$((i - 1)) ; tp=$((xarg + task))
+ mkdir -p ./x${j}/$tp/$ty
+ ~/lib/python/cc/w2c.py 2025-33 $tp ./x${j} $ty "$arg" 2> ./x${j}/$tp/$ty/w2c_errs'