#!/usr/bin/bash
# Queue an array of N nTask-task w2c.py jobs, one segment per task
# Usage: w2c.sh sI sJ N threadsPerTask nInputs time [warc|crawldiagnostics|robotstxt]
# (sJ - sI) + 1, the number of segments, should be divisible by N, the number of jobs,
#  to give the number of tasks per job
export sI=$1
sJ=$2
nS=$(((sJ - sI) + 1))
export N=$3
nTasks=$((nS / N))
ar="$sI-$sJ:$nTasks"
export threadsPerTask=$4
export nInputs=$5
time=$6
export ty=${7-warc}

c=$(((nInputs * threadsPerTask) + 1))
echo sbatch --output=slurm_${sI}-${sJ}x5-%j.out --time=$time --ntasks=$nTasks --array=$ar -c $c $HOME/bin/runme.sh -m $sI $sJ $PWD -d -t $threadsPerTask \
 -b "export ty=$ty" -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh:$W/hst/lib/python
 export CCC=/work/dc007/shared/pod12/common_crawl
 j=$((i - 1)) ; tp=$((xarg + task))
 mkdir -p ./x${j}/$tp/$ty
 ~/lib/python/cc/w2c.py 2025-33 $tp ./x${j} $ty "$arg" 2> ./x${j}/$tp/$ty/w2c_errs'
sbatch --output=slurm_${sI}-${sJ}x5-%j.out --time=$time --ntasks=$nTasks --array=$ar -c $c $HOME/bin/runme.sh -m $sI $sJ $PWD -d -t $threadsPerTask \
 -b "export ty=$ty" -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh:$W/hst/lib/python
 export CCC=/work/dc007/shared/pod12/common_crawl
 j=$((i - 1)) ; tp=$((xarg + task))
 mkdir -p ./x${j}/$tp/$ty
 ~/lib/python/cc/w2c.py 2025-33 $tp ./x${j} $ty "$arg" 2> ./x${j}/$tp/$ty/w2c_errs'