Mercurial > hg > cc > cirrus_home

comparison bin/w2c.sh @ 204:dc397748f4e1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
rework w2c.py args
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Thu, 23 Apr 2026 10:51:36 +0100
parents
children
comparison
equal deleted inserted replaced
203:2aa9db279d56 204:dc397748f4e1
1 #!/usr/bin/bash
2 # Queue an array of N nTask-task w2c.py jobs, one segment per task
3 # Usage: w2c.sh sI sJ N threadsPerTask nInputs time [warc|crawldiagnostics|robotstxt]
4 # (sJ - sI) + 1, the number of segments, should be divisible by N, the number of jobs,
5 # to give the number of tasks per job
6 export sI=$1
7 sJ=$2
8 nS=$(((sJ - sI) + 1))
9 export N=$3
10 nTasks=$((nS / N))
11 ar="$sI-$sJ:$nTasks"
12 export threadsPerTask=$4
13 export nInputs=$5
14 time=$6
15 export ty=${7-warc}
16
17 c=$(((nInputs * threadsPerTask) + 1))
18 echo sbatch --output=slurm_${sI}-${sJ}x5-%j.out --time=$time --ntasks=$nTasks --array=$ar -c $c $HOME/bin/runme.sh -m $sI $sJ $PWD -d -t $threadsPerTask \
19 -b "export ty=$ty" -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh:$W/hst/lib/python
20 export CCC=/work/dc007/shared/pod12/common_crawl
21 j=$((i - 1)) ; tp=$((xarg + task))
22 mkdir -p ./x${j}/$tp/$ty
23 ~/lib/python/cc/w2c.py 2025-33 $tp ./x${j} $ty "$arg" 2> ./x${j}/$tp/$ty/w2c_errs'
24 sbatch --output=slurm_${sI}-${sJ}x5-%j.out --time=$time --ntasks=$nTasks --array=$ar -c $c $HOME/bin/runme.sh -m $sI $sJ $PWD -d -t $threadsPerTask \
25 -b "export ty=$ty" -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh:$W/hst/lib/python
26 export CCC=/work/dc007/shared/pod12/common_crawl
27 j=$((i - 1)) ; tp=$((xarg + task))
28 mkdir -p ./x${j}/$tp/$ty
29 ~/lib/python/cc/w2c.py 2025-33 $tp ./x${j} $ty "$arg" 2> ./x${j}/$tp/$ty/w2c_errs'