Mercurial > hg > cc > cirrus_home

annotate bin/w2c.sh @ 204:dc397748f4e1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
rework w2c.py args
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Thu, 23 Apr 2026 10:51:36 +0100
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
204
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/usr/bin/bash
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 # Queue an array of N nTask-task w2c.py jobs, one segment per task
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 # Usage: w2c.sh sI sJ N threadsPerTask nInputs time [warc|crawldiagnostics|robotstxt]
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 # (sJ - sI) + 1, the number of segments, should be divisible by N, the number of jobs,
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 # to give the number of tasks per job
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 export sI=$1
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 sJ=$2
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 nS=$(((sJ - sI) + 1))
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 export N=$3
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10 nTasks=$((nS / N))
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 ar="$sI-$sJ:$nTasks"
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 export threadsPerTask=$4
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13 export nInputs=$5
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14 time=$6
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15 export ty=${7-warc}
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17 c=$(((nInputs * threadsPerTask) + 1))
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18 echo sbatch --output=slurm_${sI}-${sJ}x5-%j.out --time=$time --ntasks=$nTasks --array=$ar -c $c $HOME/bin/runme.sh -m $sI $sJ $PWD -d -t $threadsPerTask \
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19 -b "export ty=$ty" -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh:$W/hst/lib/python
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
20 export CCC=/work/dc007/shared/pod12/common_crawl
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21 j=$((i - 1)) ; tp=$((xarg + task))
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22 mkdir -p ./x${j}/$tp/$ty
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23 ~/lib/python/cc/w2c.py 2025-33 $tp ./x${j} $ty "$arg" 2> ./x${j}/$tp/$ty/w2c_errs'
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 sbatch --output=slurm_${sI}-${sJ}x5-%j.out --time=$time --ntasks=$nTasks --array=$ar -c $c $HOME/bin/runme.sh -m $sI $sJ $PWD -d -t $threadsPerTask \
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25 -b "export ty=$ty" -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh:$W/hst/lib/python
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26 export CCC=/work/dc007/shared/pod12/common_crawl
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 j=$((i - 1)) ; tp=$((xarg + task))
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28 mkdir -p ./x${j}/$tp/$ty
dc397748f4e1 rework w2c.py args
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
29 ~/lib/python/cc/w2c.py 2025-33 $tp ./x${j} $ty "$arg" 2> ./x${j}/$tp/$ty/w2c_errs'