Mercurial > hg > cc > cirrus_work

comparison bin/_s2t.sh @ 2:b4801f5696b2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
compute node workers, see cirrus_home/bin repo for login node masters
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 18 Jul 2022 19:22:42 +0100
parents
children f035d36cec45
comparison
equal deleted inserted replaced
1:d5b6748f29a9 2:b4801f5696b2
1 #!/bin/bash
2 # run cdx2tsv.py in parallel, taking input directly from cdx-00{000..299}.gz
3
4 N=$SLURM_JOB_NUM_NODES
5 n=$SLURM_NTASKS
6 c=$SLURM_CPUS_PER_TASK
7 nodename=$SLURMD_NODENAME
8 local=$SLURM_LOCALID
9 node=$SLURM_NODEID
10
11 tPerN=$((n / N))
12
13 task=$((local + (node * tPerN)))
14
15 threadsPerTask=2
16 pjobs=$((c / $threadsPerTask))
17
18
19 cc=$1
20 resdir=$W/hst/results/$1/$2
21 s1=$3
22 sn=$4
23
24 echo $(date) task $n.$task on $nodename:$N.$node start
25
26 mkdir -p $resdir
27 mkdir -p /dev/shm/hst
28
29 doit () { echo $(date) start $1 $task $PARALLEL_SEQ; uz /beegfs/common_crawl/$cc/cdx/warc/cdx-00$1.gz | python3 $WSHARED/bin/cdx2tsv.py '(filename,f.split("/",maxsplit=5)[3].split(".")[1])' '(filename,f.split("/",maxsplit=5)[4][0])' '(url,("1" if (f.split(":",maxsplit=1)[0]=="https") else "0"))' languages > /dev/shm/hst/$1.tsv; echo $(date) moving $1 $task $PARALLEL_SEQ ; mv /dev/shm/hst/$1.tsv $resdir; echo $(date) end $1 $task $PARALLEL_SEQ ;}
30
31 export -f doit
32 export cc resdir n task
33
34 seq $s1 $sn | while read i; do if [ $((i % $n)) -eq $task ]; then printf '%03g\n' $i; fi; done | \
35 parallel -j $pjobs doit '{}'
36
37 echo $(date) task $n.$task on $nodename:$N.$node end
38