Mercurial > hg > cc > cirrus_home

annotate bin/sql2tsv.sh @ 178:e1bc9d8d688c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
ec184 now, run w. unbuffered output
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 05 Jul 2023 15:02:53 +0100
parents 2643a6825f17
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
154
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/bin/bash
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 # Invoke this as e.g. sbatch -N 4 --ntasks=3 -c 5 masterJob.sh sql2tsv \
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 # CC-MAIN-2019-35 cdx_db 0-299
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 # run sql2tsv.py in parallel, taking input directly from .../0-299.n.c.tar.gz
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 n=$SLURM_NTASKS
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 c=$SLURM_CPUS_PER_TASK
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 node=$SLURMD_NODENAME
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 task=$SLURM_LOCALID
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 node=$SLURM_NODEID
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 cc=$1
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 resdir=$2
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13 segs=$3
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15 echo $(date) $nodename:$node:$task start
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17 export PYTHONPATH=$PYTHONPATH:$HOME/lib/python
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19 ld=/dev/shm/ht/$task
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
20 mkdir -p $ld
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22 cd $ld
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23 tar --wildcards -xf $HOME/results/$cc/$resdir/$segs.$node.$task.tar.gz '*.db'
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 cd $HOME
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26 ls $ld/*.db | \
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 parallel --will-cite -j $c doS2T.sh $node $task $cc $resdir $segs $ld '{}'
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
29 if [ "$ld" ]
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
30 then
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
31 cd $ld
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
32 cp *.tsv $HOME/results/$cc/$resdir
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
33 rm *
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
34 fi
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
35
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
36 echo $(date) $nodename:$node:$task end
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
37
2643a6825f17 instead of csv
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
38