Mercurial > hg > cc > cirrus_home

annotate bin/cdx2sql.sh @ 195:5f3c36e4fd6d default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add target test-core which (dangerously) avoids (we hope pointless) recompilation of all the plugins
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Thu, 26 Sep 2024 17:55:56 +0100
parents afd7879181c9
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
166
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/bin/bash
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 # Invoke this as e.g. sbatch -N 4 --ntasks=3 -c 5 masterJob.sh cdx2sql \
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 # CC-MAIN-2019-35 cdx_db
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 # run cdx2sql.py in parallel, working index files enumerated in cdx2sql/n.c.txt
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 n=$SLURM_NTASKS
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 c=$SLURM_CPUS_PER_TASK
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 node=$SLURMD_NODENAME
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 task=$SLURM_LOCALID
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 node=$SLURM_NODEID
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 cc=$1
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 resdir=$2
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14 echo $(date) $nodename:$node:$task start
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16 export PYTHONPATH=$PYTHONPATH:$HOME/lib/python
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18 ld=/dev/shm/ht/$task
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19 mkdir -p $ld
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
20
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21 parallel --will-cite -j $c doC2S.sh $node $task $cc $resdir $ld '{}' < cdx2sql/$node.$task.txt
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23 if [ "$ld" ]
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 then
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25 cd $ld
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26 tar -czf $HOME/results/$cc/$resdir/$node.$task.tar.gz *
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 rm *
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28 fi
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
29
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
30 echo $(date) $nodename:$node:$task end
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
31
afd7879181c9 old style
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
32