Mercurial > hg > cc > cirrus_work

annotate bin/_runme.sh @ 370:c5b4f3e9f909 plus

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
-a working now?
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 20 Apr 2026 18:18:11 +0100
parents bd931adc820b
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/bin/bash
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 # This runs on the compute nodes...
68
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
3 # Args: [-a i] wd [-d] [-t nthreads] [-b CMDS] [-i input] CMDS
38
e3c440666f1a doc pointer
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 2
diff changeset
4 # See ug4/azure/notes.txt for documentation
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 N=$SLURM_JOB_NUM_NODES
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 n=$SLURM_NTASKS
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 c=$SLURM_CPUS_PER_TASK
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 nodename=$SLURMD_NODENAME
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10 local=$SLURM_LOCALID
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 node=$SLURM_NODEID
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 task=$SLURM_PROCID
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13
364
bd931adc820b working on new shrunk cdb en- and de-coding
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 363
diff changeset
14 # Not working
68
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
15 if [ "$1" = "-a" ]
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
16 then
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
17 shift
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
18 xarg="$1"
364
bd931adc820b working on new shrunk cdb en- and de-coding
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 363
diff changeset
19 xa="job $1 "
68
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
20 shift
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
21 else
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
22 xarg=
294
cc2945816b75 job index arg to doit, slightly better diagnostic output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 252
diff changeset
23 xa=
68
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
24 fi
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
25
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26 cd "$1"
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 shift
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28
40
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
29 if [ "$1" = "-d" ]
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
30 then
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
31 shift
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
32 debug=1
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
33 fi
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
34
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
35 if [ "$1" = "-t" ]
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
36 then
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
37 shift
294
cc2945816b75 job index arg to doit, slightly better diagnostic output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 252
diff changeset
38 threadsPerJob=$1
40
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
39 shift
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
40 else
294
cc2945816b75 job index arg to doit, slightly better diagnostic output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 252
diff changeset
41 threadsPerJob=2
40
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
42 fi
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
43
294
cc2945816b75 job index arg to doit, slightly better diagnostic output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 252
diff changeset
44 pjobs=$((c / $threadsPerJob))
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
45
364
bd931adc820b working on new shrunk cdb en- and de-coding
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 363
diff changeset
46 echo $(date) ${xa}task $n.$task on $nodename:$N.$node start $pjobs 1>&2
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
47
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
48 PATH=$W/$USER/bin:$W/shared/bin:$PATH
370
c5b4f3e9f909 -a working now?
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 364
diff changeset
49 export task PATH n node xarg
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
50
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
51 if [ "$1" = "-b" ]
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
52 then
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
53 shift
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
54 eval "$1"
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
55 shift
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
56 fi
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
57
68
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
58 input="seq 1 $pjobs" # default to make sure something runs
235004978b22 add support for multiple calls to srun with a counter
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 40
diff changeset
59
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
60 if [ "$1" = "-i" ]
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
61 then
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
62 shift
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
63 input="$1"
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
64 shift
40
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
65 if [ "$debug" ]
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
66 then
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
67 echo $(date) task $n.$task input "|$input|"
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
68 fi
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
69 fi
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
70
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
71 export cmd="$1"
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
72 shift
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
73
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
74 doit () {
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
75 arg="$1"
294
cc2945816b75 job index arg to doit, slightly better diagnostic output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 252
diff changeset
76 i="$2"
370
c5b4f3e9f909 -a working now?
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 364
diff changeset
77 echo $(date) start $task $i "|$node|$arg|$xarg|"
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
78 eval "$cmd"
294
cc2945816b75 job index arg to doit, slightly better diagnostic output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 252
diff changeset
79 echo $(date) end $task $i
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
80 }
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
81
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
82 export -f doit
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
83
40
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
84 if [ "$debug" ]
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
85 then
294
cc2945816b75 job index arg to doit, slightly better diagnostic output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 252
diff changeset
86 echo $(date) task $n.$task cmd "|$cmd|" $(type doit)
40
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
87 fi
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
88
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
89 eval "$input" | \
40
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
90 if [ "$debug" ]
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
91 then
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
92 tee >(cat 1>&2)
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
93 else
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
94 cat
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
95 fi | \
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
96 if [ $pjobs -le 1 ]
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
97 then
307
c2b93789b292 replace xargs with an explicit serial loop plus wait
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 295
diff changeset
98 while read inp
c2b93789b292 replace xargs with an explicit serial loop plus wait
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 295
diff changeset
99 do
c2b93789b292 replace xargs with an explicit serial loop plus wait
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 295
diff changeset
100 doit "$inp"
c2b93789b292 replace xargs with an explicit serial loop plus wait
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 295
diff changeset
101 done
c2b93789b292 replace xargs with an explicit serial loop plus wait
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 295
diff changeset
102 if [ "$debug" ]
c2b93789b292 replace xargs with an explicit serial loop plus wait
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 295
diff changeset
103 then
c2b93789b292 replace xargs with an explicit serial loop plus wait
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 295
diff changeset
104 ps --ppid $$
c2b93789b292 replace xargs with an explicit serial loop plus wait
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 295
diff changeset
105 fi
c2b93789b292 replace xargs with an explicit serial loop plus wait
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 295
diff changeset
106 wait
40
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
107 else
294
cc2945816b75 job index arg to doit, slightly better diagnostic output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 252
diff changeset
108 parallel --line-buffer -j $pjobs doit '{}' '{#}'
40
316495371bbc add lots more debugging output,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 38
diff changeset
109 fi
2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
110
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
111 echo $(date) task $n.$task on $nodename:$N.$node end 1>&2
b4801f5696b2 compute node workers, see cirrus_home/bin repo for login node masters
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
112