Mercurial > hg > cc > cirrus_work

annotate bin/_runme_c.sh @ 247:7737da0ccb8c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
try adding lm to existing index from ks_0-9
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Thu, 02 Jan 2025 14:52:14 +0000
parents 4c65ae2a4bc3
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
99
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/bin/bash
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 # This runs on the compute nodes...
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 # Args: [-a i] wd [-d] [-t nthreads] [-b CMDS] [-i input] CMDS
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 # See ug4/azure/notes.txt for documentation
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 N=$SLURM_JOB_NUM_NODES
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 n=$SLURM_NTASKS
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 c=$SLURM_CPUS_PER_TASK
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 nodename=$SLURMD_NODENAME
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10 local=$SLURM_LOCALID
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 node=$SLURM_NODEID
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 task=$SLURM_PROCID
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14 if [ "$1" = "-a" ]
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15 then
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17 xarg="$1"
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19 else
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
20 xarg=
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21 fi
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23 cd "$1"
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26 if [ "$1" = "-d" ]
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 then
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
29 debug=1
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
30 fi
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
31
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
32 if [ "$1" = "-t" ]
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
33 then
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
34 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
35 threadsPerTask=$1
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
36 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
37 else
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
38 threadsPerTask=2
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
39 fi
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
40
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
41 pjobs=$((c / $threadsPerTask))
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
42
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
43 echo $(date) task $n.$task on $nodename:$N.$node start $pjobs jobs 1>&2
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
44
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
45 PATH=$W/$USER/bin:$W/shared/bin:$PATH
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
46 export task PATH n xarg
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
47
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
48 if [ "$1" = "-b" ]
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
49 then
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
50 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
51 eval "$1"
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
52 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
53 fi
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
54
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
55 input="seq 1 $pjobs" # default to make sure something runs
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
56
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
57 if [ "$1" = "-i" ]
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
58 then
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
59 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
60 input="$1"
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
61 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
62 if [ "$debug" ]
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
63 then
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
64 echo $(date) task $n.$task input "|$input|"
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
65 fi
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
66 fi
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
67
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
68 export cmd="$1"
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
69 shift
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
70
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
71 doit () {
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
72 arg="$1"
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
73 echo $(date) start $task $PARALLEL_SEQ "|$cmd|$xarg|$arg|"
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
74 eval "$cmd"
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
75 echo $(date) end $task $PARALLEL_SEQ
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
76 }
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
77
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
78 export -f doit
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
79
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
80 if [ "$debug" ]
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
81 then
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
82 echo $(date) task $n.$task cmd "|$cmd|" doit $(type doit)
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
83 fi
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
84
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
85 source $HOME/.csing_init
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
86
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
87 eval "$input" | \
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
88 if [ "$debug" ]
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
89 then
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
90 tee >(cat 1>&2)
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
91 else
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
92 cat
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
93 fi | \
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
94 if [ $pjobs -le 1 ]
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
95 then
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
96 xargs -I ^ -n 1 bash -lc "doit ^"
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
97 else
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
98 parallel --line-buffer -j $pjobs doit '{}'
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
99 fi
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
100
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
101 echo $(date) task $n.$task on $nodename:$N.$node end 1>&2
4c65ae2a4bc3 initialise paths for csing
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
102