Mercurial > hg > cc > cirrus_work

comparison bin/_runme.sh @ 40:316495371bbc

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add lots more debugging output, reinstate -t for thread allocation
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 05 Jul 2023 15:12:07 +0100
parents e3c440666f1a
children 235004978b22
comparison
equal deleted inserted replaced
39:8661062a50b1 40:316495371bbc
1 #!/bin/bash 1 #!/bin/bash
2 # This runs on the compute nodes... 2 # This runs on the compute nodes...
3 # Args: wd [-b CMDS] [-i input] CMDS 3 # Args: wd [-d] [-t nthreads] [-b CMDS] [-i input] CMDS
4 # See ug4/azure/notes.txt for documentation 4 # See ug4/azure/notes.txt for documentation
5 5
6 N=$SLURM_JOB_NUM_NODES 6 N=$SLURM_JOB_NUM_NODES
7 n=$SLURM_NTASKS 7 n=$SLURM_NTASKS
8 c=$SLURM_CPUS_PER_TASK 8 c=$SLURM_CPUS_PER_TASK
12 task=$SLURM_PROCID 12 task=$SLURM_PROCID
13 13
14 cd "$1" 14 cd "$1"
15 shift 15 shift
16 16
17 threadsPerTask=2 17 if [ "$1" = "-d" ]
18 then
19 shift
20 debug=1
21 fi
22
23 if [ "$1" = "-t" ]
24 then
25 shift
26 threadsPerTask=$1
27 shift
28 else
29 threadsPerTask=2
30 fi
31
18 pjobs=$((c / $threadsPerTask)) 32 pjobs=$((c / $threadsPerTask))
19 33
20 echo $(date) task $n.$task on $nodename:$N.$node start $(pwd) 1>&2 34 echo $(date) task $n.$task on $nodename:$N.$node start $pjobs jobs 1>&2
21 35
22 PATH=$W/$USER/bin:$W/shared/bin:$PATH 36 PATH=$W/$USER/bin:$W/shared/bin:$PATH
23 export task PATH n 37 export task PATH n
24 38
25 if [ "$1" = "-b" ] 39 if [ "$1" = "-b" ]
32 if [ "$1" = "-i" ] 46 if [ "$1" = "-i" ]
33 then 47 then
34 shift 48 shift
35 input="$1" 49 input="$1"
36 shift 50 shift
51 if [ "$debug" ]
52 then
53 echo $(date) task $n.$task input "|$input|"
54 fi
37 fi 55 fi
38 56
39 export cmd="$1" 57 export cmd="$1"
40 shift 58 shift
41 59
42 doit () { 60 doit () {
43 arg="$1" 61 arg="$1"
44 echo $(date) start $task $PARALLEL_SEQ $arg 62 echo $(date) start $task $PARALLEL_SEQ "|$cmd|$arg|"
45 eval "$cmd" 63 eval "$cmd"
46 echo $(date) end $task $PARALLEL_SEQ 64 echo $(date) end $task $PARALLEL_SEQ
47 } 65 }
48 66
49 export -f doit 67 export -f doit
50 68
69 if [ "$debug" ]
70 then
71 echo $(date) task $n.$task cmd "|$cmd|" doit $(type doit)
72 fi
73
51 eval "$input" | \ 74 eval "$input" | \
52 parallel -j $pjobs doit '{}' 75 if [ "$debug" ]
76 then
77 tee >(cat 1>&2)
78 else
79 cat
80 fi | \
81 if [ $pjobs -le 1 ]
82 then
83 xargs -I ^ -n 1 bash -c "doit ^"
84 else
85 parallel --line-buffer -j $pjobs doit '{}'
86 fi
53 87
54 echo $(date) task $n.$task on $nodename:$N.$node end 1>&2 88 echo $(date) task $n.$task on $nodename:$N.$node end 1>&2
55 89