Mercurial > hg > cc > azure

comparison master/bin/wrun.sh @ 9:55e953e5c66f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
wrun.sh, invoke.sh: add control of use of nohup, with -i(mmediately) and -w(ait) respectively wrun.sh: add support for fixed file input via -ff, handle -ff - in obvious way wbash.sh: use -ff - and -i to execute shell commands on all workers
author Henry S. Thompson <ht@markup.co.uk>
date Tue, 02 Oct 2018 10:52:45 +0000
parents a7637c994964
children 2fbefb8d1a9e
comparison
equal deleted inserted replaced
8:5db6015689a2 9:55e953e5c66f
1 #!/bin/bash 1 #!/bin/bash
2 set -e -o pipefail 2 set -e -o pipefail
3 if [ $# -lt 2 ]; then 3 if [ $# -lt 2 ]; then
4 cat 1>&2 <<EOF 4 cat 1>&2 <<EOF
5 Usage: $0 SSname cmd [-x] [-np n] [-f file] [...args] 5 Usage: $0 SSname cmd [-x] [-np n] [-f[f] file] [-i] [...args]
6 Where name is the name of a VM scale set. 6 Where name is the name of a VM scale set.
7 7
8 Runs cmd on every machine in a scale set, 8 Runs cmd on every machine in a scale set,
9 passing args and, 9 passing args and,
10 if -f, one line from file per worker 10 if -f, lines from file split per worker
11 if -ff, complete file sent to all workers
11 unless -x, worker id 12 unless -x, worker id
12 by doing as it were 13 by doing as it were
13 [ echo line-from-file |] ssh machine "$cmd [id] "$args"" 14 [ echo line(s)-from-file |] ssh machine "$cmd [id] "$args""
15 if -i, don\'t use nohup on the workers so returns immediately
16 [default is to use nohup unless neither -f or -ff]
14 EOF 17 EOF
15 exit 1 18 exit 1
16 fi 19 fi
17 group=cc 20 group=cc
18 name=$1 21 name=$1
34 np=$(az vmss get-instance-view -g $group -n $name | jq '.virtualMachine.statusesSummary | .[0].count') 37 np=$(az vmss get-instance-view -g $group -n $name | jq '.virtualMachine.statusesSummary | .[0].count')
35 fi 38 fi
36 echo "|$name|$cmd|$@|$np|" 1>&2 39 echo "|$name|$cmd|$@|$np|" 1>&2
37 me=$(az vm list-ip-addresses -g cc -n Deb1 | jq -r '.[0].virtualMachine.network.publicIpAddresses|.[0].ipAddress') 40 me=$(az vm list-ip-addresses -g cc -n Deb1 | jq -r '.[0].virtualMachine.network.publicIpAddresses|.[0].ipAddress')
38 echo me=$me 1>&2 41 echo me=$me 1>&2
42 wait='-w'
39 if [ "$1" = "-f" ] 43 if [ "$1" = "-f" ]
40 then 44 then
41 file=$2 45 file=$2
42 shift 2 46 shift 2
43 tfile=$(mktemp) 47 tfile=$(mktemp)
44 split -n r/$np -u --filter="cat > $tfile\$\$" $file & 48 split -n r/$np -u --filter="cat > $tfile\$\$" $file &
45 # Wait for fifos to be built 49 # Wait for files to be built
46 while [ $np -gt $(ls ${tfile}* | wc -l) ]; do echo -n . 1>&2; sleep 1; done 50 while [ $np -gt $(ls ${tfile}* | wc -l) ]; do echo -n . 1>&2; sleep 1; done
47 echo 1>&2 51 echo 1>&2
48 wc -l ${tfile}* 1>&2 52 wc -l ${tfile}* 1>&2
53 elif [ "$1" = "-ff" ]
54 then
55 file=$2
56 shift 2
57 if [ "$file" = "-" ]
58 then
59 file=$(mktemp)
60 cat > $file
61 fi
49 else 62 else
50 file=/dev/null 63 file=/dev/null
64 wait=''
51 fi 65 fi
52 echo "|$np|$file|$tfile|" 1>&2 66 if [ "$1" = "-i" ]
67 then
68 shift
69 wait=''
70 fi
71 echo "|$np|$file|$tfile|$wait|" 1>&2
53 paste -d ' ' \ 72 paste -d ' ' \
54 <(if [ "$file" = "/dev/null" ] 73 <(if [ -z "$tfile" ]
55 then yes /dev/null|head -$np 74 then yes $file|head -$np
56 else 75 else
57 echo ${tfile}?* | tr ' ' '\012' # the name of the file of 76 echo ${tfile}?* | tr ' ' '\012' # the name of the file of
58 # input lines for each worker 77 # input lines for each worker
59 fi 78 fi
60 ) \ 79 ) \
62 tail -n +2 | head -$np |cut -f 3-5 | \ 81 tail -n +2 | head -$np |cut -f 3-5 | \
63 while read i ip port 82 while read i ip port
64 do 83 do
65 echo $i $port $ip 84 echo $i $port $ip
66 done) | \ 85 done) | \
67 parallel --gnu --colsep ' ' -j $np "$(dirname "$0")"/internal/invoke.sh $id $me "$cmd" {} "$@" || { r=$? ; echo parallel exited with status=$r 1>&2;} 86 parallel --gnu --colsep ' ' -j $np "$(dirname "$0")"/internal/invoke.sh $wait $id $me "$cmd" {} "$@" || { r=$? ; echo parallel exited with status=$r 1>&2;}
68 if [ "${tfile}" ]; then rm ${tfile}*; fi 87 if [ "${tfile}" ]; then rm ${tfile}*; fi