Mercurial > hg > cc > azure

annotate master/bin/wrun.sh @ 7:a7637c994964

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh: From kenneth, slight mods in some cases share.sh: Distribute files to workers wrun.sh, invoke.sh: From last year, slight mods
author Henry S. Thompson <ht@markup.co.uk>
date Sun, 30 Sep 2018 20:53:43 +0000
parents
children 55e953e5c66f
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
1 #!/bin/bash
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
2 set -e -o pipefail
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
3 if [ $# -lt 2 ]; then
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
4 cat 1>&2 <<EOF
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
5 Usage: $0 SSname cmd [-x] [-np n] [-f file] [...args]
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
6 Where name is the name of a VM scale set.
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
8 Runs cmd on every machine in a scale set,
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
9 passing args and,
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
10 if -f, one line from file per worker
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
11 unless -x, worker id
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
12 by doing as it were
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
13 [ echo line-from-file |] ssh machine "$cmd [id] "$args""
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
14 EOF
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
15 exit 1
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
16 fi
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
17 group=cc
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
18 name=$1
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
19 cmd="$2"
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
20 shift 2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
21 if [ "$1" = "-x" ]
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
22 then
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
23 id=-x
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
24 shift
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
25 else
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
26 id=
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
27 fi
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
28 if [ "$1" = "-np" ]
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
29 then
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
30 np=$2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
31 shift
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
32 shift
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
33 else
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
34 np=$(az vmss get-instance-view -g $group -n $name | jq '.virtualMachine.statusesSummary | .[0].count')
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
35 fi
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
36 echo "|$name|$cmd|$@|$np|" 1>&2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
37 me=$(az vm list-ip-addresses -g cc -n Deb1 | jq -r '.[0].virtualMachine.network.publicIpAddresses|.[0].ipAddress')
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
38 echo me=$me 1>&2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
39 if [ "$1" = "-f" ]
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
40 then
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
41 file=$2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
42 shift 2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
43 tfile=$(mktemp)
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
44 split -n r/$np -u --filter="cat > $tfile\$\$" $file &
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
45 # Wait for fifos to be built
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
46 while [ $np -gt $(ls ${tfile}* | wc -l) ]; do echo -n . 1>&2; sleep 1; done
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
47 echo 1>&2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
48 wc -l ${tfile}* 1>&2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
49 else
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
50 file=/dev/null
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
51 fi
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
52 echo "|$np|$file|$tfile|" 1>&2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
53 paste -d ' ' \
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
54 <(if [ "$file" = "/dev/null" ]
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
55 then yes /dev/null|head -$np
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
56 else
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
57 echo ${tfile}?* | tr ' ' '\012' # the name of the file of
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
58 # input lines for each worker
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
59 fi
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
60 ) \
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
61 <(az vmss list-instance-connection-info -g $group -n $name | tr -s ',": ' '\t' | \
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
62 tail -n +2 | head -$np |cut -f 3-5 | \
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
63 while read i ip port
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
64 do
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
65 echo $i $port $ip
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
66 done) | \
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
67 parallel --gnu --colsep ' ' -j $np "$(dirname "$0")"/internal/invoke.sh $id $me "$cmd" {} "$@" || { r=$? ; echo parallel exited with status=$r 1>&2;}
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
68 if [ "${tfile}" ]; then rm ${tfile}*; fi