Mercurial > hg > cc > azure

annotate master/bin/wrun.sh @ 30:9275e2a8b5e2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
hacking to get id into wbash.sh, maybe buggy?
author Henry S. Thompson <ht@markup.co.uk>
date Mon, 19 Nov 2018 18:32:30 +0000
parents 2fbefb8d1a9e
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
1 #!/bin/bash
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
2 set -e -o pipefail
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
3 if [ $# -lt 2 ]; then
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
4 cat 1>&2 <<EOF
9
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
5 Usage: $0 SSname cmd [-x] [-np n] [-f[f] file] [-i] [...args]
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
6 Where name is the name of a VM scale set.
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
8 Runs cmd on every machine in a scale set,
10
2fbefb8d1a9e wrun.sh: usage catchup
Henry S. Thompson <ht@markup.co.uk>
parents: 9
diff changeset
9 (only using n machines if -np n is present)
2fbefb8d1a9e wrun.sh: usage catchup
Henry S. Thompson <ht@markup.co.uk>
parents: 9
diff changeset
10 passing args and (as ~/ifile.txt),
9
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
11 if -f, lines from file split per worker
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
12 if -ff, complete file sent to all workers
10
2fbefb8d1a9e wrun.sh: usage catchup
Henry S. Thompson <ht@markup.co.uk>
parents: 9
diff changeset
13 and, unless -x, worker id
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
14 by doing as it were
10
2fbefb8d1a9e wrun.sh: usage catchup
Henry S. Thompson <ht@markup.co.uk>
parents: 9
diff changeset
15 scp machine: <(line(s)-from-file) ifile.txt && ssh machine "$cmd [id] "$args""
9
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
16 if -i, don\'t use nohup on the workers so returns immediately
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
17 [default is to use nohup unless neither -f or -ff]
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
18 EOF
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
19 exit 1
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
20 fi
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
21 group=cc
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
22 name=$1
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
23 cmd="$2"
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
24 shift 2
30
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
25 if [ "$1" = "-d" ]
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
26 then
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
27 debug=-d
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
28 shift
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
29 fi
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
30 if [ "$1" = "-x" ]
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
31 then
30
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
32 xx=-x
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
33 shift
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
34 else
30
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
35 xx=
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
36 fi
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
37 if [ "$1" = "-np" ]
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
38 then
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
39 np=$2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
40 shift
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
41 shift
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
42 else
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
43 np=$(az vmss get-instance-view -g $group -n $name | jq '.virtualMachine.statusesSummary | .[0].count')
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
44 fi
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
45 echo "|$name|$cmd|$@|$np|" 1>&2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
46 me=$(az vm list-ip-addresses -g cc -n Deb1 | jq -r '.[0].virtualMachine.network.publicIpAddresses|.[0].ipAddress')
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
47 echo me=$me 1>&2
9
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
48 wait='-w'
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
49 if [ "$1" = "-f" ]
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
50 then
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
51 file=$2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
52 shift 2
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
53 tfile=$(mktemp)
30
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
54 tee >(wc -l 1>&2) | split -n r/$np -u --filter="cat > $tfile\$\$" $file
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
55 wc -l ${tfile}* 1>&2
9
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
56 elif [ "$1" = "-ff" ]
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
57 then
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
58 file=$2
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
59 shift 2
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
60 if [ "$file" = "-" ]
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
61 then
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
62 file=$(mktemp)
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
63 cat > $file
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
64 fi
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
65 else
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
66 file=/dev/null
9
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
67 wait=''
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
68 fi
9
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
69 if [ "$1" = "-i" ]
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
70 then
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
71 shift
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
72 wait=''
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
73 fi
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
74 echo "|$np|$file|$tfile|$wait|" 1>&2
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
75 paste -d ' ' \
9
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
76 <(if [ -z "$tfile" ]
55e953e5c66f wrun.sh, invoke.sh:
Henry S. Thompson <ht@markup.co.uk>
parents: 7
diff changeset
77 then yes $file|head -$np
7
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
78 else
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
79 echo ${tfile}?* | tr ' ' '\012' # the name of the file of
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
80 # input lines for each worker
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
81 fi
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
82 ) \
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
83 <(az vmss list-instance-connection-info -g $group -n $name | tr -s ',": ' '\t' | \
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
84 tail -n +2 | head -$np |cut -f 3-5 | \
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
85 while read i ip port
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
86 do
a7637c994964 cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
87 echo $i $port $ip
30
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
88 done) | tee /tmp/test.txt | \
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
89 parallel --gnu --colsep ' ' -j $np "$(dirname "$0")"/internal/invoke.sh $debug $wait $xx $me "$cmd" {} "$@" || { r=$? ; echo parallel exited with status=$r 1>&2;}
9275e2a8b5e2 hacking to get id into wbash.sh, maybe buggy?
Henry S. Thompson <ht@markup.co.uk>
parents: 10
diff changeset
90 #if [ "${tfile}" ]; then rm ${tfile}*; fi