Mercurial > hg > cc > azure
diff master/bin/wrun.sh @ 7:a7637c994964
cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh:
From kenneth, slight mods in some cases
share.sh:
Distribute files to workers
wrun.sh, invoke.sh:
From last year, slight mods
author | Henry S. Thompson <ht@markup.co.uk> |
---|---|
date | Sun, 30 Sep 2018 20:53:43 +0000 |
parents | |
children | 55e953e5c66f |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/master/bin/wrun.sh Sun Sep 30 20:53:43 2018 +0000 @@ -0,0 +1,68 @@ +#!/bin/bash +set -e -o pipefail +if [ $# -lt 2 ]; then + cat 1>&2 <<EOF +Usage: $0 SSname cmd [-x] [-np n] [-f file] [...args] +Where name is the name of a VM scale set. + +Runs cmd on every machine in a scale set, + passing args and, + if -f, one line from file per worker + unless -x, worker id + by doing as it were + [ echo line-from-file |] ssh machine "$cmd [id] "$args"" +EOF + exit 1 +fi +group=cc +name=$1 +cmd="$2" +shift 2 +if [ "$1" = "-x" ] +then + id=-x + shift +else + id= +fi +if [ "$1" = "-np" ] +then + np=$2 + shift + shift +else + np=$(az vmss get-instance-view -g $group -n $name | jq '.virtualMachine.statusesSummary | .[0].count') +fi +echo "|$name|$cmd|$@|$np|" 1>&2 +me=$(az vm list-ip-addresses -g cc -n Deb1 | jq -r '.[0].virtualMachine.network.publicIpAddresses|.[0].ipAddress') +echo me=$me 1>&2 +if [ "$1" = "-f" ] +then + file=$2 + shift 2 + tfile=$(mktemp) + split -n r/$np -u --filter="cat > $tfile\$\$" $file & + # Wait for fifos to be built + while [ $np -gt $(ls ${tfile}* | wc -l) ]; do echo -n . 1>&2; sleep 1; done + echo 1>&2 + wc -l ${tfile}* 1>&2 +else + file=/dev/null +fi +echo "|$np|$file|$tfile|" 1>&2 +paste -d ' ' \ + <(if [ "$file" = "/dev/null" ] + then yes /dev/null|head -$np + else + echo ${tfile}?* | tr ' ' '\012' # the name of the file of + # input lines for each worker + fi + ) \ + <(az vmss list-instance-connection-info -g $group -n $name | tr -s ',": ' '\t' | \ + tail -n +2 | head -$np |cut -f 3-5 | \ + while read i ip port + do + echo $i $port $ip + done) | \ + parallel --gnu --colsep ' ' -j $np "$(dirname "$0")"/internal/invoke.sh $id $me "$cmd" {} "$@" || { r=$? ; echo parallel exited with status=$r 1>&2;} +if [ "${tfile}" ]; then rm ${tfile}*; fi