diff master/bin/wrun.sh @ 7:a7637c994964

cull_network.py, parse_load_balance.py, vmss_setup.sh x 2, vmss_create.sh: From kenneth, slight mods in some cases share.sh: Distribute files to workers wrun.sh, invoke.sh: From last year, slight mods
author Henry S. Thompson <ht@markup.co.uk>
date Sun, 30 Sep 2018 20:53:43 +0000
parents
children 55e953e5c66f
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/master/bin/wrun.sh	Sun Sep 30 20:53:43 2018 +0000
@@ -0,0 +1,68 @@
+#!/bin/bash
+set -e -o pipefail
+if [ $# -lt 2 ]; then
+  cat 1>&2 <<EOF
+Usage: $0 SSname cmd [-x] [-np n] [-f file] [...args]
+Where name is the name of a VM scale set.
+
+Runs cmd on every machine in a scale set,
+ passing args and, 
+  if -f, one line from file per worker
+  unless -x, worker id
+ by doing as it were
+  [ echo line-from-file |] ssh machine "$cmd [id] "$args"" 
+EOF
+  exit 1
+fi
+group=cc
+name=$1
+cmd="$2"
+shift 2
+if [ "$1" = "-x" ]
+then
+ id=-x
+ shift
+else
+ id=
+fi
+if [ "$1" = "-np" ]
+then
+ np=$2
+ shift
+ shift
+else
+ np=$(az vmss get-instance-view -g $group -n $name | jq '.virtualMachine.statusesSummary | .[0].count')
+fi
+echo "|$name|$cmd|$@|$np|" 1>&2
+me=$(az vm list-ip-addresses -g cc -n Deb1 | jq -r '.[0].virtualMachine.network.publicIpAddresses|.[0].ipAddress')
+echo me=$me 1>&2
+if [ "$1" = "-f" ]
+then
+ file=$2
+ shift 2
+ tfile=$(mktemp)
+ split -n r/$np -u --filter="cat > $tfile\$\$" $file &
+ # Wait for fifos to be built
+ while [ $np -gt $(ls ${tfile}* | wc -l) ]; do echo -n . 1>&2; sleep 1; done
+ echo 1>&2
+ wc -l ${tfile}* 1>&2
+else
+ file=/dev/null
+fi
+echo "|$np|$file|$tfile|" 1>&2
+paste -d ' ' \
+ <(if [ "$file" = "/dev/null" ]
+   then yes /dev/null|head -$np
+   else
+     echo ${tfile}?* | tr ' ' '\012' # the name of the file of
+                                    # input lines for each worker
+   fi
+  ) \
+ <(az vmss list-instance-connection-info -g $group -n $name | tr -s ',": ' '\t' | \
+    tail -n +2 | head -$np |cut -f 3-5 | \
+  while read i ip port
+   do
+    echo $i $port $ip
+   done) | \
+ parallel --gnu --colsep ' ' -j $np "$(dirname "$0")"/internal/invoke.sh $id $me "$cmd" {} "$@" || { r=$? ; echo parallel exited with status=$r 1>&2;}
+if [ "${tfile}" ]; then rm ${tfile}*; fi