view master/bin/wrun.sh @ 30:9275e2a8b5e2

hacking to get id into wbash.sh, maybe buggy?
author Henry S. Thompson <ht@markup.co.uk>
date Mon, 19 Nov 2018 18:32:30 +0000
parents 2fbefb8d1a9e
children
line wrap: on
line source

#!/bin/bash
set -e -o pipefail
if [ $# -lt 2 ]; then
  cat 1>&2 <<EOF
Usage: $0 SSname cmd [-x] [-np n] [-f[f] file] [-i] [...args]
Where name is the name of a VM scale set.

Runs cmd on every machine in a scale set,
 (only using n machines if -np n is present)
 passing args and (as ~/ifile.txt), 
  if -f, lines from file split per worker
  if -ff, complete file sent to all workers
  and, unless -x, worker id
 by doing as it were
  scp machine: <(line(s)-from-file) ifile.txt && ssh machine "$cmd [id] "$args""
  if -i, don\'t use nohup on the workers so returns immediately
   [default is to use nohup unless neither -f or -ff]
EOF
  exit 1
fi
group=cc
name=$1
cmd="$2"
shift 2
if [ "$1" = "-d" ]
then
 debug=-d
 shift
fi
if [ "$1" = "-x" ]
then
 xx=-x
 shift
else
 xx=
fi
if [ "$1" = "-np" ]
then
 np=$2
 shift
 shift
else
 np=$(az vmss get-instance-view -g $group -n $name | jq '.virtualMachine.statusesSummary | .[0].count')
fi
echo "|$name|$cmd|$@|$np|" 1>&2
me=$(az vm list-ip-addresses -g cc -n Deb1 | jq -r '.[0].virtualMachine.network.publicIpAddresses|.[0].ipAddress')
echo me=$me 1>&2
wait='-w'
if [ "$1" = "-f" ]
then
 file=$2
 shift 2
 tfile=$(mktemp)
 tee >(wc -l 1>&2) | split -n r/$np -u --filter="cat > $tfile\$\$" $file
 wc -l ${tfile}* 1>&2
elif [ "$1" = "-ff" ]
then
 file=$2
 shift 2
 if [ "$file" = "-" ]
 then
  file=$(mktemp)
  cat > $file
 fi
else
 file=/dev/null
 wait=''
fi
if [ "$1" = "-i" ]
then 
 shift
 wait=''
fi
echo "|$np|$file|$tfile|$wait|" 1>&2
paste -d ' ' \
 <(if [ -z "$tfile" ]
   then yes $file|head -$np
   else
     echo ${tfile}?* | tr ' ' '\012' # the name of the file of
                                    # input lines for each worker
   fi
  ) \
 <(az vmss list-instance-connection-info -g $group -n $name | tr -s ',": ' '\t' | \
    tail -n +2 | head -$np |cut -f 3-5 | \
  while read i ip port
   do
    echo $i $port $ip
   done) | tee /tmp/test.txt | \
 parallel --gnu --colsep ' ' -j $np "$(dirname "$0")"/internal/invoke.sh $debug $wait $xx $me "$cmd" {} "$@" || { r=$? ; echo parallel exited with status=$r 1>&2;}
#if [ "${tfile}" ]; then rm ${tfile}*; fi