Mercurial > hg > cc > azure
changeset 9:55e953e5c66f
wrun.sh, invoke.sh:
add control of use of nohup, with -i(mmediately) and -w(ait) respectively
wrun.sh: add support for fixed file input via -ff, handle -ff - in obvious way
wbash.sh: use -ff - and -i to execute shell commands on all workers
| author | Henry S. Thompson <ht@markup.co.uk> |
|---|---|
| date | Tue, 02 Oct 2018 10:52:45 +0000 |
| parents | 5db6015689a2 |
| children | 2fbefb8d1a9e |
| files | master/bin/internal/invoke.sh master/bin/wbash.sh master/bin/wrun.sh |
| diffstat | 3 files changed, 53 insertions(+), 9 deletions(-) [+] |
line wrap: on
line diff
--- a/master/bin/internal/invoke.sh Mon Oct 01 18:29:39 2018 +0000 +++ b/master/bin/internal/invoke.sh Tue Oct 02 10:52:45 2018 +0000 @@ -6,6 +6,11 @@ # via ssh to ip:port # If ifile is not /dev/null, feed in as stdin # Unless -x, worker id is passed as first arg +if [ "$1" = "-w" ] +then + shift + wait=1 +fi if [ "$1" = "-x" ] then shift @@ -24,7 +29,10 @@ if [ "$ifile" != "/dev/null" ] then echo "# from $ifile" 1>&2 - scp -P $port $ifile $ip:ifile.txt && \ + scp -P $port $ifile $ip:ifile.txt +fi || echo scp failed, status=$? 1>&2 +if [ "$wait" ] +then ssh -t -p $port $ip "nohup $cmd $id $me ""$@"" > nohup.cc" else ssh -p $port $ip "$cmd $id $me ""$@"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/master/bin/wbash.sh Tue Oct 02 10:52:45 2018 +0000 @@ -0,0 +1,17 @@ +#!/bin/bash +if [ $# -lt 1 ] +then + cat 1>&2 <<EOF +Usage: cat cmds | $0 SSname [...args] +Where SSname is the name of a VM scale set. + +Runs bash [with args] on every machine in a scale set, + on commands from stdin + by doing as it were + cat | ssh machine "bash \$args" +EOF + exit 1 +fi +ss=$1 +shift +wrun.sh $ss bash -x -ff - -i "$@" '\<ifile.txt'
--- a/master/bin/wrun.sh Mon Oct 01 18:29:39 2018 +0000 +++ b/master/bin/wrun.sh Tue Oct 02 10:52:45 2018 +0000 @@ -2,15 +2,18 @@ set -e -o pipefail if [ $# -lt 2 ]; then cat 1>&2 <<EOF -Usage: $0 SSname cmd [-x] [-np n] [-f file] [...args] +Usage: $0 SSname cmd [-x] [-np n] [-f[f] file] [-i] [...args] Where name is the name of a VM scale set. Runs cmd on every machine in a scale set, passing args and, - if -f, one line from file per worker + if -f, lines from file split per worker + if -ff, complete file sent to all workers unless -x, worker id by doing as it were - [ echo line-from-file |] ssh machine "$cmd [id] "$args"" + [ echo line(s)-from-file |] ssh machine "$cmd [id] "$args"" + if -i, don\'t use nohup on the workers so returns immediately + [default is to use nohup unless neither -f or -ff] EOF exit 1 fi @@ -36,23 +39,39 @@ echo "|$name|$cmd|$@|$np|" 1>&2 me=$(az vm list-ip-addresses -g cc -n Deb1 | jq -r '.[0].virtualMachine.network.publicIpAddresses|.[0].ipAddress') echo me=$me 1>&2 +wait='-w' if [ "$1" = "-f" ] then file=$2 shift 2 tfile=$(mktemp) split -n r/$np -u --filter="cat > $tfile\$\$" $file & - # Wait for fifos to be built + # Wait for files to be built while [ $np -gt $(ls ${tfile}* | wc -l) ]; do echo -n . 1>&2; sleep 1; done echo 1>&2 wc -l ${tfile}* 1>&2 +elif [ "$1" = "-ff" ] +then + file=$2 + shift 2 + if [ "$file" = "-" ] + then + file=$(mktemp) + cat > $file + fi else file=/dev/null + wait='' fi -echo "|$np|$file|$tfile|" 1>&2 +if [ "$1" = "-i" ] +then + shift + wait='' +fi +echo "|$np|$file|$tfile|$wait|" 1>&2 paste -d ' ' \ - <(if [ "$file" = "/dev/null" ] - then yes /dev/null|head -$np + <(if [ -z "$tfile" ] + then yes $file|head -$np else echo ${tfile}?* | tr ' ' '\012' # the name of the file of # input lines for each worker @@ -64,5 +83,5 @@ do echo $i $port $ip done) | \ - parallel --gnu --colsep ' ' -j $np "$(dirname "$0")"/internal/invoke.sh $id $me "$cmd" {} "$@" || { r=$? ; echo parallel exited with status=$r 1>&2;} + parallel --gnu --colsep ' ' -j $np "$(dirname "$0")"/internal/invoke.sh $wait $id $me "$cmd" {} "$@" || { r=$? ; echo parallel exited with status=$r 1>&2;} if [ "${tfile}" ]; then rm ${tfile}*; fi