Mercurial > hg > cc > azure
diff workers/bin/_fixAndMerge.sh @ 27:dd19cf97b6dd
attempt to fix robustness pblms
| author | Henry S. Thompson <ht@markup.co.uk> |
|---|---|
| date | Sat, 10 Nov 2018 13:20:56 +0000 |
| parents | 1b9329f6b5e1 |
| children | 2bde174fe9e2 |
line wrap: on
line diff
--- a/workers/bin/_fixAndMerge.sh Wed Nov 07 19:36:30 2018 +0000 +++ b/workers/bin/_fixAndMerge.sh Sat Nov 10 13:20:56 2018 +0000 @@ -1,8 +1,16 @@ #!/bin/bash -# Usage: [cat remote filenames] | _fixAndMerge.sh id home pause +# Usage: [cat remote filenames] | _fixAndMerge.sh id home pause log id=$1 home=$2 pause=$3 +log=$4 # Don't all start at once sleep $pause -ssh $home "xargs cat" | fixDates.py +echo \#.$id $(date) starting >> $log +cat > /var/data/in$id +echo \#.$id $(date) got list >> $log +until ( rm /var/data/d$id ; xargs -n 100 ssh $home "xargs gzip -c" < /var/data/in$id | gunzip -c >> /var/data/d$id ) do echo retrying 1>&2; sleep $pause; done +echo \#.$id $(date) got data >> $log +fixDates.py < /var/data/d$id +echo \#.$id $(date) done >> $log +