#!/bin/bash
# Usage: [cat remote filenames] | _fixAndMerge.sh id home pause log
id=$1
home=$2
pause=$3
log=$4
# Don't all start at once
sleep $pause
echo \#.$id $(date) starting >> $log
cat > /var/data/in$id
echo \#.$id $(date) got list >> $log
until ( rm /var/data/d$id ; xargs -n 100 ssh $home "xargs gzip -c" < /var/data/in$id | gunzip -c >> /var/data/d$id ) do echo retrying 1>&2; sleep $pause; done
echo \#.$id $(date) got data >> $log
fixDates.py < /var/data/d$id
echo \#.$id $(date) done >> $log