changeset 29:2bde174fe9e2

as used for mergedWhich.x64700
author Henry S. Thompson <ht@markup.co.uk>
date Mon, 19 Nov 2018 18:31:21 +0000
parents db75ec542c66
children 9275e2a8b5e2
files workers/bin/_fixAndMerge.sh
diffstat 1 files changed, 4 insertions(+), 2 deletions(-) [+]
line wrap: on
line diff
--- a/workers/bin/_fixAndMerge.sh	Sat Nov 10 13:22:21 2018 +0000
+++ b/workers/bin/_fixAndMerge.sh	Mon Nov 19 18:31:21 2018 +0000
@@ -8,8 +8,10 @@
 sleep $pause
 echo \#.$id $(date) starting >> $log
 cat > /var/data/in$id
-echo \#.$id $(date) got list >> $log
-until ( rm /var/data/d$id ; xargs -n 100 ssh $home "xargs gzip -c" < /var/data/in$id | gunzip -c >> /var/data/d$id ) do echo retrying 1>&2; sleep $pause; done
+echo \#.$id $(date) got list $(wc -l /var/data/in$id) >> $log
+rm -f /var/data/d$id
+xargs -n 100 _doFetch.sh "$@" < /var/data/in$id >/var/data/d$id
+echo \#.$id $(wc -l /var/data/d$id)
 echo \#.$id $(date) got data >> $log
 fixDates.py < /var/data/d$id
 echo \#.$id $(date) done >> $log