annotate workers/bin/_fixAndMerge.sh @ 29:2bde174fe9e2

as used for mergedWhich.x64700
author Henry S. Thompson <ht@markup.co.uk>
date Mon, 19 Nov 2018 18:31:21 +0000
parents dd19cf97b6dd
children 4c117ee8ed75
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
25
1b9329f6b5e1 works on which.16
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
1 #!/bin/bash
27
dd19cf97b6dd attempt to fix robustness pblms
Henry S. Thompson <ht@markup.co.uk>
parents: 25
diff changeset
2 # Usage: [cat remote filenames] | _fixAndMerge.sh id home pause log
25
1b9329f6b5e1 works on which.16
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
3 id=$1
1b9329f6b5e1 works on which.16
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
4 home=$2
1b9329f6b5e1 works on which.16
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
5 pause=$3
27
dd19cf97b6dd attempt to fix robustness pblms
Henry S. Thompson <ht@markup.co.uk>
parents: 25
diff changeset
6 log=$4
25
1b9329f6b5e1 works on which.16
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
7 # Don't all start at once
1b9329f6b5e1 works on which.16
Henry S. Thompson <ht@markup.co.uk>
parents:
diff changeset
8 sleep $pause
27
dd19cf97b6dd attempt to fix robustness pblms
Henry S. Thompson <ht@markup.co.uk>
parents: 25
diff changeset
9 echo \#.$id $(date) starting >> $log
dd19cf97b6dd attempt to fix robustness pblms
Henry S. Thompson <ht@markup.co.uk>
parents: 25
diff changeset
10 cat > /var/data/in$id
29
2bde174fe9e2 as used for mergedWhich.x64700
Henry S. Thompson <ht@markup.co.uk>
parents: 27
diff changeset
11 echo \#.$id $(date) got list $(wc -l /var/data/in$id) >> $log
2bde174fe9e2 as used for mergedWhich.x64700
Henry S. Thompson <ht@markup.co.uk>
parents: 27
diff changeset
12 rm -f /var/data/d$id
2bde174fe9e2 as used for mergedWhich.x64700
Henry S. Thompson <ht@markup.co.uk>
parents: 27
diff changeset
13 xargs -n 100 _doFetch.sh "$@" < /var/data/in$id >/var/data/d$id
2bde174fe9e2 as used for mergedWhich.x64700
Henry S. Thompson <ht@markup.co.uk>
parents: 27
diff changeset
14 echo \#.$id $(wc -l /var/data/d$id)
27
dd19cf97b6dd attempt to fix robustness pblms
Henry S. Thompson <ht@markup.co.uk>
parents: 25
diff changeset
15 echo \#.$id $(date) got data >> $log
dd19cf97b6dd attempt to fix robustness pblms
Henry S. Thompson <ht@markup.co.uk>
parents: 25
diff changeset
16 fixDates.py < /var/data/d$id
dd19cf97b6dd attempt to fix robustness pblms
Henry S. Thompson <ht@markup.co.uk>
parents: 25
diff changeset
17 echo \#.$id $(date) done >> $log
dd19cf97b6dd attempt to fix robustness pblms
Henry S. Thompson <ht@markup.co.uk>
parents: 25
diff changeset
18