Mercurial > hg > cc > azure
changeset 26:58d46e6983fa
-mforce (?) multiple processors to be used
| author | Henry S. Thompson <ht@markup.co.uk> |
|---|---|
| date | Wed, 07 Nov 2018 19:36:30 +0000 |
| parents | 1b9329f6b5e1 |
| children | dd19cf97b6dd |
| files | workers/bin/fixAndMerge.sh |
| diffstat | 1 files changed, 6 insertions(+), 5 deletions(-) [+] |
line wrap: on
line diff
--- a/workers/bin/fixAndMerge.sh Wed Nov 07 17:37:27 2018 +0000 +++ b/workers/bin/fixAndMerge.sh Wed Nov 07 19:36:30 2018 +0000 @@ -28,8 +28,8 @@ cd /var/data tar -czhf - m.* res* | \ ssh -o StrictHostKeyChecking=no -q $home \"{ cd data - mkdir -p mergedWhich.16 - cd mergedWhich.16 + mkdir -p mergedWhich + cd mergedWhich tar -xzf - ; } 2>>errs\" rm -rf res* m.* cd @@ -40,10 +40,11 @@ log=$res/log echo \# $(date) "running |$proc|$home|$pause|$wp|" >> $log pRes=0 -echo "# $(date) $proc $(wc -l ifile.txt)" >> $log -parallel --round-robin --pipe -j $wp "_fixAndMerge.sh {#} $home $pause > $res/m.{#} 2>>$res/errs{#}" <ifile.txt || echo "ppfailed $? ${PIPESTATUS[@]}" 1>&2 +N=$(wc -l< ifile.txt) +echo "# $(date) $proc $N" >> $log +parallel --pipe -N$((N / wp)) -j $wp "_fixAndMerge.sh {#} $home $pause > $res/m.{#} 2>>$res/errs{#}" <ifile.txt 2>>$res/errs|| echo "ppfailed $? ${PIPESTATUS[@]}" >> $res/errs echo \# $(date) subprocs done >> $log -cat $res/m.* | awk '{c[$1 "\t" $2 "\t" $3]+=$4} END {for (k in c) {print k "\t" c[k]}}' > /var/data/m.$proc +cat $res/m.* | awk '{c[$1 "\t" $2 "\t" $3]+=$4} END {for (k in c) {print k "\t" c[k]}}' > /var/data/m.$proc 2>> $res/errs echo \# $(date) subres merged >> $log rm .running