Mercurial > hg > cc > azure

comparison workers/bin/fixAndMerge.sh @ 26:58d46e6983fa

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
-mforce (?) multiple processors to be used
author Henry S. Thompson <ht@markup.co.uk>
date Wed, 07 Nov 2018 19:36:30 +0000
parents 1b9329f6b5e1
children dd19cf97b6dd
comparison
equal deleted inserted replaced
25:1b9329f6b5e1 26:58d46e6983fa
26 trap "{ 26 trap "{
27 set -e -o pipefail 27 set -e -o pipefail
28 cd /var/data 28 cd /var/data
29 tar -czhf - m.* res* | \ 29 tar -czhf - m.* res* | \
30 ssh -o StrictHostKeyChecking=no -q $home \"{ cd data 30 ssh -o StrictHostKeyChecking=no -q $home \"{ cd data
31 mkdir -p mergedWhich.16 31 mkdir -p mergedWhich
32 cd mergedWhich.16 32 cd mergedWhich
33 tar -xzf - ; } 2>>errs\" 33 tar -xzf - ; } 2>>errs\"
34 rm -rf res* m.* 34 rm -rf res* m.*
35 cd 35 cd
36 rm ifile.txt *.pid 36 rm ifile.txt *.pid
37 ( sleep 5 ; rm -f nohup.cc ) & 37 ( sleep 5 ; rm -f nohup.cc ) &
38 }" EXIT 38 }" EXIT
39 mkdir -p $res 39 mkdir -p $res
40 log=$res/log 40 log=$res/log
41 echo \# $(date) "running |$proc|$home|$pause|$wp|" >> $log 41 echo \# $(date) "running |$proc|$home|$pause|$wp|" >> $log
42 pRes=0 42 pRes=0
43 echo "# $(date) $proc $(wc -l ifile.txt)" >> $log 43 N=$(wc -l< ifile.txt)
44 parallel --round-robin --pipe -j $wp "_fixAndMerge.sh {#} $home $pause > $res/m.{#} 2>>$res/errs{#}" <ifile.txt || echo "ppfailed $? ${PIPESTATUS[@]}" 1>&2 44 echo "# $(date) $proc $N" >> $log
45 parallel --pipe -N$((N / wp)) -j $wp "_fixAndMerge.sh {#} $home $pause > $res/m.{#} 2>>$res/errs{#}" <ifile.txt 2>>$res/errs|| echo "ppfailed $? ${PIPESTATUS[@]}" >> $res/errs
45 echo \# $(date) subprocs done >> $log 46 echo \# $(date) subprocs done >> $log
46 cat $res/m.* | awk '{c[$1 "\t" $2 "\t" $3]+=$4} END {for (k in c) {print k "\t" c[k]}}' > /var/data/m.$proc 47 cat $res/m.* | awk '{c[$1 "\t" $2 "\t" $3]+=$4} END {for (k in c) {print k "\t" c[k]}}' > /var/data/m.$proc 2>> $res/errs
47 echo \# $(date) subres merged >> $log 48 echo \# $(date) subres merged >> $log
48 rm .running 49 rm .running
49 50
50 51