25
|
1 #!/bin/bash
|
|
2 # Test script to split timedWhich output files [found in ifile.txt] across threads
|
|
3 # to merge earlier tabulations of http vs. https by last-modified date:
|
|
4 # Usage: fixAndMerge.sh id home [-t] numWorkerProcesses
|
|
5 # If -t, no random wait, just id seconds
|
|
6 # remove >>errs once tested
|
|
7 #set -e -o pipefail
|
|
8 echo $$ > test1.pid
|
|
9 proc=$1
|
|
10 res=/var/data/res$proc
|
|
11 home=$2
|
|
12 shift 2
|
|
13 function lrand {
|
|
14 # cheap bad little random number generator
|
|
15 echo $(( 1 + ($(openssl rand 1 | od -d | head -1 | tr -s ' ' | cut -f 2 -d ' ') % $1)))
|
|
16 }
|
|
17 if [ "$1" = "-t" ]
|
|
18 then
|
|
19 shift
|
|
20 pause=$proc
|
|
21 else
|
|
22 pause=$(lrand 10)
|
|
23 fi
|
|
24 wp=$1
|
|
25 touch .running
|
|
26 trap "{
|
|
27 set -e -o pipefail
|
|
28 cd /var/data
|
|
29 tar -czhf - m.* res* | \
|
|
30 ssh -o StrictHostKeyChecking=no -q $home \"{ cd data
|
|
31 mkdir -p mergedWhich.16
|
|
32 cd mergedWhich.16
|
|
33 tar -xzf - ; } 2>>errs\"
|
|
34 rm -rf res* m.*
|
|
35 cd
|
|
36 rm ifile.txt *.pid
|
|
37 ( sleep 5 ; rm -f nohup.cc ) &
|
|
38 }" EXIT
|
|
39 mkdir -p $res
|
|
40 log=$res/log
|
|
41 echo \# $(date) "running |$proc|$home|$pause|$wp|" >> $log
|
|
42 pRes=0
|
|
43 echo "# $(date) $proc $(wc -l ifile.txt)" >> $log
|
|
44 parallel --round-robin --pipe -j $wp "_fixAndMerge.sh {#} $home $pause > $res/m.{#} 2>>$res/errs{#}" <ifile.txt || echo "ppfailed $? ${PIPESTATUS[@]}" 1>&2
|
|
45 echo \# $(date) subprocs done >> $log
|
|
46 cat $res/m.* | awk '{c[$1 "\t" $2 "\t" $3]+=$4} END {for (k in c) {print k "\t" c[k]}}' > /var/data/m.$proc
|
|
47 echo \# $(date) subres merged >> $log
|
|
48 rm .running
|
|
49
|
|
50
|