#!/bin/bash
# Test script to split timedWhich output files [found in ifile.txt] across threads
#   to merge earlier tabulations of http vs. https by last-modified date:
# Usage: fixAndMerge.sh id home [-t] numWorkerProcesses
#   If -t, no random wait, just id seconds
# remove >>errs once tested
#set -e -o pipefail
echo $$ > test1.pid
proc=$1
res=/var/data/res$proc
home=$2
shift 2
function lrand {
# cheap bad little random number generator
echo $(( 1 + ($(openssl rand 1 | od -d | head -1 | tr -s ' ' | cut -f 2 -d ' ') % $1)))
}
if [ "$1" = "-t" ]
then
 shift
 pause=$proc
else
 pause=$(lrand 10)
fi
wp=$1
touch .running
trap "{ 
  set -e -o pipefail
  cd /var/data
  tar -czhf - m.* res* | \
   ssh -o StrictHostKeyChecking=no -q $home \"{ cd data
                    mkdir -p mergedWhich
                    cd mergedWhich
                    tar -xzf - ; } 2>>errs\"
  rm -rf res* m.*
  cd
  rm ifile.txt *.pid
  ( sleep 5 ; rm -f nohup.cc ) &
  }" EXIT
mkdir -p $res
log=$res/log
echo \# $(date) "running |$proc|$home|$pause|$wp|" >> $log
pRes=0
N=$(wc -l< ifile.txt)
echo "# $(date) $proc $N" >> $log
parallel --pipe -N$((N / wp)) -j $wp "_fixAndMerge.sh {#} $home $pause > $res/m.{#} 2>>$res/errs{#}" <ifile.txt 2>>$res/errs|| echo "ppfailed $? ${PIPESTATUS[@]}" >> $res/errs
echo \# $(date) subprocs done >> $log
cat $res/m.* | awk '{c[$1 "\t" $2 "\t" $3]+=$4} END {for (k in c) {print k "\t" c[k]}}' > /var/data/m.$proc 2>> $res/errs
echo \# $(date) subres merged >> $log
rm .running