#!/bin/bash
# This runs on the compute nodes...
# Args: CC-MAIN-2019-35 nl1_counts langs 0 299
# count languages (from file named by langs, found in resdir) in parallel, taking input directly from $cc/cdx_counts/xxx.tsv

N=$SLURM_JOB_NUM_NODES
n=$SLURM_NTASKS
c=$SLURM_CPUS_PER_TASK
nodename=$SLURMD_NODENAME
local=$SLURM_LOCALID
node=$SLURM_NODEID
task=$SLURM_PROCID

threadsPerTask=2
pjobs=$((c / $threadsPerTask))

cc=$1
resdir=$W/$USER/results/$cc/$2
srcdir=$W/hst/results/$cc/cdx_counts
langs=$3
langfile=$resdir/$langs
s1=$4
sn=$5

echo $(date) task $n.$task on $nodename:$N.$node start $(pwd) 1>&2

mkdir -p $resdir

doit () {
 echo $(date) start $1 $task $PARALLEL_SEQ 1>&2
 fgrep '	w	' $srcdir/$1.tsv | \
   awk 'BEGIN {while (getline < "'$langfile'") {l[$0]=1}}
        {if (l[$4]) {print $1,$4}}' | uniq -c | \
    $W/shared/bin/uniq_merge.py > $resdir/${langs}_$1.tsv
 echo $(date) end $1 $task $PARALLEL_SEQ 1>&2
}

export -f doit
export srcdir resdir task langs langfile

seq $s1 $sn | while read i
 do if [ $((i % $n)) -eq $task ]
 then printf '%03g\n' $i
 fi
 done | \
   parallel -j $pjobs doit '{}'

echo $(date) task $n.$task on $nodename:$N.$node end 1>&2