#!/bin/bash
# This runs on the compute nodes...
# Args: CC-MAIN-2019-35 nl1_counts langs 0 299
# count languages (from file named by langs, found in resdir) in parallel, taking input directly from $cc/cdx_counts/xxx.tsv

N=$SLURM_JOB_NUM_NODES
n=$SLURM_NTASKS
c=$SLURM_CPUS_PER_TASK
nodename=$SLURMD_NODENAME
local=$SLURM_LOCALID
node=$SLURM_NODEID
task=$SLURM_PROCID

threadsPerTask=2
pjobs=$((c / $threadsPerTask))

cc=$1
resdir=$W/$USER/results/$cc/$2
srcdir=$W/hst/results/$cc/cdx_counts
langs=$3
langfile=$resdir/$langs
s1=$4
sn=$5

echo $(date) task $n.$task on $nodename:$N.$node start $(pwd) 1>&2

mkdir -p $resdir

doit () {
 echo $(date) start $1 $task $PARALLEL_SEQ 1>&2
 fgrep '	w	' $srcdir/$1.tsv | \
   awk 'BEGIN {while (getline < "'$langfile'") {l[$0]=1}}
        {if (l[$5]) {print $1,$2,$5}}' | \
    $W/shared/bin/uniq_merge.py > $resdir/${langs}_$1.tsv
 echo $(date) end $1 $task $PARALLEL_SEQ 1>&2
}

export -f doit
export srcdir resdir task langs langfile

$W/hst/bin/share_by_task.sh -f "%03g\n" -s $s1 $sn $n $task | parallel -j $pjobs doit '{}'

echo $(date) task $n.$task on $nodename:$N.$node end 1>&2