#!/bin/bash
# Usage doExtract.sh ccid jobid segid
ccid=$1
export jobid=$2
segid=$3
echo $(date) $(hostname) $jobid $segid
mkdir -p $segid/logs
cd $segid
ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \
parallel --joblog job_${jobid}.log -j 18 -N 1 'id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) ; \
    fid=$(printf "%03.0f" $(({#} - 1))) ; \
    lf=logs/${jobid}_${fid}_log ; \
    echo starting $id $(date) >> $lf ; \
    unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> $lf ; \
    echo finished ${id} $(date) >> $lf'
res=$?
echo $(date) $(hostname) $jobid $segid $res
if [ $res = 0 ]
 then
  pfx=$(ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*-00000.warc.gz |\
        cut -f 6 -d / | cut -f 3,4 -d -)

  cat ../by11s.txt | while read i j
     do ((n=i/11))
     tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extract_${n}.tar \
       $(seq $i $j | xargs -I ^ bash -c '{ k=^; printf "${0}-%05.0f_* logs/*_%03.0f_log\n" $k $k ; }' $pfx)
     done &&
  echo $(date) $(hostname) $jobid /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts_\{0..50\}.tar &&
  cd .. &&
  rm -rf $segid
fi