Mercurial > hg > cc > cirrus_home

#!/bin/bash
# Usage doExtract.sh ccid jobid segid
ccid=$1
export jobid=$2
segid=$3
echo $(date) $(hostname) $jobid $segid
mkdir -p $segid/logs
cd $segid
ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \
parallel -j 12 -N 1 id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) \; \
    echo starting \$id \$(date) >> logs/${jobid}_{#}_log \; \
    unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log \; \
    echo finished ${id} $(date) >> logs/${jobid}_{#}_log
res=$?
echo $(date) $(hostname) $jobid $segid $res
if [ $res = 0 ]
 then
  tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar *
  echo $(date) $(hostname) $jobid /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar
fi
author	Henry S. Thompson <ht@inf.ed.ac.uk>
date	Wed, 18 Mar 2020 21:54:47 +0000
parents	7a2bc060230d
children	1e479e3ea9c4