Mercurial > hg > cc > cirrus_home
diff bin/doExtract.sh @ 27:7a2bc060230d
first cut at doing extraction here
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Wed, 18 Mar 2020 15:23:53 +0000 |
| parents | |
| children | a95a1b31f5e9 |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/doExtract.sh Wed Mar 18 15:23:53 2020 +0000 @@ -0,0 +1,18 @@ +#!/bin/bash +# Usage extract.sh ccid jobid segid +ccid=$1 +jobid=$2 +segid=$3 +echo $(date) $(hostname) $jobid $segid +mkdir -p $segid/logs +cd $segid +ls /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \ +parallel -j 12 -N 1 '{ + id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) + echo starting $id $(date) >> logs/${jobid}_{#}_log + unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log + echo finished ${id} $(date) >> logs/${jobid}_{#}_log + }' +echo $(date) $(hostname) $jobid $segid $? +tar -cf /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/extracts.tar * +echo $(date) $(hostname) $jobid /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/extracts.tar