Mercurial > hg > cc > cirrus_home

comparison bin/doExtract.sh @ 33:4c23b1766692

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
towards sub-division of resulting tar files
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 24 Mar 2020 17:53:35 +0000
parents 1e479e3ea9c4
children ec99b2d1d2fc
comparison
equal deleted inserted replaced
32:9676e7ad02e1 33:4c23b1766692
5 segid=$3 5 segid=$3
6 echo $(date) $(hostname) $jobid $segid 6 echo $(date) $(hostname) $jobid $segid
7 mkdir -p $segid/logs 7 mkdir -p $segid/logs
8 cd $segid 8 cd $segid
9 ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \ 9 ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \
10 parallel --joblog job_${jobid}.log -j 12 -N 1 'id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) ; \ 10 parallel --joblog job_${jobid}.log -j 18 -N 1 'id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) ; \
11 echo starting $id $(date) >> logs/${jobid}_{#}_log ; \ 11 echo starting $id $(date) >> logs/${jobid}_{#}_log ; \
12 unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log ; \ 12 unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log ; \
13 echo finished ${id} $(date) >> logs/${jobid}_{#}_log' 13 echo finished ${id} $(date) >> logs/${jobid}_{#}_log'
14 res=$? 14 res=$?
15 echo $(date) $(hostname) $jobid $segid $res 15 echo $(date) $(hostname) $jobid $segid $res
16 if [ $res = 0 ] 16 if [ $res = 0 ]
17 then 17 then
18 tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar * 18 cat ../by11s.txt | while read
19 tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar * &&
19 echo $(date) $(hostname) $jobid /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar 20 echo $(date) $(hostname) $jobid /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar
20 fi 21 fi