Mercurial > hg > cc > cirrus_home

comparison bin/doExtract.sh @ 35:ec99b2d1d2fc

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
sync up filenames and log names, attempt to save more smaller tar files
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Thu, 26 Mar 2020 12:24:30 +0000
parents 4c23b1766692
children e912ed51146a
comparison
equal deleted inserted replaced
34:e34cda3e7483 35:ec99b2d1d2fc
6 echo $(date) $(hostname) $jobid $segid 6 echo $(date) $(hostname) $jobid $segid
7 mkdir -p $segid/logs 7 mkdir -p $segid/logs
8 cd $segid 8 cd $segid
9 ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \ 9 ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \
10 parallel --joblog job_${jobid}.log -j 18 -N 1 'id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) ; \ 10 parallel --joblog job_${jobid}.log -j 18 -N 1 'id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) ; \
11 echo starting $id $(date) >> logs/${jobid}_{#}_log ; \ 11 fid=$(printf "%03.0f" $(({#} - 1))) ; \
12 unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log ; \ 12 lf=logs/${jobid}_${fid}_log ; \
13 echo finished ${id} $(date) >> logs/${jobid}_{#}_log' 13 echo starting $id $(date) >> $lf ; \
14 unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> $lf ; \
15 echo finished ${id} $(date) >> $lf'
14 res=$? 16 res=$?
15 echo $(date) $(hostname) $jobid $segid $res 17 echo $(date) $(hostname) $jobid $segid $res
16 if [ $res = 0 ] 18 if [ $res = 0 ]
17 then 19 then
18 cat ../by11s.txt | while read 20 pfx=$(ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*-00000.warc.gz |\
19 tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar * && 21 cut -f 6 -d / | cut -f 3,4 -d -)
20 echo $(date) $(hostname) $jobid /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar 22
23 cat ../by11s.txt | while read i j
24 do ((n=i/11))
25 tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extract_${n}.tar.gz \
26 $(seq $i $j | xargs -I ^ bash -c '{ k=^; printf "${pfx}-%05.0f_* logs/?_%03.0f_log\n" $k $k ; }')
27 done &&
28 echo $(date) $(hostname) $jobid /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts_\{0..${n}\}.tar
21 fi 29 fi