Mercurial > hg > cc > cirrus_home
changeset 30:1e479e3ea9c4
better quoting
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Thu, 19 Mar 2020 13:13:02 +0000 |
| parents | a95a1b31f5e9 |
| children | 6f28ac7d5255 |
| files | bin/doExtract.sh bin/extract.sh |
| diffstat | 2 files changed, 6 insertions(+), 6 deletions(-) [+] |
line wrap: on
line diff
--- a/bin/doExtract.sh Wed Mar 18 21:54:47 2020 +0000 +++ b/bin/doExtract.sh Thu Mar 19 13:13:02 2020 +0000 @@ -7,10 +7,10 @@ mkdir -p $segid/logs cd $segid ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \ -parallel -j 12 -N 1 id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) \; \ - echo starting \$id \$(date) >> logs/${jobid}_{#}_log \; \ - unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log \; \ - echo finished ${id} $(date) >> logs/${jobid}_{#}_log +parallel --joblog job_${jobid}.log -j 12 -N 1 'id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) ; \ + echo starting $id $(date) >> logs/${jobid}_{#}_log ; \ + unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log ; \ + echo finished ${id} $(date) >> logs/${jobid}_{#}_log' res=$? echo $(date) $(hostname) $jobid $segid $res if [ $res = 0 ]
--- a/bin/extract.sh Wed Mar 18 21:54:47 2020 +0000 +++ b/bin/extract.sh Thu Mar 19 13:13:02 2020 +0000 @@ -5,7 +5,7 @@ h=$(hostname) hn=${h##*n} head -1 extract_${hn}.txt |\ - read cc \; \ + { read cc ; \ tail -n +2 extract_${hn}.txt |\ - parallel --will-cite -j 4 -N 1 \$HOME/bin/doExtract.sh "$cc" '{#}' '{}' + parallel --joblog topjob.log --will-cite -j 4 -N 1 \$HOME/bin/doExtract.sh "$cc" '{#}' '{}' ; } echo $(date) $(hostname) $?