Mercurial > hg > cc > cirrus_home
changeset 29:a95a1b31f5e9
try to fix multi-line lossage
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Wed, 18 Mar 2020 21:54:47 +0000 |
| parents | 70c04a452599 |
| children | 1e479e3ea9c4 |
| files | bin/doExtract.sh bin/extract.sh |
| diffstat | 2 files changed, 17 insertions(+), 15 deletions(-) [+] |
line wrap: on
line diff
--- a/bin/doExtract.sh Wed Mar 18 21:52:06 2020 +0000 +++ b/bin/doExtract.sh Wed Mar 18 21:54:47 2020 +0000 @@ -1,18 +1,20 @@ #!/bin/bash -# Usage extract.sh ccid jobid segid +# Usage doExtract.sh ccid jobid segid ccid=$1 -jobid=$2 +export jobid=$2 segid=$3 echo $(date) $(hostname) $jobid $segid mkdir -p $segid/logs cd $segid -ls /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \ -parallel -j 12 -N 1 '{ - id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) - echo starting $id $(date) >> logs/${jobid}_{#}_log - unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log +ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \ +parallel -j 12 -N 1 id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) \; \ + echo starting \$id \$(date) >> logs/${jobid}_{#}_log \; \ + unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log \; \ echo finished ${id} $(date) >> logs/${jobid}_{#}_log - }' -echo $(date) $(hostname) $jobid $segid $? -tar -cf /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/extracts.tar * -echo $(date) $(hostname) $jobid /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/extracts.tar +res=$? +echo $(date) $(hostname) $jobid $segid $res +if [ $res = 0 ] + then + tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar * + echo $(date) $(hostname) $jobid /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar +fi
--- a/bin/extract.sh Wed Mar 18 21:52:06 2020 +0000 +++ b/bin/extract.sh Wed Mar 18 21:54:47 2020 +0000 @@ -1,11 +1,11 @@ #!/usr/bin/bash # Launch parallel tasks, one per segment from extract_?.txt +# First line thereof gives CC identifier echo $(date) $(hostname) h=$(hostname) hn=${h##*n} head -1 extract_${hn}.txt |\ - { read cc - tail -n +2 extract_${hn}.txt |\ - parallel --will-cite -j 4 -N 1 $HOME/bin/doExtract.sh "$cc" '{#}' '{}' - } + read cc \; \ + tail -n +2 extract_${hn}.txt |\ + parallel --will-cite -j 4 -N 1 \$HOME/bin/doExtract.sh "$cc" '{#}' '{}' echo $(date) $(hostname) $?