Mercurial > hg > cc > cirrus_home
diff bin/extract.sh @ 25:a82c325e8b32
(none)
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Wed, 18 Mar 2020 11:08:47 +0000 |
| parents | c858a4d4bd4f |
| children | 7a2bc060230d |
line wrap: on
line diff
--- a/bin/extract.sh Wed Mar 18 11:08:23 2020 +0000 +++ b/bin/extract.sh Wed Mar 18 11:08:47 2020 +0000 @@ -1,11 +1,13 @@ #!/bin/bash -me=$$ -SHARED=/home/shared/ht -cd $SHARED/data/$(hostname) +# Usage extract.sh ccid segid +ccid=$1 +segid=$2 +@@me=$$ +@@cd $SHARED/data/$(hostname) mkdir -p logs while read id do echo starting ${id} $(date) >> logs/${me}_log - unpigz -dp 1 -c /data/common_crawl/CC-MAIN-2019-35/CC-MAIN-${id}.warc.gz|$SHARED/bin/warc.sh ${id} application/pdf 2>> logs/${me}_log - echo finished ${id} $(date) >> logs/${me}_log + unpigz -dp 1 -c /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-${id}.warc.gz|$HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/@@${me}_log + echo finished ${id} $(date) >> logs/@@${me}_log done