Mercurial > hg > cc > cirrus_home
changeset 25:a82c325e8b32
(none)
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Wed, 18 Mar 2020 11:08:47 +0000 |
| parents | 6fdebccf6492 |
| children | a9a9f6f1832e |
| files | bin/extract.sh bin/test.sh bin/unfold.sh testJob.sh |
| diffstat | 4 files changed, 15 insertions(+), 13 deletions(-) [+] |
line wrap: on
line diff
--- a/bin/extract.sh Wed Mar 18 11:08:23 2020 +0000 +++ b/bin/extract.sh Wed Mar 18 11:08:47 2020 +0000 @@ -1,11 +1,13 @@ #!/bin/bash -me=$$ -SHARED=/home/shared/ht -cd $SHARED/data/$(hostname) +# Usage extract.sh ccid segid +ccid=$1 +segid=$2 +@@me=$$ +@@cd $SHARED/data/$(hostname) mkdir -p logs while read id do echo starting ${id} $(date) >> logs/${me}_log - unpigz -dp 1 -c /data/common_crawl/CC-MAIN-2019-35/CC-MAIN-${id}.warc.gz|$SHARED/bin/warc.sh ${id} application/pdf 2>> logs/${me}_log - echo finished ${id} $(date) >> logs/${me}_log + unpigz -dp 1 -c /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-${id}.warc.gz|$HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/@@${me}_log + echo finished ${id} $(date) >> logs/@@${me}_log done
--- a/bin/test.sh Wed Mar 18 11:08:23 2020 +0000 +++ b/bin/test.sh Wed Mar 18 11:08:47 2020 +0000 @@ -1,5 +1,5 @@ -#!/bin/bash -pwd -parallel --will-cite --nonall -S r1i5n0 -S r1i5n1 'echo $$ $(hostname); n=$(echo $(hostname)|cut -c 6); nohup sleep $((n*5)); echo done $n' -echo pdone - +#!/usr/bin/bash +# test master +echo $(date) $(hostname) "$1" +echo testing... $(cat test_$1.txt) +echo $(date) $(hostname) $?