Mercurial > hg > cc > cirrus_home
changeset 33:4c23b1766692
towards sub-division of resulting tar files
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Tue, 24 Mar 2020 17:53:35 +0000 |
| parents | 9676e7ad02e1 |
| children | e34cda3e7483 |
| files | bin/doExtract.sh bin/masterMaster.sh masterJob.sh |
| diffstat | 3 files changed, 5 insertions(+), 4 deletions(-) [+] |
line wrap: on
line diff
--- a/bin/doExtract.sh Tue Mar 24 17:52:52 2020 +0000 +++ b/bin/doExtract.sh Tue Mar 24 17:53:35 2020 +0000 @@ -7,7 +7,7 @@ mkdir -p $segid/logs cd $segid ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \ -parallel --joblog job_${jobid}.log -j 12 -N 1 'id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) ; \ +parallel --joblog job_${jobid}.log -j 18 -N 1 'id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) ; \ echo starting $id $(date) >> logs/${jobid}_{#}_log ; \ unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log ; \ echo finished ${id} $(date) >> logs/${jobid}_{#}_log' @@ -15,6 +15,7 @@ echo $(date) $(hostname) $jobid $segid $res if [ $res = 0 ] then - tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar * + cat ../by11s.txt | while read + tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar * && echo $(date) $(hostname) $jobid /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar fi
--- a/bin/masterMaster.sh Tue Mar 24 17:52:52 2020 +0000 +++ b/bin/masterMaster.sh Tue Mar 24 17:53:35 2020 +0000 @@ -1,5 +1,5 @@ #!/bin/bash # This runs on login machine to launch the real job (named by $1) on two machines echo $(date) Launching master workers for "$@" -seq 1 -1 0 | parallel --will-cite --joblog master_$1.log -S r1i5n1 -S r1i5n0 -N 1 --workdir /dev/shm --transferfile $1_'{}'.txt $HOME/bin/$1.sh '{}' +seq 1 -1 0 | parallel --will-cite --joblog master_$1.log -S r1i5n1 -S r1i5n0 -N 1 --workdir /dev/shm --transferfile $1_'{}'.txt --transferfile by11s.txt $HOME/bin/$1.sh '{}' echo $(date) "$@" workers done
--- a/masterJob.sh Tue Mar 24 17:52:52 2020 +0000 +++ b/masterJob.sh Tue Mar 24 17:53:35 2020 +0000 @@ -2,7 +2,7 @@ # Attempt at a master Master job # Usage: qsub -v t=taskName masterJob.sh # Implies that ~/taskName_{0,1}.txt contain the necessary parameterisation -#PBS -l select=2:ncpus=2 +#PBS -l select=2:ncpus=36 #PBS -l place=exclhost #PBS -l walltime=08:00:00 #PBS -V