# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1585072415 0
# Node ID 4c23b1766692a74d7cd9985c9592daec698607b5
# Parent  9676e7ad02e15e31351fb6d2245f3268e4296a03
towards sub-division of resulting tar files

diff -r 9676e7ad02e1 -r 4c23b1766692 bin/doExtract.sh
--- a/bin/doExtract.sh	Tue Mar 24 17:52:52 2020 +0000
+++ b/bin/doExtract.sh	Tue Mar 24 17:53:35 2020 +0000
@@ -7,7 +7,7 @@
 mkdir -p $segid/logs
 cd $segid
 ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \
-parallel --joblog job_${jobid}.log -j 12 -N 1 'id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) ; \
+parallel --joblog job_${jobid}.log -j 18 -N 1 'id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) ; \
     echo starting $id $(date) >> logs/${jobid}_{#}_log ; \
     unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log ; \
     echo finished ${id} $(date) >> logs/${jobid}_{#}_log'
@@ -15,6 +15,7 @@
 echo $(date) $(hostname) $jobid $segid $res
 if [ $res = 0 ]
  then
-  tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar *
+  cat ../by11s.txt | while read 
+  tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar * && 
   echo $(date) $(hostname) $jobid /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar
 fi
diff -r 9676e7ad02e1 -r 4c23b1766692 bin/masterMaster.sh
--- a/bin/masterMaster.sh	Tue Mar 24 17:52:52 2020 +0000
+++ b/bin/masterMaster.sh	Tue Mar 24 17:53:35 2020 +0000
@@ -1,5 +1,5 @@
 #!/bin/bash
 # This runs on login machine to launch the real job (named by $1) on two machines
 echo $(date) Launching master workers for "$@"
-seq 1 -1 0 | parallel --will-cite --joblog master_$1.log -S r1i5n1 -S r1i5n0 -N 1 --workdir /dev/shm --transferfile $1_'{}'.txt $HOME/bin/$1.sh '{}'
+seq 1 -1 0 | parallel --will-cite --joblog master_$1.log -S r1i5n1 -S r1i5n0 -N 1 --workdir /dev/shm --transferfile $1_'{}'.txt --transferfile by11s.txt $HOME/bin/$1.sh '{}'
 echo $(date) "$@" workers done
diff -r 9676e7ad02e1 -r 4c23b1766692 masterJob.sh
--- a/masterJob.sh	Tue Mar 24 17:52:52 2020 +0000
+++ b/masterJob.sh	Tue Mar 24 17:53:35 2020 +0000
@@ -2,7 +2,7 @@
 # Attempt at a master Master job
 # Usage: qsub -v t=taskName masterJob.sh
 # Implies that ~/taskName_{0,1}.txt contain the necessary parameterisation
-#PBS -l select=2:ncpus=2
+#PBS -l select=2:ncpus=36
 #PBS -l place=exclhost
 #PBS -l walltime=08:00:00
 #PBS -V