Mercurial > hg > cc > cirrus_home

changeset 8:3b56c2c9d0ee

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
longer run, terser logging
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 24 Feb 2020 12:16:10 +0000
parents 25ca3505b4d7
children 7a93e190c74d
files bin/doPlinks.sh bin/plinks.sh plinksJob.sh
diffstat 3 files changed, 6 insertions(+), 5 deletions(-) [+]
line wrap: on
line diff
--- a/bin/doPlinks.sh	Mon Feb 24 00:44:53 2020 +0000
+++ b/bin/doPlinks.sh	Mon Feb 24 12:16:10 2020 +0000
@@ -11,9 +11,10 @@
 plinks.py $tfn || { echo $(date) $hn aborted job $jn for $tfn \[remember to clean up\] ; rm -f /dev/shm/stopJob ; exit 1 ; }
 echo $(date) $hn tarring $(ls badpdfs_*|wc -l)/$(ls links_*_*|wc -l) results from job $jn for $tfn in $(pwd) 1>&2
 tar -cf /beegfs/common_crawl/CC-MAIN-2019-35/pdfs/links/${tfn}.tar badpdfs_${tfn} links_${tfn}_*
-echo $(pwd) rm * 1>&2
+echo $(date) $(pwd) rm $(ls -lt badpdfs_*)
+echo . . .  $(ls -lt links_*_* | tee >(tail -1 1>&2) | head -1) 1>&2
 rm *
 cd ..
-echo $(pwd) rmdir ${tfn} 1>&2
+echo $(date) $(pwd) rmdir ${tfn} 1>&2
 rmdir ${tfn}
 echo $(date) $hn finished job ${jn} for ${tfn}
--- a/bin/plinks.sh	Mon Feb 24 00:44:53 2020 +0000
+++ b/bin/plinks.sh	Mon Feb 24 12:16:10 2020 +0000
@@ -3,6 +3,6 @@
 echo $(date) $(hostname)
 h=$(hostname)
 hn=${h##*n}
-if [ $hn -eq 0 ]; then echo {001..002}; else echo {003..004}; fi |\
-tr ' ' '\012' |parallel --will-cite -j 71 -N 1 doPlinks.sh ${hn} '{#}' '{}'
+if [ $hn -eq 0 ]; then echo {005..008}; else echo {009..012}; fi |\
+tr ' ' '\012' |parallel --will-cite -j 4 -N 1 doPlinks.sh ${hn} '{#}' '{}'
 echo $(date) $(hostname) $?
--- a/plinksJob.sh	Mon Feb 24 00:44:53 2020 +0000
+++ b/plinksJob.sh	Mon Feb 24 12:16:10 2020 +0000
@@ -11,7 +11,7 @@
 cd ${PBS_O_WORKDIR}
 
 export MPI_SHEPHERD=true
-export MPI_UNBUFFERED_STDIO=true
+#export MPI_UNBUFFERED_STDIO=true not needed anymore since debug logging pruned
 
 mpiexec_mpt -ppn 1 -n 2 bin/plinks.sh 2019-35