Mercurial > hg > cc > cirrus_work

changeset 40:316495371bbc

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add lots more debugging output, reinstate -t for thread allocation
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 05 Jul 2023 15:12:07 +0100
parents 8661062a50b1
children fa43c318749b
files bin/_runme.sh
diffstat 1 files changed, 39 insertions(+), 5 deletions(-) [+]
line wrap: on
line diff
--- a/bin/_runme.sh	Wed Jul 05 15:09:57 2023 +0100
+++ b/bin/_runme.sh	Wed Jul 05 15:12:07 2023 +0100
@@ -1,6 +1,6 @@
 #!/bin/bash
 # This runs on the compute nodes...
-# Args: wd [-b CMDS] [-i input] CMDS
+# Args: wd [-d] [-t nthreads] [-b CMDS] [-i input] CMDS
 # See ug4/azure/notes.txt for documentation
 
 N=$SLURM_JOB_NUM_NODES
@@ -14,10 +14,24 @@
 cd "$1"
 shift
 
-threadsPerTask=2
+if [ "$1" = "-d" ]
+then
+  shift
+  debug=1
+fi
+
+if [ "$1" = "-t" ]
+then
+  shift
+  threadsPerTask=$1
+  shift
+else
+  threadsPerTask=2
+fi
+
 pjobs=$((c / $threadsPerTask))
 
-echo $(date) task $n.$task on $nodename:$N.$node start $(pwd) 1>&2
+echo $(date) task $n.$task on $nodename:$N.$node start $pjobs jobs 1>&2
 
 PATH=$W/$USER/bin:$W/shared/bin:$PATH
 export task PATH n
@@ -34,6 +48,10 @@
  shift
  input="$1"
  shift
+ if [ "$debug" ]
+ then
+     echo $(date) task $n.$task input "|$input|"
+ fi
 fi
 
 export cmd="$1"
@@ -41,15 +59,31 @@
 
 doit () {
  arg="$1"
- echo $(date) start $task $PARALLEL_SEQ $arg
+ echo $(date) start $task $PARALLEL_SEQ "|$cmd|$arg|"
  eval "$cmd"
  echo $(date) end $task $PARALLEL_SEQ
 }
 
 export -f doit
 
+if [ "$debug" ]
+then
+ echo $(date) task $n.$task cmd "|$cmd|" doit $(type doit)
+fi
+
 eval "$input" | \
-   parallel -j $pjobs doit '{}'
+   if [ "$debug" ]
+   then
+     tee >(cat 1>&2)
+   else
+     cat
+   fi | \
+   if [ $pjobs -le 1 ]
+   then
+    xargs -I ^ -n 1 bash -c "doit ^"
+   else
+    parallel --line-buffer -j $pjobs doit '{}'
+   fi
 
 echo $(date) task $n.$task on $nodename:$N.$node end 1>&2