view bin/masterMaster.sh @ 190:f2bf736c2d40

working, with issues: 1) warc.export.cdx.path is not set 2) mime property isn't getting through 3) lastMod isn't showing up
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 24 Sep 2024 17:08:05 +0100
parents e5d5958bf3fe
children
line wrap: on
line source

#!/bin/bash
# This runs on login machine to launch the real task (named by $1) on multiple machines
# Any further args are passed to $task.sh
echo $(date) Launching master workers for "$@" from $(hostname)
task=$1
shift
srun $HOME/bin/$task.sh "$@"
echo $(date) $task "$@" workers done

#nodes=($(scontrol show hostname "$SLURM_JOB_NODELIST"))
#seq 1 -1 0 | $HOME/gentoo/usr/bin/parallel --will-cite --joblog master_$task.log -S "ssh -o StrictHostKeyChecking=accept-new ${nodes[1]}" -S "ssh -o StrictHostKeyChecking=accept-new  ${nodes[0]}" -N 1 --workdir /dev/shm --transferfile ${task}_'{}'.txt $* $HOME/bin/$task.sh '{}' "$@"