view bin/plinks.sh @ 190:f2bf736c2d40

working, with issues: 1) warc.export.cdx.path is not set 2) mime property isn't getting through 3) lastMod isn't showing up
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 24 Sep 2024 17:08:05 +0100
parents 29263ba42361
children
line wrap: on
line source

#!/usr/bin/bash
module load miniconda/python3
echo $(date) $(hostname)
h=$(hostname)
hn=${h##*n}
if [ $hn -eq 0 ]
then seq --format="%03.0f" $1 $2
else seq --format="%03.0f" $3 $4
fi |\
parallel --will-cite -j 48 -N 1 bin/doPlinks.sh ${hn} '{#}' '{}'
echo $(date) $(hostname) $?