view master/src/wecu/run_sac.sh @ 58:a3edba8dab11

move to right place in tree
author Henry S. Thompson <ht@markup.co.uk>
date Thu, 28 May 2020 09:56:42 +0000
parents master/wecu/run_sac.sh@ac1a20e627a9
children 8332faef25e1
line wrap: on
line source

#!/bin/bash
cores=$1
hosts=$2
wd=$3
shift
shift
shift
rm -f allout

parallel -v \
    --sshloginfile $hosts \
    --retries 3 \
    --transferfile $(which sac_mapper.py|sed 's/sac_/.\/sac_/') \
    --will-cite \
    --jobs $cores \
    --workdir $wd \
    -a input_paths \
    "hostname 1>&2 ; export PYTHONIOENCODING=utf-8; curl -s -N https://commoncrawl.s3.amazonaws.com/{} | unpigz -dp 1 -c | tee >(wc -l 1>&2) | ./sac_mapper.py $* 2>&1" | tee -a allout | grep -v 'Authorized uses only' | \
    sac_reducer.py "$*"