Mercurial > hg > cc > azure

comparison master/wecu/run_mapreduce.sh @ 57:ac1a20e627a9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
from lukasz git repo 2020-05-26 (see ~/src/wecu), then editted, sac not quite working yet
author Henry S. Thompson <ht@markup.co.uk>
date Wed, 27 May 2020 20:54:34 +0000
parents
children
comparison
equal deleted inserted replaced
56:8ce6a81e2bb4 57:ac1a20e627a9
1 cores=`cat cores.txt`
2 if [ "$1" -ne "-1" ];
3 then
4 cores=$1
5 fi
6
7 time parallel \
8 --sshloginfile hosts \
9 --transferfile "$2" \
10 --transferfile "$3" \
11 --will-cite \
12 --jobs $cores \
13 --retries 3 \
14 --workdir $PWD \
15 -a input_paths \
16 "curl -s -N 'https://commoncrawl.s3.amazonaws.com/{}' | unpigz -dp 1 -c | $2" 2>&1 | grep -v 'Authorized uses only' | \
17 sort | \
18 eval $3
19