Mercurial > hg > cc > azure
changeset 43:c2b72d29a3ee
update to use _timedWhich.py
| author | Henry S. Thompson <ht@markup.co.uk> |
|---|---|
| date | Fri, 30 Nov 2018 18:37:40 +0000 |
| parents | 1d776e96c16a |
| children | 1342f6669352 |
| files | workers/bin/timedWhich.sh |
| diffstat | 1 files changed, 2 insertions(+), 3 deletions(-) [+] |
line wrap: on
line diff
--- a/workers/bin/timedWhich.sh Fri Nov 30 15:41:02 2018 +0000 +++ b/workers/bin/timedWhich.sh Fri Nov 30 18:37:40 2018 +0000 @@ -27,8 +27,7 @@ m=0 set -o pipefail until if [ $((m+=1)) -gt 5 ]; then echo "tried to read $1 5 times w/o success, giving up" 1>&2; return 1; fi && \ - curl -s --insecure -o - "$1"| zcat | fgrep msgtype=response | \ -{ egrep -i '"WARC-Target-URI":"https?:' || : ;} | doit + curl -s --insecure -o - "$1"| zcat | doit do # try to avoid lockstep retries echo ${PIPESTATUS[@]} 1>&2 @@ -60,7 +59,7 @@ { echo -n "# $(date) $id " >> $log tee >(wc -l >> $log) |\ - parallel --round-robin --pipe --block-size 2M -j $wp "_timedWhich.sh {#} > $res/$cci.{#} 2>>$res/errs{#}" || echo "ppfailed $? ${PIPESTATUS[@]}" 1>&2 + parallel --round-robin --pipe --block-size 2M -j $wp "_timedWhich.py {#} > $res/$cci.{#} 2>>$res/errs{#}" || echo "ppfailed $? ${PIPESTATUS[@]}" 1>&2 } while read s id do