Mercurial > hg > cc > azure
changeset 8:5db6015689a2
slightly updated, 4 W only, versions of old test4.sh and subcount.sh
| author | Henry S. Thompson <ht@markup.co.uk> |
|---|---|
| date | Mon, 01 Oct 2018 18:29:39 +0000 |
| parents | a7637c994964 |
| children | 55e953e5c66f |
| files | workers/bin/count1.sh workers/bin/test1.sh |
| diffstat | 2 files changed, 74 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/workers/bin/count1.sh Mon Oct 01 18:29:39 2018 +0000 @@ -0,0 +1,7 @@ +#!/bin/bash +echo "# $ID" +jq '.Envelope|.["WARC-Header-Metadata"]["WARC-Target-URI"]'|cut -f 1 -d ':' + + + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/workers/bin/test1.sh Mon Oct 01 18:29:39 2018 +0000 @@ -0,0 +1,67 @@ +#!/bin/bash +# Test script to split CC WAT files across threads +# to count http: vs. https: +# Usage: [echo file file_id] | test1.sh id home +# remove >>errs once tested +#set -e -o pipefail +echo $$ > test1.pid +proc=$1 +res=res$proc +home=$2 +touch .running +function lrand { +# cheap bad little random number generator +echo $(( 1 + ($(openssl rand 1 | od -d | head -1 | tr -s ' ' | cut -f 2 -d ' ') % $1))) +} +function tryRead { +m=0 +set -o pipefail +until if [ $((m+=1)) -gt 5 ]; then echo "tried to read $1 5 times w/o success, giving up" 1>&2; return 1; fi && \ + curl -s --insecure -o - "$1"| zcat | fgrep msgtype=response | \ +{ egrep -i '"WARC-Target-URI":"https?:' || : ;} > $2 + do + # try to avoid lockstep retries + echo ${PIPESTATUS[@]} 1>&2 + sleep $(lrand 10) + echo $(date) $2 retry number $m 1>&2 +done +set +o pipefail +} +trap "{ + #set -e -o pipefail + cd $res + ln -s ../nohup.cc . + tar -czhf - * | \ + ssh -o StrictHostKeyChecking=no -q $home \"{ cd data/jobs/ + mkdir -p test1.$proc + cd test1.$proc + tar -xzf - ; } 2>>errs.1\" + cd + rm -rf $res + ( sleep 5 ; rm nohup.cc ) & + }" EXIT +mkdir -p $res +log=$res/log +# Don't all start at once +sleep $(lrand 60) +echo \# $(date) > $log +pRes=0 +while read s id +do + url="https://commoncrawl.s3.amazonaws.com/$s" + export ID=$id + echo $(date) "running |$@|$id|" >> $log + # Experimental retry loop + tryRead "$url" crawl$id + if [ -s crawl$id ] + then + echo \# $id $(wc -l crawl$id) >> $log + parallel --round-robin --pipe -j 4 "count1.sh >> $res/{#} 2>>$res/errs{#}" < crawl$id || echo "ppfailed $? ${PIPESTATUS[@]}" 1>&2 + else + echo "crawl$id empty" 1>&2 + fi + rm crawl$id +done < ifile.txt 2>> $res/errs || pRes=$? +( cd $res && fgrep -h -v \# [1-9]* ) | sort | uniq -c | tr -d \" > $res/tots +echo \# $(date) main loop exit code=$pRes >> $log +rm .running