Mercurial > hg > cc > cirrus_work
diff bin/getcc_multi.aws @ 173:10c87f5c704d
add minimal logging and don't return until finished
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Fri, 24 Nov 2023 19:50:12 +0000 |
| parents | 143d2c6d56da |
| children | 6faed9e5d9c9 |
line wrap: on
line diff
--- a/bin/getcc_multi.aws Wed Nov 15 10:24:32 2023 +0000 +++ b/bin/getcc_multi.aws Fri Nov 24 19:50:12 2023 +0000 @@ -35,6 +35,7 @@ # max_concurrent_requests = 1 # multipart_chunksize = 32MB +echo $(date) start $SEG s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1) mkdir -p $s/orig/warc fgrep -v -f <(cd $s/orig/warc && ls *.warc.gz || :) <(fgrep -w $s warc.paths) > /tmp/hst/$s @@ -56,3 +57,5 @@ sleep 30 } done +while pgrep -a aws |grep -c s3; do sleep 60; done +echo $(date) end $SEG