Mercurial > hg > cc > cirrus_work
changeset 173:10c87f5c704d
add minimal logging and don't return until finished
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Fri, 24 Nov 2023 19:50:12 +0000 |
| parents | bc66c6098e5e |
| children | d88c8d40259a |
| files | bin/getcc_multi.aws |
| diffstat | 1 files changed, 3 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/bin/getcc_multi.aws Wed Nov 15 10:24:32 2023 +0000 +++ b/bin/getcc_multi.aws Fri Nov 24 19:50:12 2023 +0000 @@ -35,6 +35,7 @@ # max_concurrent_requests = 1 # multipart_chunksize = 32MB +echo $(date) start $SEG s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1) mkdir -p $s/orig/warc fgrep -v -f <(cd $s/orig/warc && ls *.warc.gz || :) <(fgrep -w $s warc.paths) > /tmp/hst/$s @@ -56,3 +57,5 @@ sleep 30 } done +while pgrep -a aws |grep -c s3; do sleep 60; done +echo $(date) end $SEG