# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1700855412 0
# Node ID 10c87f5c704d66c2f3eb1177c64a73dfb769d449
# Parent  bc66c6098e5e7bcef22bfcf7cff621b7c3d4f18b
add minimal logging and don't return until finished

diff -r bc66c6098e5e -r 10c87f5c704d bin/getcc_multi.aws
--- a/bin/getcc_multi.aws	Wed Nov 15 10:24:32 2023 +0000
+++ b/bin/getcc_multi.aws	Fri Nov 24 19:50:12 2023 +0000
@@ -35,6 +35,7 @@
 #     max_concurrent_requests = 1
 #     multipart_chunksize = 32MB
 
+echo $(date) start $SEG
 s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1)
 mkdir -p $s/orig/warc
 fgrep -v -f <(cd $s/orig/warc && ls *.warc.gz || :) <(fgrep -w $s warc.paths) > /tmp/hst/$s
@@ -56,3 +57,5 @@
   sleep 30
   }
 done
+while pgrep -a aws |grep -c s3; do sleep 60; done
+echo $(date) end $SEG