Mercurial > hg > cc > cirrus_work

changeset 374:ecb73c2f03d1 plus

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add -d flag
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 21 Apr 2026 09:28:47 +0100
parents cf25927ef965
children a74a3335e0d2
files bin/getcc_multi.aws
diffstat 1 files changed, 11 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/bin/getcc_multi.aws	Mon Apr 20 18:19:43 2026 +0100
+++ b/bin/getcc_multi.aws	Tue Apr 21 09:28:47 2026 +0100
@@ -1,6 +1,13 @@
 # courtesy wwaites
 # Usage: getcc.aws <archive, e.g. CC-MAIN-2019-35> <segment ID, e.g. 68> [nthreads]
 # Single segment, multiple threads
+
+if [ "$1" = "-d" ]
+then
+  shift
+  debug=1
+fi
+
 ARCHIVE="$1"
 SEG=$2
 nthreads=${3:-2}
@@ -50,6 +57,10 @@
     g=$s/orig/warc/${f##*/}
     if [ ! -f "$g" ]
     then
+      if [ "$debug" ]
+      then
+	echo aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors  2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_${SEG}_$i )
+      fi
       aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors  2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_${SEG}_$i )
     fi
   done &