Mercurial > hg > cc > cirrus_work

comparison bin/getcc_multi.aws @ 374:ecb73c2f03d1 plus

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add -d flag
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 21 Apr 2026 09:28:47 +0100
parents 875724011bc3
children fb2f5c598256
comparison
equal deleted inserted replaced
373:cf25927ef965 374:ecb73c2f03d1
1 # courtesy wwaites 1 # courtesy wwaites
2 # Usage: getcc.aws <archive, e.g. CC-MAIN-2019-35> <segment ID, e.g. 68> [nthreads] 2 # Usage: getcc.aws <archive, e.g. CC-MAIN-2019-35> <segment ID, e.g. 68> [nthreads]
3 # Single segment, multiple threads 3 # Single segment, multiple threads
4
5 if [ "$1" = "-d" ]
6 then
7 shift
8 debug=1
9 fi
10
4 ARCHIVE="$1" 11 ARCHIVE="$1"
5 SEG=$2 12 SEG=$2
6 nthreads=${3:-2} 13 nthreads=${3:-2}
7 14
8 wf=warc.paths 15 wf=warc.paths
48 while read f 55 while read f
49 do 56 do
50 g=$s/orig/warc/${f##*/} 57 g=$s/orig/warc/${f##*/}
51 if [ ! -f "$g" ] 58 if [ ! -f "$g" ]
52 then 59 then
60 if [ "$debug" ]
61 then
62 echo aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors 2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_${SEG}_$i )
63 fi
53 aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors 2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_${SEG}_$i ) 64 aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors 2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_${SEG}_$i )
54 fi 65 fi
55 done & 66 done &
56 sleep 30 67 sleep 30
57 } 68 }