Mercurial > hg > cc > cirrus_work

comparison bin/getccwat.aws @ 395:54304add8adf plus

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
sic
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 30 Jun 2026 15:40:40 +0100
parents
children
comparison
equal deleted inserted replaced
394:fb2f5c598256 395:54304add8adf
1 # courtesy wwaites
2 # Usage: getccwat.aws [-d] <archive, e.g. CC-MAIN-2019-35> <segment ID, e.g. 68> [nthreads]
3 # Single segment, multiple threads
4
5 if [ "$1" = "-d" ]
6 then
7 shift
8 debug=1
9 fi
10
11 ARCHIVE="$1"
12 SEG=$2
13 nthreads=${3:-2}
14
15 wf=wat.paths
16
17 WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz
18
19 mkdir -p $CCC/${ARCHIVE}
20 cd $CCC/${ARCHIVE}
21
22 if [ ! -f $wf ]
23 then
24 curl --retry 4 -s ${WARCS} | gzip -dc > $wf
25 fi
26
27 #export AWS_ACCESS_KEY_ID="AKIAIKBLNO2XNVMWM5JA"
28 #export AWS_SECRET_ACCESS_KEY="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"
29 #export PASSPHRASE="annebooththompson"
30 #export AWS_RETRY_MODE=adaptive
31 #export AWS_MAX_ATTEMPTS=100
32 # Set these in ~/.aws/credentials as follows
33 # [hst]
34 # aws_access_key_id = AKIAIKBLNO2XNVMWM5JA
35 # aws_secret_access_key = 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
36 # And these in ~/.aws/config as follows
37 # [profile hst]
38 # retry_mode = adaptive
39 # max_attempts = 100
40 # s3 =
41 # multipart_threshold = 4GB
42 # max_concurrent_requests = 1
43 # multipart_chunksize = 32MB
44
45 echo $(date) start $SEG
46 s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1)
47 mkdir -p $s/orig/wat
48 fgrep -v -f <(cd $s/orig/wat && ls *.warc.wat.gz || :) <(fgrep -w $s wat.paths) > /tmp/hst/$s
49
50 split -a 2 --numeric-suffixes=1 -n l/$nthreads /tmp/hst/$s /tmp/hst/${s}_
51 printf "%02d\n" $(seq 1 $nthreads) | while read i
52 do
53 cat /tmp/hst/${s}_$i | {
54 printf "thread\t%s\t%s\t%s\n" $i $$ $(ps -o pgid= -p "$$") >> errlog_wat_${SEG}_$i
55 while read f
56 do
57 g=$s/orig/wat/${f##*/}
58 if [ ! -f "$g" ]
59 then
60 if [ "$debug" ]
61 then
62 echo aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors 2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_wat_${SEG}_$i )
63 fi
64 aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors 2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_wat_${SEG}_$i )
65 fi
66 done &
67 sleep 30
68 }
69 done
70 while pgrep -a aws |grep -c "aws s3.*${s}"; do sleep 60; done
71 echo $(date) end $SEG
72