Mercurial > hg > cc > cirrus_work

annotate bin/getccwat.aws @ 395:54304add8adf plus

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
sic
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 30 Jun 2026 15:40:40 +0100
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
395
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 # courtesy wwaites
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 # Usage: getccwat.aws [-d] <archive, e.g. CC-MAIN-2019-35> <segment ID, e.g. 68> [nthreads]
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 # Single segment, multiple threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 if [ "$1" = "-d" ]
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 then
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 shift
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 debug=1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 fi
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 ARCHIVE="$1"
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 SEG=$2
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13 nthreads=${3:-2}
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15 wf=wat.paths
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17 WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19 mkdir -p $CCC/${ARCHIVE}
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
20 cd $CCC/${ARCHIVE}
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22 if [ ! -f $wf ]
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23 then
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 curl --retry 4 -s ${WARCS} | gzip -dc > $wf
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25 fi
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 #export AWS_ACCESS_KEY_ID="AKIAIKBLNO2XNVMWM5JA"
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28 #export AWS_SECRET_ACCESS_KEY="WaH4SAhsrqqJ/GLo/jkw+u9ER1ny05e1W45sSgYu"
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
29 #export PASSPHRASE="annebooththompson"
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
30 #export AWS_RETRY_MODE=adaptive
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
31 #export AWS_MAX_ATTEMPTS=100
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
32 # Set these in ~/.aws/credentials as follows
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
33 # [hst]
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
34 # aws_access_key_id = AKIAIKBLNO2XNVMWM5JA
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
35 # aws_secret_access_key = WaH4SAhsrqqJ/GLo/jkw+u9ER1ny05e1W45sSgYu
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
36 # And these in ~/.aws/config as follows
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
37 # [profile hst]
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
38 # retry_mode = adaptive
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
39 # max_attempts = 100
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
40 # s3 =
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
41 # multipart_threshold = 4GB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
42 # max_concurrent_requests = 1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
43 # multipart_chunksize = 32MB
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
44
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
45 echo $(date) start $SEG
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
46 s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
47 mkdir -p $s/orig/wat
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
48 fgrep -v -f <(cd $s/orig/wat && ls *.warc.wat.gz || :) <(fgrep -w $s wat.paths) > /tmp/hst/$s
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
49
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
50 split -a 2 --numeric-suffixes=1 -n l/$nthreads /tmp/hst/$s /tmp/hst/${s}_
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
51 printf "%02d\n" $(seq 1 $nthreads) | while read i
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
52 do
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
53 cat /tmp/hst/${s}_$i | {
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
54 printf "thread\t%s\t%s\t%s\n" $i $$ $(ps -o pgid= -p "$$") >> errlog_wat_${SEG}_$i
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
55 while read f
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
56 do
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
57 g=$s/orig/wat/${f##*/}
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
58 if [ ! -f "$g" ]
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
59 then
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
60 if [ "$debug" ]
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
61 then
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
62 echo aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors 2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_wat_${SEG}_$i )
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
63 fi
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
64 aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors 2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_wat_${SEG}_$i )
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
65 fi
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
66 done &
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
67 sleep 30
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
68 }
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
69 done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
70 while pgrep -a aws |grep -c "aws s3.*${s}"; do sleep 60; done
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
71 echo $(date) end $SEG
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
72