Mercurial > hg > cc > cirrus_work

annotate bin/getcc_multi.aws @ 394:fb2f5c598256 plus

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
tweak usage
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 30 Jun 2026 15:40:08 +0100
parents ecb73c2f03d1
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
161
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
1 # courtesy wwaites
394
fb2f5c598256 tweak usage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 374
diff changeset
2 # Usage: getcc_multi.aws [-d] <archive, e.g. CC-MAIN-2019-35> <segment ID, e.g. 68> [nthreads]
161
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
3 # Single segment, multiple threads
374
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
4
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
5 if [ "$1" = "-d" ]
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
6 then
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
7 shift
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
8 debug=1
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
9 fi
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
10
158
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 ARCHIVE="$1"
161
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
12 SEG=$2
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
13 nthreads=${3:-2}
158
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15 wf=warc.paths
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17 WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18
371
875724011bc3 use $CCC
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 210
diff changeset
19 mkdir -p $CCC/${ARCHIVE}
875724011bc3 use $CCC
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 210
diff changeset
20 cd $CCC/${ARCHIVE}
158
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22 if [ ! -f $wf ]
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23 then
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 curl --retry 4 -s ${WARCS} | gzip -dc > $wf
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25 fi
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 #export AWS_ACCESS_KEY_ID="AKIAIKBLNO2XNVMWM5JA"
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28 #export AWS_SECRET_ACCESS_KEY="WaH4SAhsrqqJ/GLo/jkw+u9ER1ny05e1W45sSgYu"
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
29 #export PASSPHRASE="annebooththompson"
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
30 #export AWS_RETRY_MODE=adaptive
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
31 #export AWS_MAX_ATTEMPTS=100
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
32 # Set these in ~/.aws/credentials as follows
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
33 # [hst]
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
34 # aws_access_key_id = AKIAIKBLNO2XNVMWM5JA
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
35 # aws_secret_access_key = WaH4SAhsrqqJ/GLo/jkw+u9ER1ny05e1W45sSgYu
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
36 # And these in ~/.aws/config as follows
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
37 # [profile hst]
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
38 # retry_mode = adaptive
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
39 # max_attempts = 100
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
40 # s3 =
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
41 # multipart_threshold = 4GB
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
42 # max_concurrent_requests = 1
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
43 # multipart_chunksize = 32MB
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
44
173
10c87f5c704d add minimal logging and don't return until finished
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 171
diff changeset
45 echo $(date) start $SEG
161
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
46 s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1)
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
47 mkdir -p $s/orig/warc
171
143d2c6d56da cross-language confusion :-)
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 163
diff changeset
48 fgrep -v -f <(cd $s/orig/warc && ls *.warc.gz || :) <(fgrep -w $s warc.paths) > /tmp/hst/$s
161
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
49
210
6faed9e5d9c9 use 2-digit suffixes,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 173
diff changeset
50 split -a 2 --numeric-suffixes=1 -n l/$nthreads /tmp/hst/$s /tmp/hst/${s}_
6faed9e5d9c9 use 2-digit suffixes,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 173
diff changeset
51 printf "%02d\n" $(seq 1 $nthreads) | while read i
158
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
52 do
163
348f4a31228f bug-fix wrt 1st time,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 161
diff changeset
53 cat /tmp/hst/${s}_$i | {
348f4a31228f bug-fix wrt 1st time,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 161
diff changeset
54 printf "thread\t%s\t%s\t%s\n" $i $$ $(ps -o pgid= -p "$$") >> errlog_${SEG}_$i
348f4a31228f bug-fix wrt 1st time,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 161
diff changeset
55 while read f
161
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
56 do
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
57 g=$s/orig/warc/${f##*/}
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
58 if [ ! -f "$g" ]
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
59 then
374
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
60 if [ "$debug" ]
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
61 then
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
62 echo aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors 2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_${SEG}_$i )
ecb73c2f03d1 add -d flag
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 371
diff changeset
63 fi
161
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
64 aws s3 cp s3://commoncrawl/$f $g $(cat debug) --only-show-errors 2> >( { echo $(date +%D:%T) $f ; cat ; } >>errlog_${SEG}_$i )
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
65 fi
d0dbfefd6fc0 forget parallel, just do (default 2) parallel single threads
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 158
diff changeset
66 done &
163
348f4a31228f bug-fix wrt 1st time,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 161
diff changeset
67 sleep 30
348f4a31228f bug-fix wrt 1st time,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 161
diff changeset
68 }
158
5d1c3359e210 resurrect parallel fetch
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
69 done
210
6faed9e5d9c9 use 2-digit suffixes,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 173
diff changeset
70 while pgrep -a aws |grep -c "aws s3.*${s}"; do sleep 60; done
173
10c87f5c704d add minimal logging and don't return until finished
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 171
diff changeset
71 echo $(date) end $SEG