Mercurial > hg > cc > cirrus_work

annotate bin/getcd.aws @ 396:10813bab9072 plus tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added test 9 for wat that pretty-prints json content
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 30 Jun 2026 16:50:56 +0100
parents a74a3335e0d2
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
375
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 # courtesy wwaites
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 # Usage: getcrt.aws [archive, e.g. CC-MAIN-2019-35] [segment ID, e.g. 68]
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 # Single segment, single thread, crawldiagnostics
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 ARCHIVE="$1"
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 export SEG="$2"
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 wf="${3-non200responses.paths}"
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 mkdir -p $CCC/${ARCHIVE}
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 cd $CCC/${ARCHIVE}
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14 if [ ! -f $wf ]
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15 then
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16 curl -s ${WARCS} | gzip -dc > $wf
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17 fi
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19 s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1)
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
20 mkdir -p $s/orig/crawldiagnostics
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21 fgrep -w $s $wf | while read c; do \
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22 g=${c##*/}
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23 if [ ! -f "$s/orig/crawldiagnostics/$g" ]
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 then
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25 echo "$s $g $c"
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26 fi; done |\
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 parallel --colsep '\t' --will-cite -j 16 \
a74a3335e0d2 now linked from $CCC,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28 "aws s3 cp s3://commoncrawl/'{3}' '{1}/orig/crawldiagnostics/{2}' --only-show-errors 2> >( { echo \$(date +%D:%T) '{3}' ; cat ; } >>cerrlog_\${SEG})"