Mercurial > hg > cc > cirrus_work
comparison bin/getcd.aws @ 375:a74a3335e0d2 plus
now linked from $CCC,
updated to use $CCC and better logging
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Wed, 22 Apr 2026 15:09:30 +0100 |
| parents | |
| children |
comparison
equal
deleted
inserted
replaced
| 374:ecb73c2f03d1 | 375:a74a3335e0d2 |
|---|---|
| 1 # courtesy wwaites | |
| 2 # Usage: getcrt.aws [archive, e.g. CC-MAIN-2019-35] [segment ID, e.g. 68] | |
| 3 # Single segment, single thread, crawldiagnostics | |
| 4 ARCHIVE="$1" | |
| 5 export SEG="$2" | |
| 6 | |
| 7 wf="${3-non200responses.paths}" | |
| 8 | |
| 9 WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz | |
| 10 | |
| 11 mkdir -p $CCC/${ARCHIVE} | |
| 12 cd $CCC/${ARCHIVE} | |
| 13 | |
| 14 if [ ! -f $wf ] | |
| 15 then | |
| 16 curl -s ${WARCS} | gzip -dc > $wf | |
| 17 fi | |
| 18 | |
| 19 s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1) | |
| 20 mkdir -p $s/orig/crawldiagnostics | |
| 21 fgrep -w $s $wf | while read c; do \ | |
| 22 g=${c##*/} | |
| 23 if [ ! -f "$s/orig/crawldiagnostics/$g" ] | |
| 24 then | |
| 25 echo "$s $g $c" | |
| 26 fi; done |\ | |
| 27 parallel --colsep '\t' --will-cite -j 16 \ | |
| 28 "aws s3 cp s3://commoncrawl/'{3}' '{1}/orig/crawldiagnostics/{2}' --only-show-errors 2> >( { echo \$(date +%D:%T) '{3}' ; cat ; } >>cerrlog_\${SEG})" |