Mercurial > hg > cc > cirrus_work

changeset 375:a74a3335e0d2 plus

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
now linked from $CCC, updated to use $CCC and better logging
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 22 Apr 2026 15:09:30 +0100
parents ecb73c2f03d1
children af18b5074464
files bin/getcd.aws bin/getrt.aws
diffstat 2 files changed, 56 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/getcd.aws	Wed Apr 22 15:09:30 2026 +0100
@@ -0,0 +1,28 @@
+# courtesy wwaites
+# Usage: getcrt.aws [archive, e.g. CC-MAIN-2019-35] [segment ID, e.g. 68]
+# Single segment, single thread, crawldiagnostics
+ARCHIVE="$1"
+export SEG="$2"
+
+wf="${3-non200responses.paths}"
+
+WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz
+
+mkdir -p $CCC/${ARCHIVE}
+cd $CCC/${ARCHIVE}
+
+if [ ! -f $wf ]
+then
+ curl -s ${WARCS} | gzip -dc > $wf
+fi
+
+s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1)
+mkdir -p $s/orig/crawldiagnostics
+fgrep -w $s $wf | while read c; do \
+  g=${c##*/}
+  if [ ! -f "$s/orig/crawldiagnostics/$g" ]
+  then
+    echo "$s	$g	$c"
+  fi; done |\
+parallel --colsep '\t' --will-cite -j 16 \
+    "aws s3 cp s3://commoncrawl/'{3}'  '{1}/orig/crawldiagnostics/{2}' --only-show-errors 2> >( { echo \$(date +%D:%T) '{3}' ; cat ; } >>cerrlog_\${SEG})"
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/getrt.aws	Wed Apr 22 15:09:30 2026 +0100
@@ -0,0 +1,28 @@
+# courtesy wwaites
+# Usage: getcrt.aws [archive, e.g. CC-MAIN-2019-35] [segment ID, e.g. 68]
+# Single segment, single thread, robotstxt
+ARCHIVE="$1"
+export SEG="$2"
+
+wf="${3-robotstxt.paths}"
+
+WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz
+
+mkdir -p $CCC/${ARCHIVE}
+cd $CCC/${ARCHIVE}
+
+if [ ! -f $wf ]
+then
+ curl -s ${WARCS} | gzip -dc > $wf
+fi
+
+s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1)
+mkdir -p $s/orig/robotstxt
+fgrep -w $s $wf | while read c; do \
+  g=${c##*/}
+  if [ ! -f "$s/orig/robotstxt/$g" ]
+  then
+    echo "$s	$g	$c"
+  fi; done |\
+parallel --colsep '\t' --will-cite -j 16 \
+    "aws s3 cp s3://commoncrawl/'{3}'  '{1}/orig/robotstxt/{2}' --only-show-errors 2> >( { echo \$(date +%D:%T) '{3}' ; cat ; } >>rerrlog_\${SEG})"