# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1776866970 -3600
# Node ID a74a3335e0d2b20b9c413c62fa14ae2c6bab10ce
# Parent  ecb73c2f03d1d9d99af55398b716e02784f2ca54
now linked from $CCC,
updated to use $CCC and better logging

diff -r ecb73c2f03d1 -r a74a3335e0d2 bin/getcd.aws
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/getcd.aws	Wed Apr 22 15:09:30 2026 +0100
@@ -0,0 +1,28 @@
+# courtesy wwaites
+# Usage: getcrt.aws [archive, e.g. CC-MAIN-2019-35] [segment ID, e.g. 68]
+# Single segment, single thread, crawldiagnostics
+ARCHIVE="$1"
+export SEG="$2"
+
+wf="${3-non200responses.paths}"
+
+WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz
+
+mkdir -p $CCC/${ARCHIVE}
+cd $CCC/${ARCHIVE}
+
+if [ ! -f $wf ]
+then
+ curl -s ${WARCS} | gzip -dc > $wf
+fi
+
+s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1)
+mkdir -p $s/orig/crawldiagnostics
+fgrep -w $s $wf | while read c; do \
+  g=${c##*/}
+  if [ ! -f "$s/orig/crawldiagnostics/$g" ]
+  then
+    echo "$s	$g	$c"
+  fi; done |\
+parallel --colsep '\t' --will-cite -j 16 \
+    "aws s3 cp s3://commoncrawl/'{3}'  '{1}/orig/crawldiagnostics/{2}' --only-show-errors 2> >( { echo \$(date +%D:%T) '{3}' ; cat ; } >>cerrlog_\${SEG})"
diff -r ecb73c2f03d1 -r a74a3335e0d2 bin/getrt.aws
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/getrt.aws	Wed Apr 22 15:09:30 2026 +0100
@@ -0,0 +1,28 @@
+# courtesy wwaites
+# Usage: getcrt.aws [archive, e.g. CC-MAIN-2019-35] [segment ID, e.g. 68]
+# Single segment, single thread, robotstxt
+ARCHIVE="$1"
+export SEG="$2"
+
+wf="${3-robotstxt.paths}"
+
+WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz
+
+mkdir -p $CCC/${ARCHIVE}
+cd $CCC/${ARCHIVE}
+
+if [ ! -f $wf ]
+then
+ curl -s ${WARCS} | gzip -dc > $wf
+fi
+
+s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1)
+mkdir -p $s/orig/robotstxt
+fgrep -w $s $wf | while read c; do \
+  g=${c##*/}
+  if [ ! -f "$s/orig/robotstxt/$g" ]
+  then
+    echo "$s	$g	$c"
+  fi; done |\
+parallel --colsep '\t' --will-cite -j 16 \
+    "aws s3 cp s3://commoncrawl/'{3}'  '{1}/orig/robotstxt/{2}' --only-show-errors 2> >( { echo \$(date +%D:%T) '{3}' ; cat ; } >>rerrlog_\${SEG})"