# courtesy wwaites
# Usage: getcrt.aws [archive, e.g. CC-MAIN-2019-35] [segment ID, e.g. 68]
# Single segment, single thread, robotstxt
ARCHIVE="$1"
export SEG="$2"

wf="${3-robotstxt.paths}"

WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz

mkdir -p $CCC/${ARCHIVE}
cd $CCC/${ARCHIVE}

if [ ! -f $wf ]
then
 curl -s ${WARCS} | gzip -dc > $wf
fi

s=$(grep -Eow "[0-9]*\.$SEG" $wf | head -1)
mkdir -p $s/orig/robotstxt
fgrep -w $s $wf | while read c; do \
  g=${c##*/}
  if [ ! -f "$s/orig/robotstxt/$g" ]
  then
    echo "$s	$g	$c"
  fi; done |\
parallel --colsep '\t' --will-cite -j 16 \
    "aws s3 cp s3://commoncrawl/'{3}'  '{1}/orig/robotstxt/{2}' --only-show-errors 2> >( { echo \$(date +%D:%T) '{3}' ; cat ; } >>rerrlog_\${SEG})"