Mercurial > hg > cc > cirrus_work

comparison bin/getcc_multi.aws @ 371:875724011bc3 plus

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
use $CCC
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 20 Apr 2026 18:18:32 +0100
parents 6faed9e5d9c9
children ecb73c2f03d1
comparison
equal deleted inserted replaced
370:c5b4f3e9f909 371:875724011bc3
7 7
8 wf=warc.paths 8 wf=warc.paths
9 9
10 WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz 10 WARCS=https://data.commoncrawl.org/crawl-data/${ARCHIVE}/${wf}.gz
11 11
12 mkdir -p /beegfs/common_crawl/${ARCHIVE} 12 mkdir -p $CCC/${ARCHIVE}
13 cd /beegfs/common_crawl/${ARCHIVE} 13 cd $CCC/${ARCHIVE}
14 14
15 if [ ! -f $wf ] 15 if [ ! -f $wf ]
16 then 16 then
17 curl --retry 4 -s ${WARCS} | gzip -dc > $wf 17 curl --retry 4 -s ${WARCS} | gzip -dc > $wf
18 fi 18 fi