Mercurial > hg > cc > cirrus_home

view bin/plinksRedo.sh @ 79:4b8e4e3d60eb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
track redirects, need to us full crawldiagnostics.warc.gz for "location:" and "Uri:"
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Thu, 07 May 2020 18:47:24 +0100
parents b976a7449d41
children
line wrap: on
line source

#!/usr/bin/bash
module load miniconda/python3
echo $(date) $(hostname)
h=$(hostname)
hn=${h##*n}
cat /beegfs/common_crawl/CC-MAIN-2019-35/pdfs/links/redo_$hn |\
parallel --will-cite -j 48 -N 1 bin/doPlinks.sh ${hn} '{#}' '{}'
echo $(date) $(hostname) $?