Mercurial > hg > cc > cirrus_home

#!/bin/bash
me=$$
SHARED=/home/shared/ht
cd $SHARED/data/$(hostname)
mkdir -p logs
while read id
do
    echo starting ${id} $(date) >> logs/${me}_log
    unpigz -dp 1 -c /data/common_crawl/CC-MAIN-2019-35/CC-MAIN-${id}.warc.gz|$SHARED/bin/warc.sh ${id} application/pdf 2>> logs/${me}_log
    echo finished ${id} $(date) >> logs/${me}_log
done
author	Henry S. Thompson <ht@inf.ed.ac.uk>
date	Mon, 16 Mar 2020 15:57:23 +0000
parents
children	a82c325e8b32