Mercurial > hg > cc > cirrus_home

#!/bin/bash
# Usage extract.sh ccid segid
ccid=$1
segid=$2
@@me=$$
@@cd $SHARED/data/$(hostname)
mkdir -p logs
while read id
do
    echo starting ${id} $(date) >> logs/${me}_log
    unpigz -dp 1 -c /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-${id}.warc.gz|$HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/@@${me}_log
    echo finished ${id} $(date) >> logs/@@${me}_log
done
author	Henry S. Thompson <ht@inf.ed.ac.uk>
date	Wed, 18 Mar 2020 11:08:47 +0000
parents	c858a4d4bd4f
children	7a2bc060230d