Mercurial > hg > cc > valhalla

comparison bin/extract.sh @ 0:fdd3f8a16fd4 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
shared scripts on valhalla cluster
author Henry Thompson <ht@markup.co.uk>
date Sat, 14 Mar 2020 11:00:58 +0000
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:fdd3f8a16fd4
1 #!/bin/bash
2 me=$$
3 SHARED=/home/shared/ht
4 cd $SHARED/data/$(hostname)
5 mkdir -p logs
6 while read id
7 do
8 echo starting ${id} $(date) >> logs/${me}_log
9 unpigz -dp 1 -c /data/common_crawl/CC-MAIN-2019-35/CC-MAIN-${id}.warc.gz|$SHARED/bin/warc.sh ${id} application/pdf 2>> logs/${me}_log
10 echo finished ${id} $(date) >> logs/${me}_log
11 done