Mercurial > hg > cc > cirrus_home

comparison bin/extract.sh @ 25:a82c325e8b32

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
(none)
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 18 Mar 2020 11:08:47 +0000
parents c858a4d4bd4f
children 7a2bc060230d
comparison
equal deleted inserted replaced
24:6fdebccf6492 25:a82c325e8b32
1 #!/bin/bash 1 #!/bin/bash
2 me=$$ 2 # Usage extract.sh ccid segid
3 SHARED=/home/shared/ht 3 ccid=$1
4 cd $SHARED/data/$(hostname) 4 segid=$2
5 @@me=$$
6 @@cd $SHARED/data/$(hostname)
5 mkdir -p logs 7 mkdir -p logs
6 while read id 8 while read id
7 do 9 do
8 echo starting ${id} $(date) >> logs/${me}_log 10 echo starting ${id} $(date) >> logs/${me}_log
9 unpigz -dp 1 -c /data/common_crawl/CC-MAIN-2019-35/CC-MAIN-${id}.warc.gz|$SHARED/bin/warc.sh ${id} application/pdf 2>> logs/${me}_log 11 unpigz -dp 1 -c /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-${id}.warc.gz|$HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/@@${me}_log
10 echo finished ${id} $(date) >> logs/${me}_log 12 echo finished ${id} $(date) >> logs/@@${me}_log
11 done 13 done