Mercurial > hg > cc > cirrus_home

comparison bin/doExtract.sh @ 27:7a2bc060230d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first cut at doing extraction here
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 18 Mar 2020 15:23:53 +0000
parents
children a95a1b31f5e9
comparison
equal deleted inserted replaced
26:a9a9f6f1832e 27:7a2bc060230d
1 #!/bin/bash
2 # Usage extract.sh ccid jobid segid
3 ccid=$1
4 jobid=$2
5 segid=$3
6 echo $(date) $(hostname) $jobid $segid
7 mkdir -p $segid/logs
8 cd $segid
9 ls /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \
10 parallel -j 12 -N 1 '{
11 id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .)
12 echo starting $id $(date) >> logs/${jobid}_{#}_log
13 unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log
14 echo finished ${id} $(date) >> logs/${jobid}_{#}_log
15 }'
16 echo $(date) $(hostname) $jobid $segid $?
17 tar -cf /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/extracts.tar *
18 echo $(date) $(hostname) $jobid /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/extracts.tar