Mercurial > hg > cc > cirrus_home

annotate bin/doExtract.sh @ 29:a95a1b31f5e9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
try to fix multi-line lossage
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 18 Mar 2020 21:54:47 +0000
parents 7a2bc060230d
children 1e479e3ea9c4
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
27
7a2bc060230d first cut at doing extraction here
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/bin/bash
29
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
2 # Usage doExtract.sh ccid jobid segid
27
7a2bc060230d first cut at doing extraction here
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 ccid=$1
29
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
4 export jobid=$2
27
7a2bc060230d first cut at doing extraction here
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 segid=$3
7a2bc060230d first cut at doing extraction here
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 echo $(date) $(hostname) $jobid $segid
7a2bc060230d first cut at doing extraction here
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 mkdir -p $segid/logs
7a2bc060230d first cut at doing extraction here
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 cd $segid
29
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
9 ls /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
10 parallel -j 12 -N 1 id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .) \; \
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
11 echo starting \$id \$(date) >> logs/${jobid}_{#}_log \; \
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
12 unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log \; \
27
7a2bc060230d first cut at doing extraction here
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13 echo finished ${id} $(date) >> logs/${jobid}_{#}_log
29
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
14 res=$?
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
15 echo $(date) $(hostname) $jobid $segid $res
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
16 if [ $res = 0 ]
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
17 then
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
18 tar -cf /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar *
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
19 echo $(date) $(hostname) $jobid /beegfs/common_crawl/CC-MAIN-${ccid}/${segid}/extracts.tar
a95a1b31f5e9 try to fix multi-line lossage
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 27
diff changeset
20 fi