Mercurial > hg > cc > cirrus_home

annotate bin/extract.sh @ 26:a9a9f6f1832e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
finally hacked something that works
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 18 Mar 2020 13:42:47 +0000
parents a82c325e8b32
children 7a2bc060230d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
19
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/bin/bash
25
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 19
diff changeset
2 # Usage extract.sh ccid segid
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 19
diff changeset
3 ccid=$1
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 19
diff changeset
4 segid=$2
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 19
diff changeset
5 @@me=$$
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 19
diff changeset
6 @@cd $SHARED/data/$(hostname)
19
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 mkdir -p logs
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 while read id
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 do
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10 echo starting ${id} $(date) >> logs/${me}_log
25
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 19
diff changeset
11 unpigz -dp 1 -c /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-${id}.warc.gz|$HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/@@${me}_log
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 19
diff changeset
12 echo finished ${id} $(date) >> logs/@@${me}_log
19
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13 done