Mercurial > hg > cc > cirrus_home

annotate bin/extract.sh @ 19:c858a4d4bd4f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
copied from valhalla/bin
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 16 Mar 2020 15:57:23 +0000
parents
children a82c325e8b32
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
19
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/bin/bash
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 me=$$
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 SHARED=/home/shared/ht
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 cd $SHARED/data/$(hostname)
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 mkdir -p logs
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 while read id
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 do
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 echo starting ${id} $(date) >> logs/${me}_log
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 unpigz -dp 1 -c /data/common_crawl/CC-MAIN-2019-35/CC-MAIN-${id}.warc.gz|$SHARED/bin/warc.sh ${id} application/pdf 2>> logs/${me}_log
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10 echo finished ${id} $(date) >> logs/${me}_log
c858a4d4bd4f copied from valhalla/bin
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 done