Mercurial > hg > cc > cirrus_home

comparison bin/extract.sh @ 27:7a2bc060230d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first cut at doing extraction here
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 18 Mar 2020 15:23:53 +0000
parents a82c325e8b32
children a95a1b31f5e9
comparison
equal deleted inserted replaced
26:a9a9f6f1832e 27:7a2bc060230d
1 #!/bin/bash 1 #!/usr/bin/bash
2 # Usage extract.sh ccid segid 2 # Launch parallel tasks, one per segment from extract_?.txt
3 ccid=$1 3 echo $(date) $(hostname)
4 segid=$2 4 h=$(hostname)
5 @@me=$$ 5 hn=${h##*n}
6 @@cd $SHARED/data/$(hostname) 6 head -1 extract_${hn}.txt |\
7 mkdir -p logs 7 { read cc
8 while read id 8 tail -n +2 extract_${hn}.txt |\
9 do 9 parallel --will-cite -j 4 -N 1 $HOME/bin/doExtract.sh "$cc" '{#}' '{}'
10 echo starting ${id} $(date) >> logs/${me}_log 10 }
11 unpigz -dp 1 -c /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-${id}.warc.gz|$HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/@@${me}_log 11 echo $(date) $(hostname) $?
12 echo finished ${id} $(date) >> logs/@@${me}_log
13 done