Mercurial > hg > cc > cirrus_home

changeset 27:7a2bc060230d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first cut at doing extraction here
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 18 Mar 2020 15:23:53 +0000
parents a9a9f6f1832e
children 70c04a452599
files bin/doExtract.sh bin/extract.sh
diffstat 2 files changed, 29 insertions(+), 13 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/doExtract.sh	Wed Mar 18 15:23:53 2020 +0000
@@ -0,0 +1,18 @@
+#!/bin/bash
+# Usage extract.sh ccid jobid segid
+ccid=$1
+jobid=$2
+segid=$3
+echo $(date) $(hostname) $jobid $segid
+mkdir -p $segid/logs
+cd $segid
+ls /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \
+parallel -j 12 -N 1 '{
+    id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .)
+    echo starting $id $(date) >> logs/${jobid}_{#}_log
+    unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log
+    echo finished ${id} $(date) >> logs/${jobid}_{#}_log
+   }'
+echo $(date) $(hostname) $jobid $segid $?
+tar -cf /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/extracts.tar *
+echo $(date) $(hostname) $jobid /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/extracts.tar
--- a/bin/extract.sh	Wed Mar 18 13:42:47 2020 +0000
+++ b/bin/extract.sh	Wed Mar 18 15:23:53 2020 +0000
@@ -1,13 +1,11 @@
-#!/bin/bash
-# Usage extract.sh ccid segid
-ccid=$1
-segid=$2
-@@me=$$
-@@cd $SHARED/data/$(hostname)
-mkdir -p logs
-while read id
-do
-    echo starting ${id} $(date) >> logs/${me}_log
-    unpigz -dp 1 -c /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-${id}.warc.gz|$HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/@@${me}_log
-    echo finished ${id} $(date) >> logs/@@${me}_log
-done
+#!/usr/bin/bash
+# Launch parallel tasks, one per segment from extract_?.txt
+echo $(date) $(hostname)
+h=$(hostname)
+hn=${h##*n}
+head -1 extract_${hn}.txt |\
+ {  read cc
+    tail -n +2 extract_${hn}.txt |\
+    parallel --will-cite  -j 4 -N 1 $HOME/bin/doExtract.sh "$cc" '{#}' '{}'
+ }
+echo $(date) $(hostname) $?