# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1584545033 0
# Node ID 7a2bc060230d391e739ccfb04f3cc98bdcc28ce3
# Parent  a9a9f6f1832e7b6c85c473b481f2cebfc92bda22
first cut at doing extraction here

diff -r a9a9f6f1832e -r 7a2bc060230d bin/doExtract.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/doExtract.sh	Wed Mar 18 15:23:53 2020 +0000
@@ -0,0 +1,18 @@
+#!/bin/bash
+# Usage extract.sh ccid jobid segid
+ccid=$1
+jobid=$2
+segid=$3
+echo $(date) $(hostname) $jobid $segid
+mkdir -p $segid/logs
+cd $segid
+ls /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-*.warc.gz | \
+parallel -j 12 -N 1 '{
+    id=$(echo {} | cut -f 6 -d / | cut -f 3- -d - | cut -f 1 -d .)
+    echo starting $id $(date) >> logs/${jobid}_{#}_log
+    unpigz -dp 1 -c {} | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/${jobid}_{#}_log
+    echo finished ${id} $(date) >> logs/${jobid}_{#}_log
+   }'
+echo $(date) $(hostname) $jobid $segid $?
+tar -cf /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/extracts.tar *
+echo $(date) $(hostname) $jobid /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/extracts.tar
diff -r a9a9f6f1832e -r 7a2bc060230d bin/extract.sh
--- a/bin/extract.sh	Wed Mar 18 13:42:47 2020 +0000
+++ b/bin/extract.sh	Wed Mar 18 15:23:53 2020 +0000
@@ -1,13 +1,11 @@
-#!/bin/bash
-# Usage extract.sh ccid segid
-ccid=$1
-segid=$2
-@@me=$$
-@@cd $SHARED/data/$(hostname)
-mkdir -p logs
-while read id
-do
-    echo starting ${id} $(date) >> logs/${me}_log
-    unpigz -dp 1 -c /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-${id}.warc.gz|$HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/@@${me}_log
-    echo finished ${id} $(date) >> logs/@@${me}_log
-done
+#!/usr/bin/bash
+# Launch parallel tasks, one per segment from extract_?.txt
+echo $(date) $(hostname)
+h=$(hostname)
+hn=${h##*n}
+head -1 extract_${hn}.txt |\
+ {  read cc
+    tail -n +2 extract_${hn}.txt |\
+    parallel --will-cite  -j 4 -N 1 $HOME/bin/doExtract.sh "$cc" '{#}' '{}'
+ }
+echo $(date) $(hostname) $?