--- a/bin/extract.sh	Wed Mar 18 13:42:47 2020 +0000
+++ b/bin/extract.sh	Wed Mar 18 15:23:53 2020 +0000
@@ -1,13 +1,11 @@
-#!/bin/bash
-# Usage extract.sh ccid segid
-ccid=$1
-segid=$2
-@@me=$$
-@@cd $SHARED/data/$(hostname)
-mkdir -p logs
-while read id
-do
-    echo starting ${id} $(date) >> logs/${me}_log
-    unpigz -dp 1 -c /beegfs/common-crawl/CC-MAIN-${ccid}/${segid}/CC-MAIN-${id}.warc.gz|$HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> logs/@@${me}_log
-    echo finished ${id} $(date) >> logs/@@${me}_log
-done
+#!/usr/bin/bash
+# Launch parallel tasks, one per segment from extract_?.txt
+echo $(date) $(hostname)
+h=$(hostname)
+hn=${h##*n}
+head -1 extract_${hn}.txt |\
+ {  read cc
+    tail -n +2 extract_${hn}.txt |\
+    parallel --will-cite  -j 4 -N 1 $HOME/bin/doExtract.sh "$cc" '{#}' '{}'
+ }
+echo $(date) $(hostname) $?