# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1584374243 0
# Node ID c858a4d4bd4fa9e8770a6c65a0a08ce368be32f5
# Parent  6662a353379a150ee2afe5a917cfea47e2d51e2c
copied from valhalla/bin

diff -r 6662a353379a -r c858a4d4bd4f bin/extract.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/extract.sh	Mon Mar 16 15:57:23 2020 +0000
@@ -0,0 +1,11 @@
+#!/bin/bash
+me=$$
+SHARED=/home/shared/ht
+cd $SHARED/data/$(hostname)
+mkdir -p logs
+while read id
+do
+    echo starting ${id} $(date) >> logs/${me}_log
+    unpigz -dp 1 -c /data/common_crawl/CC-MAIN-2019-35/CC-MAIN-${id}.warc.gz|$SHARED/bin/warc.sh ${id} application/pdf 2>> logs/${me}_log
+    echo finished ${id} $(date) >> logs/${me}_log
+done