# HG changeset patch
# User Henry Thompson <ht@markup.co.uk>
# Date 1582061615 0
# Node ID a4b0359456bcc7784fe43b7b9b0661a453046ad3
# Parent  65a56c0d1c1fdb8d54407df4a760295d82db9a60
switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob

diff -r 65a56c0d1c1f -r a4b0359456bc bin/doPlinks.sh
--- a/bin/doPlinks.sh	Tue Feb 18 13:15:05 2020 +0000
+++ b/bin/doPlinks.sh	Tue Feb 18 21:33:35 2020 +0000
@@ -1,15 +1,14 @@
 #!/usr/bin/bash
-me=$1
-mine=0
-mkdir /dev/shm/x
-while read f
-do
-    if plinks.py $f > /dev/shm/x/links_${me}_${mine} 2>/dev/null
-    then
-	((mine+=1))
-    else
-	echo $f >> /dev/shm/x/badpdfs_$me
-	rm -f /dev/shm/x/links_${me}_${mine}
-    fi
-done
+mkdir -p /dev/shm/x
+plinks.py $1
+# while read f
+# do
+#     if plinks.py $f > /dev/shm/x/links_${me}_${mine} 2>/dev/null
+#     then
+# 	((mine+=1))
+#     else
+# 	echo $f >> /dev/shm/x/badpdfs_$me
+# 	rm -f /dev/shm/x/links_${me}_${mine}
+#     fi
+# done
 rsync -a /dev/shm/x/ links
diff -r 65a56c0d1c1f -r a4b0359456bc bin/plinks.py
--- a/bin/plinks.py	Tue Feb 18 13:15:05 2020 +0000
+++ b/bin/plinks.py	Tue Feb 18 21:33:35 2020 +0000
@@ -1,24 +1,24 @@
 #!/lustre/sw/miniconda3/bin/python3
 import sys,pdfx,traceback
-def run():
+from os import path
+
+def run(file):
   global pdf
-  try:
-    pdf=pdfx.PDFx(sys.argv[1])
-    links=pdf.get_references_as_dict()
-  except:
-    traceback.print_exc()
-    print("\nFailed: %s"%sys.argv[1],file=sys.stderr)
-    exit(1)
-  for k in links.keys():
-    for l in links[k]:
-         print("%s\t%s"%(k,l))
+  pdf=pdfx.PDFx(file)
+  return pdf.get_references_as_dict()
 
-if sys.argv[1]=='-t':
-  import timeit
-  sys.argv.pop(1)
-  n=sys.argv[1]
-  sys.argv.pop(1)
-  print(timeit.timeit("run()",number=int(n),
-                      setup="from __main__ import run"),file=sys.stderr)
-else:
-  run()
+me=sys.argv[1]
+with open('/dev/shm/x/badpdfs_%s'%me,'w') as bf:
+  for l in sys.stdin:
+    (fno,f)=l.rstrip().split()
+    try:
+      links=run(file)
+      with open('/dev/shm/x/links_%s'%fno,'w') as of:
+        for k in links.keys():
+          for l in links[k]:
+            print("%s\t%s\t%s"%(k,l),file=of)
+    except Exception as e:
+      print("%s\t%s\t%s"%(fno,e),file=bf)
+    if (path.exists('/dev/shm/stopJob')):
+      print("Quiting early: %s %s"%(me,fno),file=sys.stderr)
+      exit(1)