Mercurial > hg > cc > cirrus_home

annotate bin/plinks.py @ 6:0f494c76a887

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
refactor to address tarred-up pdfs
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Sun, 23 Feb 2020 16:48:34 +0000
parents a28d731977da
children 25ca3505b4d7
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
65a56c0d1c1f bolting the barn door...
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/lustre/sw/miniconda3/bin/python3
65a56c0d1c1f bolting the barn door...
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 import sys,pdfx,traceback
1
a4b0359456bc switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob
Henry Thompson <ht@markup.co.uk>
parents: 0
diff changeset
3 from os import path
a4b0359456bc switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob
Henry Thompson <ht@markup.co.uk>
parents: 0
diff changeset
4
a4b0359456bc switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob
Henry Thompson <ht@markup.co.uk>
parents: 0
diff changeset
5 def run(file):
0
65a56c0d1c1f bolting the barn door...
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 global pdf
1
a4b0359456bc switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob
Henry Thompson <ht@markup.co.uk>
parents: 0
diff changeset
7 pdf=pdfx.PDFx(file)
a4b0359456bc switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob
Henry Thompson <ht@markup.co.uk>
parents: 0
diff changeset
8 return pdf.get_references_as_dict()
0
65a56c0d1c1f bolting the barn door...
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9
6
0f494c76a887 refactor to address tarred-up pdfs
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 5
diff changeset
10 tarnum=sys.argv[1]
0f494c76a887 refactor to address tarred-up pdfs
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 5
diff changeset
11 with open('badpdfs_%s'%tarnum,'w') as bf:
1
a4b0359456bc switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob
Henry Thompson <ht@markup.co.uk>
parents: 0
diff changeset
12 for l in sys.stdin:
6
0f494c76a887 refactor to address tarred-up pdfs
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 5
diff changeset
13 (fno,f)=l.split()
1
a4b0359456bc switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob
Henry Thompson <ht@markup.co.uk>
parents: 0
diff changeset
14 try:
2
Henry Thompson <ht@markup.co.uk>
parents: 1
diff changeset
15 links=run(f)
4
462179da7dc2 try harder not to write empty links files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 2
diff changeset
16 if bool(links) and (links.get('scrape',False) or
462179da7dc2 try harder not to write empty links files
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 2
diff changeset
17 links.get('annot',False)):
6
0f494c76a887 refactor to address tarred-up pdfs
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 5
diff changeset
18 with open('links_%s_%s'%(tarnum,fno),'w') as of:
3
cbd13beb0922 only create links file if there are some
Henry Thompson <ht@markup.co.uk>
parents: 2
diff changeset
19 for k in links.keys():
cbd13beb0922 only create links file if there are some
Henry Thompson <ht@markup.co.uk>
parents: 2
diff changeset
20 for l in links[k]:
cbd13beb0922 only create links file if there are some
Henry Thompson <ht@markup.co.uk>
parents: 2
diff changeset
21 print("%s\t%s"%(k,l),file=of)
1
a4b0359456bc switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob
Henry Thompson <ht@markup.co.uk>
parents: 0
diff changeset
22 except Exception as e:
6
0f494c76a887 refactor to address tarred-up pdfs
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 5
diff changeset
23 if str(e)=='Unexpected EOF':
0f494c76a887 refactor to address tarred-up pdfs
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 5
diff changeset
24 print("%s\t%s\t%s"%(tarnum,fno,e),file=bf)
0f494c76a887 refactor to address tarred-up pdfs
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 5
diff changeset
25 else:
0f494c76a887 refactor to address tarred-up pdfs
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 5
diff changeset
26 traceback.print_exc(file=bf)
0f494c76a887 refactor to address tarred-up pdfs
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 5
diff changeset
27
1
a4b0359456bc switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob
Henry Thompson <ht@markup.co.uk>
parents: 0
diff changeset
28 if (path.exists('/dev/shm/stopJob')):
6
0f494c76a887 refactor to address tarred-up pdfs
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 5
diff changeset
29 print("Quiting early: %s %s"%(tarnum,fno),file=sys.stderr)
1
a4b0359456bc switch to file loop inside python, assume file index integer in pipe as well as filename, check /dev/shm/stopJob
Henry Thompson <ht@markup.co.uk>
parents: 0
diff changeset
30 exit(1)