Mercurial > hg > cc > cirrus_work
annotate bin/sort_date.py @ 102:e606c609f813
reinstate better check to start queuing,
check for missed match leading to potential runaway
author | Henry S. Thompson <ht@inf.ed.ac.uk> |
---|---|
date | Tue, 12 Sep 2023 12:14:04 +0100 |
parents | 135a8c56dcc2 |
children | fc9a045c872b |
rev | line source |
---|---|
73
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
1 #!/usr/bin/python3 |
97
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
2 '''Process output of lmh_warc [original 2-column version] |
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
3 Usage: <(uz ....warc.gz | fgrep $'\t'|sed "/GMT$/s/\([^ ]\)GMT$/\1 GMT/") |
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
4 ''' |
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
5 |
73
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
6 # Assumes you have used grep -v $'\t' on input for speed |
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
7 # Recommended to also sed '/GMT$/s/\([^ ]\)GMT$/\1 GMT/' |
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
8 # to fix a common 'bad' timestamp (~ .2% of inputs) |
97
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
9 |
73
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
10 import email.utils |
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
11 import sys |
85
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
12 from urllib.parse import urlsplit, quote, unquote |
83
d92bd8527718
use surt instead of trying to create index term by hand
Henry Thompson <ht@markup.co.uk>
parents:
82
diff
changeset
|
13 from surt import surt |
85
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
14 |
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
15 import re, codecs |
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
16 |
88
49faf679d7df
final keystroke fixes, recurse and decimal www stripping
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
86
diff
changeset
|
17 WPAT = re.compile('(,www\\d*)+\\)') |
49faf679d7df
final keystroke fixes, recurse and decimal www stripping
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
86
diff
changeset
|
18 |
77
bf09a1d80d7b
make CC's own sorting explicit
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
76
diff
changeset
|
19 # Thanks to https://stackoverflow.com/a/8776871 |
bf09a1d80d7b
make CC's own sorting explicit
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
76
diff
changeset
|
20 import locale |
bf09a1d80d7b
make CC's own sorting explicit
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
76
diff
changeset
|
21 from functools import cmp_to_key |
76
eeef811f734d
handle corner cases with final . and initial www..+
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
75
diff
changeset
|
22 |
85
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
23 def percent_encode(ude): |
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
24 #print(ude.object,ude.object[ude.start:ude.end]) |
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
25 return (''.join('%%%X'%c for c in ude.object[ude.start:ude.end]), |
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
26 ude.end) |
75
177f7df2bf46
handle %-encoded utf-8 as idna
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
73
diff
changeset
|
27 |
85
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
28 codecs.register_error('percent',percent_encode) |
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
29 |
86
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
30 # From RFC-3986: |
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
31 # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" |
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
32 # sub-delims = "!" / "$" / "&" / "'" / "(" / ")" |
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
33 # / "*" / "+" / "," / ";" / "=" |
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
34 # But # _is_ escaped in Java surt results |
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
35 # and additionally " \ : < = > ? \ ^ _ ` { | } are not |
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
36 |
88
49faf679d7df
final keystroke fixes, recurse and decimal www stripping
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
86
diff
changeset
|
37 # Note also that although quote already does _not_ quote - . / _ ~ |
97
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
38 # they are included below as that's what we find in surt.surt 0.3.1 |
88
49faf679d7df
final keystroke fixes, recurse and decimal www stripping
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
86
diff
changeset
|
39 |
49faf679d7df
final keystroke fixes, recurse and decimal www stripping
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
86
diff
changeset
|
40 # Also, Java surt strips _all_ leading 'www\d*.', |
86
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
41 # where python3 surt only strips the first one. |
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
42 |
85
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
43 def cdx_key(uristring): |
86
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
44 _surt = quote(unquote(surt(uristring), |
3a2ae6057242
handle double .www, more keep-me chars
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
85
diff
changeset
|
45 errors='percent'), |
88
49faf679d7df
final keystroke fixes, recurse and decimal www stripping
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
86
diff
changeset
|
46 safe='''!"$&'()*+,-./:;<=>?@[\]^_`{|}~''' # ' |
49faf679d7df
final keystroke fixes, recurse and decimal www stripping
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
86
diff
changeset
|
47 ).lower() |
49faf679d7df
final keystroke fixes, recurse and decimal www stripping
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
86
diff
changeset
|
48 return WPAT.sub(')',_surt) |
73
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
49 |
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
50 def keyed(l): |
85
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
51 uri, cc_stamp, dateTime = l.split('\t',2) |
77
bf09a1d80d7b
make CC's own sorting explicit
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
76
diff
changeset
|
52 #print('ul',uri,file=sys.stderr) |
73
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
53 try: |
76
eeef811f734d
handle corner cases with final . and initial www..+
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
75
diff
changeset
|
54 try: |
eeef811f734d
handle corner cases with final . and initial www..+
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
75
diff
changeset
|
55 epoch = email.utils.parsedate_to_datetime(dateTime).timestamp() |
eeef811f734d
handle corner cases with final . and initial www..+
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
75
diff
changeset
|
56 except OverflowError: |
eeef811f734d
handle corner cases with final . and initial www..+
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
75
diff
changeset
|
57 epoch = 32535215999.0 |
97
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
58 return ((cdx_key(uri), cc_stamp, uri), epoch) |
75
177f7df2bf46
handle %-encoded utf-8 as idna
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
73
diff
changeset
|
59 except (TypeError,IndexError,ValueError) as e: |
73
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
60 print(dateTime.rstrip(),e,sep='\t',file=sys.stderr) |
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
61 return |
e8c667bf8965
compute timestamps, key and sort lmh lines
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
62 |
97
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
63 fstr = sys.argv[1] |
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
64 |
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
65 with open(fstr,"r") as ff: |
85
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
66 # crucial that the following is done _after_ the file is opened |
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
67 # with the default (utf-8) locale! |
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
68 locale.setlocale(locale.LC_ALL, "C") |
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
69 ctk=cmp_to_key(locale.strcoll) |
97
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
70 for key, ts in sorted((kk for l in ff if (kk:=keyed(l)) is not None), |
85
1daa8e444cfe
work-around for weird handling of %-encoding in Java impl. of SURT
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
84
diff
changeset
|
71 key=lambda x:x[0]): |
97
135a8c56dcc2
include full URI in output
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
88
diff
changeset
|
72 print(key[0],key[1],key[2],ts,sep='\t') |
77
bf09a1d80d7b
make CC's own sorting explicit
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
76
diff
changeset
|
73 |