Mercurial > hg > cc > cirrus_work

annotate results/CC-MAIN-2019-35/warc_lmhx/mcdb/idx/tcb.py @ 342:4759d6ac4625 trim

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
update ROOT, fix deprecated patterns
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 10 Mar 2026 21:17:17 +0000
parents 252cf1263ad0
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
333
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
1 '''For use as callbacks from cdx2tsv.py, e.g.
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
2 uz cdx-00120.gz | fgrep '"lastmod":'| cdx2tsv.py -c tcb /tmp/hst/cby status lastmod
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
3 '''
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
4 import sys, os, bisect
334
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
5 from collections import Counter
333
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
6
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
7 class Year:
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
8 def __init__(self, year, far_end):
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
9 self.year = year
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
10 self.far_end = far_end
334
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
11 self.stats = Counter()
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
12
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
13 def write(self):
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
14 global res_dir
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
15 if self.stats:
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
16 with open("%s/%s"%(res_dir, self.year),"bw") as f:
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
17 for s,c in sorted(list(self.stats.items()),
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
18 key=lambda i:i[1], reverse=True):
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
19 f.write(b'%d\t%d\n'%(c, s))
332
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
20
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21 def init():
334
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
22 global years, nyears, res_dir, far_ends, NLM
332
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 with open("%s/results/CC-MAIN-2019-35/warc_lmhx/years.txt"%os.getenv("HOME"),"r") \
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25 as yf:
333
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
26 y = 1993
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
27 years=[Year(y:=(y+1),int(l)) for l in yf]
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
28
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
29 far_ends = [y.far_end for y in years]
332
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
30
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
31 nyears = len(years)
334
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
32 res_dir = sys.argv.pop(1)
333
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
33 years.append(Year("oob",0))
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
34
334
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
35 years.append(NLM := Year("no_LM",0))
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
36
333
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
37 #print('i',years, nyears, file = sys.stderr)
332
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
38
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
39 def finish():
334
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
40 global years
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
41 for y in years:
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
42 y.write()
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
43
332
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
44
333
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
45 def find_year(lastmod):
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
46 global far_ends
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
47 return years[bisect.bisect_right(far_ends, lastmod)]
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
48
332
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
49 def cbf(status,lastmod):
334
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
50 global NLM
332
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
51 stat = int(status)
334
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
52 if lastmod == 'NA':
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
53 NLM.stats[stat] += 1
332
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
54 else:
334
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
55 lm = int(lastmod)
252cf1263ad0 do merge in python working
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 333
diff changeset
56 find_year(lm).stats[stat]+=1
333
282a35927943 begin rework to do merge in python
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 332
diff changeset
57
332
83450325421f not working yet
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
58