|
332
|
1 import sys,os
|
|
|
2
|
|
|
3 def init():
|
|
|
4 global i, year, count, cur_status, years, nyears, res_dir, ff
|
|
|
5
|
|
|
6 i = 0
|
|
|
7 year = 1995
|
|
|
8 count = 0
|
|
|
9 cur_status = 0
|
|
|
10
|
|
|
11 with open("%s/results/CC-MAIN-2019-35/warc_lmhx/years.txt"%os.getenv("HOME"),"r") \
|
|
|
12 as yf:
|
|
|
13 years=[int(l) for l in yf]
|
|
|
14
|
|
|
15 nyears = len(years)
|
|
|
16 ff = [None] * (nyears + 1)
|
|
|
17 res_dir = sys.argv.pop(1)
|
|
|
18 ff[nyears] = open("%s/oob"%res_dir,"bw")
|
|
|
19
|
|
|
20 def finish():
|
|
|
21 global ff, i, count, cur_status
|
|
|
22 ff[i].write(b'%d\t%d\n'%(count, cur_status))
|
|
|
23
|
|
|
24 def cbf(status,lastmod):
|
|
|
25 global cur_status, count, i, year, nyears
|
|
|
26 stat = int(status)
|
|
|
27 lm = int(lastmod)
|
|
|
28 print(i, year, cur_status, count, stat, lm, file = sys.stderr)
|
|
|
29 while (not year == 0) and (lm > years[i]):
|
|
|
30 # change of year
|
|
|
31 if count != 0:
|
|
|
32 ff[i].write(b'%s\t%d\n'%(count, cur_status))
|
|
|
33 if i < nyears:
|
|
|
34 i += 1
|
|
|
35 year += 1
|
|
|
36 count = 0
|
|
|
37 cur_status = 0
|
|
|
38 if i == nyears:
|
|
|
39 year = 0
|
|
|
40 break
|
|
|
41 if not ff[i]:
|
|
|
42 ff[i] = open("%s/%s"%(res_dir, year),"bw")
|
|
|
43 if stat == cur_status:
|
|
|
44 count += 1
|
|
|
45 else:
|
|
|
46 ff[i].write(b'%d\t%d\n'%(count, cur_status))
|
|
|
47 cur_status = stat
|
|
|
48 count = 1
|
|
|
49
|
|
|
50
|
