Mercurial > hg > cc > cirrus_work
comparison results/CC-MAIN-2019-35/warc_lmhx/mcdb/idx/tcb.py @ 332:83450325421f trim
not working yet
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Sat, 28 Feb 2026 14:48:32 +0000 |
| parents | |
| children | 282a35927943 |
comparison
equal
deleted
inserted
replaced
| 331:191127a9a663 | 332:83450325421f |
|---|---|
| 1 import sys,os | |
| 2 | |
| 3 def init(): | |
| 4 global i, year, count, cur_status, years, nyears, res_dir, ff | |
| 5 | |
| 6 i = 0 | |
| 7 year = 1995 | |
| 8 count = 0 | |
| 9 cur_status = 0 | |
| 10 | |
| 11 with open("%s/results/CC-MAIN-2019-35/warc_lmhx/years.txt"%os.getenv("HOME"),"r") \ | |
| 12 as yf: | |
| 13 years=[int(l) for l in yf] | |
| 14 | |
| 15 nyears = len(years) | |
| 16 ff = [None] * (nyears + 1) | |
| 17 res_dir = sys.argv.pop(1) | |
| 18 ff[nyears] = open("%s/oob"%res_dir,"bw") | |
| 19 | |
| 20 def finish(): | |
| 21 global ff, i, count, cur_status | |
| 22 ff[i].write(b'%d\t%d\n'%(count, cur_status)) | |
| 23 | |
| 24 def cbf(status,lastmod): | |
| 25 global cur_status, count, i, year, nyears | |
| 26 stat = int(status) | |
| 27 lm = int(lastmod) | |
| 28 print(i, year, cur_status, count, stat, lm, file = sys.stderr) | |
| 29 while (not year == 0) and (lm > years[i]): | |
| 30 # change of year | |
| 31 if count != 0: | |
| 32 ff[i].write(b'%s\t%d\n'%(count, cur_status)) | |
| 33 if i < nyears: | |
| 34 i += 1 | |
| 35 year += 1 | |
| 36 count = 0 | |
| 37 cur_status = 0 | |
| 38 if i == nyears: | |
| 39 year = 0 | |
| 40 break | |
| 41 if not ff[i]: | |
| 42 ff[i] = open("%s/%s"%(res_dir, year),"bw") | |
| 43 if stat == cur_status: | |
| 44 count += 1 | |
| 45 else: | |
| 46 ff[i].write(b'%d\t%d\n'%(count, cur_status)) | |
| 47 cur_status = stat | |
| 48 count = 1 | |
| 49 | |
| 50 |