Mercurial > hg > cc > cirrus_work

comparison bin/merge_date.py @ 89:a62580816f1c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
merge a stream of ks files with a set of cdx files
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 30 Aug 2023 21:49:43 +0100
parents
children c1a70532444c
comparison
equal deleted inserted replaced
88:49faf679d7df 89:a62580816f1c
1 #!/usr/bin/python3
2 '''Add timestamps from Last-Modified-dated (ks.tsv) files into
3 that year's index
4
5 Usage: merge_date.py ksvstream cdx-dir outdir
6
7 ksvstream consists of tab-separated key, CC date and Unix timestamp
8 ''' # '
9
10 import sys, io, os
11 from isal import igzip
12
13 xpath = "%s/cdx-00%%0.3d.gz"%sys.argv[2]
14 npath = "%s/cdx-00%%0.3d.gz"%sys.argv[3]
15
16 #print(sys.argv[3],npath,file=sys.stderr)
17
18 os.makedirs(sys.argv[3], exist_ok=True)
19
20 fn = -1
21 xf = igzip.IGzipFile(filename=xpath%0)
22 nf = open(npath%0, 'wb')
23
24 df = open(sys.argv[1], 'rb')
25
26 xl = b''
27 xkey = xdate = None
28
29 for dl in df:
30 (dkey, ddate, dtime) = dl.split(b'\t')
31 while dkey != xkey or ddate != xdate:
32 try:
33 if xl == b'':
34 # need to move to next index file
35 nf.close()
36 fn += 1
37 try:
38 xf = igzip.IGzipFile(filename=xpath%fn)
39 except Exception as e:
40 print("No more index input for %s: %s\nUnmatched: |%s|%s|\n"
41 "Last index line: |%s|%s|"%(fn,e,dkey,ddate,xkey,xdate),
42 sys.stderr)
43 exit(1)
44 xl = xf.readline()
45 nf = open(npath%fn, 'wb')
46 #print('xl',xl,file=sys.stderr)
47 (xkey, xdate, xprops) = xl.split(b' ', maxsplit=2)
48 continue
49 else:
50 (xkey, xdate, xprops) = xl.split(b' ', maxsplit=2)
51 except:
52 breakpoint()
53 nf.write(xl)
54 xl = xf.readline()
55 nf.write(xkey)
56 nf.write(b' ')
57 nf.write(xdate)
58 nf.write(b' ')
59 nf.write(xprops[:-2])
60 nf.write(b', "lastmod": "%d"}\n'%int(dtime[:-3]))
61 xl=xf.readline()