# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1772372452 0
# Node ID 282a35927943b095fcf1300061bbc9b8829f8a88
# Parent  83450325421fd457910d98e841347fce1d307294
begin rework to do merge in python

diff -r 83450325421f -r 282a35927943 results/CC-MAIN-2019-35/warc_lmhx/mcdb/idx/tcb.py
--- a/results/CC-MAIN-2019-35/warc_lmhx/mcdb/idx/tcb.py	Sat Feb 28 14:48:32 2026 +0000
+++ b/results/CC-MAIN-2019-35/warc_lmhx/mcdb/idx/tcb.py	Sun Mar 01 13:40:52 2026 +0000
@@ -1,50 +1,63 @@
-import sys,os
+'''For use as callbacks from cdx2tsv.py, e.g.
+   uz cdx-00120.gz | fgrep '"lastmod":'| cdx2tsv.py -c tcb /tmp/hst/cby status lastmod
+'''
+import sys, os, bisect
+
+class Year:
+  def __init__(self, year, far_end):
+    self.year = year
+    self.far_end = far_end
 
 def init():
-  global i, year, count, cur_status, years, nyears, res_dir, ff
-
-  i = 0
-  year = 1995
-  count = 0
-  cur_status = 0
+  global i, year, count, cur_status, years, nyears, res_dir, ff, far_ends
 
   with open("%s/results/CC-MAIN-2019-35/warc_lmhx/years.txt"%os.getenv("HOME"),"r") \
         as yf:
-    years=[int(l) for l in yf]
+    y = 1993
+    years=[Year(y:=(y+1),int(l)) for l in yf]
+
+  far_ends = [y.far_end for y in years]
 
   nyears = len(years)
-  ff = [None] * (nyears + 1)
-  res_dir = sys.argv.pop(1)
-  ff[nyears] = open("%s/oob"%res_dir,"bw")
+  #res_dir = sys.argv.pop(1)
+  years.append(Year("oob",0))
+
+  #print('i',years, nyears, file = sys.stderr)
 
 def finish():
   global ff, i, count, cur_status
   ff[i].write(b'%d\t%d\n'%(count, cur_status))
 
+def find_year(lastmod):
+  global far_ends
+  return years[bisect.bisect_right(far_ends, lastmod)]
+
 def cbf(status,lastmod):
-  global cur_status, count, i, year, nyears
+  global i, year, count, cur_status, years, nyears, res_dir, ff
   stat = int(status)
   lm = int(lastmod)
-  print(i, year, cur_status, count, stat, lm, file = sys.stderr)
-  while (not year == 0) and (lm > years[i]):
+  #print('c',i, year, cur_status, count, stat, lm, years[i], file = sys.stderr)
+  if (not year == 0) and (lm > years[i]):
     # change of year
     if count != 0:
-      ff[i].write(b'%s\t%d\n'%(count, cur_status))
-    if i < nyears:
+      ff[i].write(b'%d\t%d\n'%(count, cur_status))
+    cur_status = stat
+    count = 1
+    while lm > years[i]:
+      #print('ny',year,i,lm,years[i],count,file=sys.stderr)
       i += 1
       year += 1
-      count = 0
-      cur_status = 0
       if i == nyears:
         year = 0
-        break
+        return
     if not ff[i]:
-      ff[i] = open("%s/%s"%(res_dir, year),"bw")
-  if stat == cur_status:
+      ff[i] = open("%s/%s"%(res_dir, year),"ba")
+  elif stat == cur_status:
     count += 1
   else:
+    #print('ns',count,i,year,ff[i],file=sys.stderr)
     ff[i].write(b'%d\t%d\n'%(count, cur_status))
     cur_status = stat
     count = 1
+
   
-