--- a/results/CC-MAIN-2019-35/warc_lmhx/mcdb/idx/tcb.py	Sun Mar 01 13:40:52 2026 +0000
+++ b/results/CC-MAIN-2019-35/warc_lmhx/mcdb/idx/tcb.py	Sun Mar 01 15:22:10 2026 +0000
@@ -2,14 +2,24 @@
    uz cdx-00120.gz | fgrep '"lastmod":'| cdx2tsv.py -c tcb /tmp/hst/cby status lastmod
 '''
 import sys, os, bisect
+from collections import Counter
 
 class Year:
   def __init__(self, year, far_end):
     self.year = year
     self.far_end = far_end
+    self.stats = Counter()
+
+  def write(self):
+    global res_dir
+    if self.stats:
+      with open("%s/%s"%(res_dir, self.year),"bw") as f:
+        for s,c in sorted(list(self.stats.items()),
+                          key=lambda i:i[1], reverse=True):
+          f.write(b'%d\t%d\n'%(c, s))
 
 def init():
-  global i, year, count, cur_status, years, nyears, res_dir, ff, far_ends
+  global years, nyears, res_dir, far_ends, NLM
 
   with open("%s/results/CC-MAIN-2019-35/warc_lmhx/years.txt"%os.getenv("HOME"),"r") \
         as yf:
@@ -19,45 +29,30 @@
   far_ends = [y.far_end for y in years]
 
   nyears = len(years)
-  #res_dir = sys.argv.pop(1)
+  res_dir = sys.argv.pop(1)
   years.append(Year("oob",0))
 
+  years.append(NLM := Year("no_LM",0))
+
   #print('i',years, nyears, file = sys.stderr)
 
 def finish():
-  global ff, i, count, cur_status
-  ff[i].write(b'%d\t%d\n'%(count, cur_status))
+  global years
+  for y in years:
+    y.write()
+  
 
 def find_year(lastmod):
   global far_ends
   return years[bisect.bisect_right(far_ends, lastmod)]
 
 def cbf(status,lastmod):
-  global i, year, count, cur_status, years, nyears, res_dir, ff
+  global NLM
   stat = int(status)
-  lm = int(lastmod)
-  #print('c',i, year, cur_status, count, stat, lm, years[i], file = sys.stderr)
-  if (not year == 0) and (lm > years[i]):
-    # change of year
-    if count != 0:
-      ff[i].write(b'%d\t%d\n'%(count, cur_status))
-    cur_status = stat
-    count = 1
-    while lm > years[i]:
-      #print('ny',year,i,lm,years[i],count,file=sys.stderr)
-      i += 1
-      year += 1
-      if i == nyears:
-        year = 0
-        return
-    if not ff[i]:
-      ff[i] = open("%s/%s"%(res_dir, year),"ba")
-  elif stat == cur_status:
-    count += 1
+  if lastmod == 'NA':
+    NLM.stats[stat] += 1
   else:
-    #print('ns',count,i,year,ff[i],file=sys.stderr)
-    ff[i].write(b'%d\t%d\n'%(count, cur_status))
-    cur_status = stat
-    count = 1
+    lm = int(lastmod)
+    find_year(lm).stats[stat]+=1