changeset 321:cf50a54b8b53 trim

handle pipelined input
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 20 Jan 2026 19:52:50 +0000
parents c68714dee9f2
children 783d65d43fb0
files lib/python/cc/lmh/dbt.pyx
diffstat 1 files changed, 43 insertions(+), 25 deletions(-) [+]
line wrap: on
line diff
--- a/lib/python/cc/lmh/dbt.pyx	Tue Jan 20 19:46:48 2026 +0000
+++ b/lib/python/cc/lmh/dbt.pyx	Tue Jan 20 19:52:50 2026 +0000
@@ -2,43 +2,61 @@
 # # distutils: sources = [cdb_min/cdb.c, cdb_min/error.c, cdb_min/open_read.c, cdb_min/seek_cur.c, cdb_min/open_trunc.c, cdb_min/seek_set.c, cdb_min/byte_copy.c, cdb_min/byte_diff.c, cdb_min/error_str.c, cdb_min/uint32_unpack.c, cdb_min/cdb_hash.c]
 # distutils: extra_objects = cdb/libcdb.a
 # distutils: include_dirs = cdb
+# cython: profile=False, language_level=3str
 
-# Usage: testc cdb-path probe nreps
+# Usage: python3 -c 'import dbt' cdb-path ( -b infile | probe )
 
-# E.g. python3 -c 'import db' ~/results/CC-MAIN-2019-35/warc_lmhx/ks_0.cdb 20190825142846http://71.43.189.10/dermorph/ 10000000
+# E.g. python3 -c 'import db' ~/results/CC-MAIN-2019-35/warc_lmhx/ks_0.cdb 20190825142846http://71.43.189.10/dermorph/
 
-import sys, timeit
+import sys, typing
 
 cimport cdb
 cimport db
 
-cdef db.CCdb CC
+cdef db.CCdb CC = db.CCdb()
 
-cdef db.CCdb testMe(bytes probe):
+cdef bint testMe(bytes probe, bint verbose = True):
   res: int = -1
-  print('testing... %s %s x %s'%(sys.argv[1],probe,int(sys.argv[3])))
-  cdef db.CCdb cd = db.CCdb()
-  with open(sys.argv[1],'rb') as dbf:
-    cd.init(dbf.fileno())
-    cdb.cdb_findstart(cd._c_cdb)
-    if (res:=cdb.cdb_find(cd._c_cdb,probe,len(probe))) == 1:
-      cdb.cdb_write(cd._c_cdb,1)
-      sys.stdout.buffer.write(b'\n')
-    else:
-      print(0)
-      print(probe,len(probe),cdb.cdb_find(cd._c_cdb,probe,len(probe)))
-      print(res,o:=cdb.cdb_pos(cd._c_cdb), l:=cdb.cdb_len(cd._c_cdb))
-      sys.stdout.buffer.write(b'\n')
-  print('tested')
-  return cd
+  if verbose:
+    print('testing... %s %s'%(sys.argv[1],probe #  x %s
+                                   #,int(sys.argv[3])
+                                   ))
+  cdb.cdb_findstart(CC._c_cdb)
+  if (res:=cdb.cdb_find(CC._c_cdb,probe,len(probe))) == 1:
+    if not verbose:
+      return True
+    cdb.cdb_write(CC._c_cdb,1)
+    sys.stdout.buffer.write(b'\n')
+  else:
+    if not verbose:
+      return False
+    print(0)
+    print(probe,len(probe),cdb.cdb_find(CC._c_cdb,probe,len(probe)))
+    print(res,o:=cdb.cdb_pos(CC._c_cdb), l:=cdb.cdb_len(CC._c_cdb))
+    sys.stdout.buffer.write(b'\n')
 
 def cfind(p: bytes) -> int:
   global CC
   return cdb.cdb_find(CC._c_cdb, p, len(p))
 
-CC = testMe(sys.argv[2].encode('utf8'))
+with open(sys.argv[1],'rb') as dbf:
+  CC.init(dbf.fileno())
+  if sys.argv[2] == '-b':
+    N: int = 0
+    w: int = 0
+    p: bytes
+    pfile: typing.BinaryIO
+    with open(sys.argv[3],'rb') as pfile:
+      for p in pfile:
+        N += 1
+        if testMe(p.rstrip(),False):
+          w += 1
+        else:
+          sys.stderr.write(p.decode('utf8'))
+    print("%d hits out of %d"%(w,N))
+  else:
+    testMe(sys.argv[2].encode('utf8'))
+    print('tested')
 
-exit()
-
-print(timeit.timeit('cfind(sys.argv[2].encode("utf8"))',
-               number=int(sys.argv[3]), globals=globals()))
+#print(timeit.timeit('cfind(sys.argv[2].encode("utf8"))',
+#               number=int(sys.argv[3]), globals=globals()))