# HG changeset patch # User Henry S. Thompson # Date 1737731220 0 # Node ID d3db9e4a05333699d50fc74b8f049c2ed7044aa9 # Parent 5b2e675ac55626713762f3dba35b1c097c6ec074 finally get test code separated from db.pyx to work diff -r 5b2e675ac556 -r d3db9e4a0533 lib/python/cc/lmh/nndb.pyx --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/python/cc/lmh/nndb.pyx Fri Jan 24 15:07:00 2025 +0000 @@ -0,0 +1,45 @@ +# distutils: extra_objects = cdb_min/libcdb.a +# sources = [cdb_min/cdb.c, cdb_min/error.c, cdb_min/open_read.c, cdb_min/seek_cur.c, cdb_min/open_trunc.c, cdb_min/seek_set.c, cdb_min/byte_copy.c, cdb_min/byte_diff.c, cdb_min/error_str.c, cdb_min/uint32_unpack.c, cdb_min/cdb_hash.c] +# distutils: include_dirs = cdb_min + +cimport cdb +cimport db + +# Usage: ... cdb-path probe nreps + +# E.g. python3 -c 'import nndb' ~/results/CC-MAIN-2019-35/warc_lmhx/ks_0.cdb 20190825142846http://71.43.189.10/dermorph/ 10000000 + +import sys, timeit + +cdef db.CCdb CC = db.CCdb() + +cdef db.CCdb testMe(bytes probe): + global CC + res: int = -1 + cdef cdb.Cdb* c_cdb + print('testing... %s %s x %s'%(sys.argv[1],probe,int(sys.argv[3]))) + with open(sys.argv[1],'rb') as dbf: + mv = CC.init(dbf.fileno()) + c_cdb = CC._c_cdb + print("mv %s cdb %s"%(len(mv),(c_cdb))) + cdb.cdb_findstart(c_cdb) + res=CC.find(probe) + if res == 1: + print(res,o:=cdb.cdb_pos(), l:=cdb.cdb_len()) + sys.stdout.buffer.write(mv[o:o+l]) + sys.stdout.buffer.write(b'\n') + else: + print('losing %s'%res) + print('tested') + return CC + +def cfind(p: bytes) -> int: + global CC + return CC.find(p) + +testMe(sys.argv[2].encode('utf8')) + +probe = sys.argv[2].encode("utf8") + +print(timeit.timeit('cfind(probe)', + number=int(sys.argv[3]), globals=globals())) diff -r 5b2e675ac556 -r d3db9e4a0533 lib/python/cc/lmh/setup_nn.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/python/cc/lmh/setup_nn.py Fri Jan 24 15:07:00 2025 +0000 @@ -0,0 +1,6 @@ +from setuptools import Extension, setup +from Cython.Build import cythonize + +setup( + ext_modules = cythonize([Extension("nndb", ["nndb.pyx"])]) +)