Mercurial > hg > cc > cirrus_work
changeset 321:cf50a54b8b53 trim
handle pipelined input
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Tue, 20 Jan 2026 19:52:50 +0000 |
| parents | c68714dee9f2 |
| children | 783d65d43fb0 |
| files | lib/python/cc/lmh/dbt.pyx |
| diffstat | 1 files changed, 43 insertions(+), 25 deletions(-) [+] |
line wrap: on
line diff
--- a/lib/python/cc/lmh/dbt.pyx Tue Jan 20 19:46:48 2026 +0000 +++ b/lib/python/cc/lmh/dbt.pyx Tue Jan 20 19:52:50 2026 +0000 @@ -2,43 +2,61 @@ # # distutils: sources = [cdb_min/cdb.c, cdb_min/error.c, cdb_min/open_read.c, cdb_min/seek_cur.c, cdb_min/open_trunc.c, cdb_min/seek_set.c, cdb_min/byte_copy.c, cdb_min/byte_diff.c, cdb_min/error_str.c, cdb_min/uint32_unpack.c, cdb_min/cdb_hash.c] # distutils: extra_objects = cdb/libcdb.a # distutils: include_dirs = cdb +# cython: profile=False, language_level=3str -# Usage: testc cdb-path probe nreps +# Usage: python3 -c 'import dbt' cdb-path ( -b infile | probe ) -# E.g. python3 -c 'import db' ~/results/CC-MAIN-2019-35/warc_lmhx/ks_0.cdb 20190825142846http://71.43.189.10/dermorph/ 10000000 +# E.g. python3 -c 'import db' ~/results/CC-MAIN-2019-35/warc_lmhx/ks_0.cdb 20190825142846http://71.43.189.10/dermorph/ -import sys, timeit +import sys, typing cimport cdb cimport db -cdef db.CCdb CC +cdef db.CCdb CC = db.CCdb() -cdef db.CCdb testMe(bytes probe): +cdef bint testMe(bytes probe, bint verbose = True): res: int = -1 - print('testing... %s %s x %s'%(sys.argv[1],probe,int(sys.argv[3]))) - cdef db.CCdb cd = db.CCdb() - with open(sys.argv[1],'rb') as dbf: - cd.init(dbf.fileno()) - cdb.cdb_findstart(cd._c_cdb) - if (res:=cdb.cdb_find(cd._c_cdb,probe,len(probe))) == 1: - cdb.cdb_write(cd._c_cdb,1) - sys.stdout.buffer.write(b'\n') - else: - print(0) - print(probe,len(probe),cdb.cdb_find(cd._c_cdb,probe,len(probe))) - print(res,o:=cdb.cdb_pos(cd._c_cdb), l:=cdb.cdb_len(cd._c_cdb)) - sys.stdout.buffer.write(b'\n') - print('tested') - return cd + if verbose: + print('testing... %s %s'%(sys.argv[1],probe # x %s + #,int(sys.argv[3]) + )) + cdb.cdb_findstart(CC._c_cdb) + if (res:=cdb.cdb_find(CC._c_cdb,probe,len(probe))) == 1: + if not verbose: + return True + cdb.cdb_write(CC._c_cdb,1) + sys.stdout.buffer.write(b'\n') + else: + if not verbose: + return False + print(0) + print(probe,len(probe),cdb.cdb_find(CC._c_cdb,probe,len(probe))) + print(res,o:=cdb.cdb_pos(CC._c_cdb), l:=cdb.cdb_len(CC._c_cdb)) + sys.stdout.buffer.write(b'\n') def cfind(p: bytes) -> int: global CC return cdb.cdb_find(CC._c_cdb, p, len(p)) -CC = testMe(sys.argv[2].encode('utf8')) +with open(sys.argv[1],'rb') as dbf: + CC.init(dbf.fileno()) + if sys.argv[2] == '-b': + N: int = 0 + w: int = 0 + p: bytes + pfile: typing.BinaryIO + with open(sys.argv[3],'rb') as pfile: + for p in pfile: + N += 1 + if testMe(p.rstrip(),False): + w += 1 + else: + sys.stderr.write(p.decode('utf8')) + print("%d hits out of %d"%(w,N)) + else: + testMe(sys.argv[2].encode('utf8')) + print('tested') -exit() - -print(timeit.timeit('cfind(sys.argv[2].encode("utf8"))', - number=int(sys.argv[3]), globals=globals())) +#print(timeit.timeit('cfind(sys.argv[2].encode("utf8"))', +# number=int(sys.argv[3]), globals=globals()))
