# HG changeset patch # User Henry S. Thompson # Date 1739391459 0 # Node ID 5c013fd18ea085c18aed97e2ee8dacad78b56d56 # Parent 603b46f3062d2752f495d7a85b7a2a8a6b47a72e working, but very slowly diff -r 603b46f3062d -r 5c013fd18ea0 lib/python/cc/lmh/test_cdb.py --- a/lib/python/cc/lmh/test_cdb.py Wed Feb 12 13:01:05 2025 +0000 +++ b/lib/python/cc/lmh/test_cdb.py Wed Feb 12 20:17:39 2025 +0000 @@ -35,7 +35,7 @@ CC = [CCdb() for i in range(D)] mv = [CC[i].init(ff[i].fileno()) for i in range(D)] - with igzip.IGzipFile('/beegfs/common_crawl/CC-MAIN-2019-35/cdx/warc/cdx-00100.gz','rb') as cdx_in, igzip.IGzipFile('results/CC-MAIN-2019-35/warc_lmhx/cdxp-00100.gz', 'wb') as cdx_out: + with igzip.IGzipFile('/beegfs/common_crawl/CC-MAIN-2019-35/cdx/warc/cdx-00100.gz','rb') as cdx_in, igzip.IGzipFile('/work/dc007/dc007/hst/results/CC-MAIN-2019-35/warc_lmhx/cdx-00100.gz', 'wb') as cdx_out: for l in cdx_in: key: cython.bytes cdate: cython.bytes @@ -45,10 +45,10 @@ if (m:=PAT.search(props)): seg = int(m[2]) N += 1 - int: i = int(seg / d) - key = cdate+m[1] + i: int = int(seg / d) + k: cython.bytes = cdate+m[1] CC[i].findstart() - if (res := CC[i].find(key)) == 1: + if (res := CC[i].find(k)) == 1: hits += 1 cdx_out.write(key) cdx_out.write(b' ')