Mercurial > hg > cc > cirrus_work
changeset 365:a9ecdd85d205 plus
new shrunk cdb en- and de-coding just barely working
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Tue, 07 Apr 2026 23:03:15 +0100 |
| parents | bd931adc820b |
| children | 1ff1bbf30472 |
| files | lib/python/cc/lmh/cdb_one.py lib/python/cc/lmh/shrink_key.py lib/python/cc/lmh/warc2cdb.py |
| diffstat | 3 files changed, 20 insertions(+), 21 deletions(-) [+] |
line wrap: on
line diff
--- a/lib/python/cc/lmh/cdb_one.py Tue Apr 07 15:04:09 2026 +0100 +++ b/lib/python/cc/lmh/cdb_one.py Tue Apr 07 23:03:15 2026 +0100 @@ -29,10 +29,7 @@ S: int = int(sys.argv[2]) E: int = int(sys.argv[3]) - DP: cython.bytes = sys.argv[4].encode('ASCII') - dpl: int = len(DP) - - f = open(sys.argv[1]%(S,E-1),'rb') + f = open(sys.argv[1]%(S,E),'rb') C = CCdb() C.init(f.fileno()) @@ -50,16 +47,17 @@ i: int nd: cython.bytes if (m:=C_PAT.match(l)): - (segb, kind, offsetb, fnob) = m.groups() + (offsetb, segb, kind, fnob) = m.groups() else: raise ValueError(l) N += 1 seg = int(segb) - if (seg >= S and seg < E): + print(seg, kind, offsetb, fnob,file=sys.stderr) + if (seg >= S and seg <= E): M += 1 shrink_key.set_sd_from_bytes(kind) key = shrink_key.shrink_key(seg, int(fnob), int(offsetb)) - if (res := C.find(key) == 1: + if (res := C.find(key)) == 1: hits += 1 cdx_out.write(memoryview(l)[:-2]) cdx_out.write(b', "lastmod": "')
--- a/lib/python/cc/lmh/shrink_key.py Tue Apr 07 15:04:09 2026 +0100 +++ b/lib/python/cc/lmh/shrink_key.py Tue Apr 07 23:03:15 2026 +0100 @@ -12,8 +12,8 @@ and converts a last-modified unix-numeric datetime to a 4-byte integer WARNING: If a crawl ever has more than 1023 files per segment, this -approach won't work any more''' -''' +approach won't work any more +''' #' import cython, typing, timeit, re, sys, array LAST_DATE: bytes = int.to_bytes(2147483647,4) @@ -27,17 +27,21 @@ CRAWLDIAGNOSTICS: int = 2 SUBDIR_CODES={b'w':WARC,b'c':CRAWLDIAGNOSTICS,b'r':ROBOTSTXT} -def set_sd_from_string(str: subdir): +def set_sd_from_string(subdir: str): global SD, SUBDIR_CODES SD = (shrink_key.SUBDIR_CODES[b'w'] if subdir.startswith('w') else (shrink_key.SUBDIR_CODES[b'c'] if subdir.startswith('c') - else shrink_key.SUBDIR_CODES(bytes(ord(subdir[0]))))) + else shrink_key.SUBDIR_CODES[bytes(ord(subdir[0]))])) -def set_sd_from_bytes(bytes: subdir): +def set_sd_from_bytes(subdir: bytes): global SD, SUBDIR_CODES - SD = (shrink_key.SUBDIR_CODES[b'w'] if subdir.startswith(b'w') - else (shrink_key.SUBDIR_CODES[b'c'] if subdir.startswith('bc') - else shrink_key.SUBDIR_CODES(subdir[0]))) + z: int = subdir[0] + SD = (SUBDIR_CODES[b'w'] if z == 119 + else (SUBDIR_CODES[b'c'] if z == 99 + else (SUBDIR_CODES[b'r'] if z == 114 + else -1))) + if SD == -1: + raise KeyError(subdir) def shrink_entry(seg: int, fileno: int, offset: int, lm: int) -> tuple[bytes,bytes]: lmb: bytes @@ -53,10 +57,10 @@ lmb = int.to_bytes(lm, 4, signed=True) return(shrink_key(seg, fileno, offset), lm) -def shrink_key(seg: int, fileno: int, offset: int) -> bytes +def shrink_key(seg: int, fileno: int, offset: int) -> bytes: global SD - return (int.to_bytes(seg,1)+int.to_bytes(fileno+(SD << 10),2)+int.to_bytes(offset,4),lmb) + return int.to_bytes(seg,1)+int.to_bytes(fileno+(SD << 10),2)+int.to_bytes(offset,4) def shrink_date(wdate: bytes, dp: bytes) -> bytes: dpl: int = len(dp)
--- a/lib/python/cc/lmh/warc2cdb.py Tue Apr 07 15:04:09 2026 +0100 +++ b/lib/python/cc/lmh/warc2cdb.py Tue Apr 07 23:03:15 2026 +0100 @@ -79,10 +79,7 @@ infile_pat='bash -c "ls $CCC/CC-MAIN-%s/*.%s/orig/%s/*00%s.warc.gz | sort -k8"'%(CCdate, segment, subdir, fpat) try: - shrink_key.SD = (shrink_key.SUBDIR_CODES[b'w'] if subdir.startswith('w') - else (shrink_key.SUBDIR_CODES[b'c'] if subdir.startswith('c') - else shrink_key.SUBDIR_CODES(bytes(ord(subdir[0]))))) - + shrink_key.set_ftype_from_string(subdir) except KeyError: print('bogus type %s'%subdir, file=sys.stderr) exit(1)