Mercurial > hg > cc > cirrus_work
changeset 311:2ecf29fd1cee trim
CC has moved
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Mon, 22 Dec 2025 22:19:54 +0000 |
| parents | 4f7977c697ef |
| children | cc70ae0b156d |
| files | lib/python/cc/lmh/warc2cdb.py |
| diffstat | 1 files changed, 3 insertions(+), 3 deletions(-) [+] |
line wrap: on
line diff
--- a/lib/python/cc/lmh/warc2cdb.py Mon Dec 22 22:19:07 2025 +0000 +++ b/lib/python/cc/lmh/warc2cdb.py Mon Dec 22 22:19:54 2025 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env python3 '''Produce cdb_input-style files from warc responses with lmh header value - Usage: warc2cdb.py CC-date segment output-dir''' + Usage: warc2cdb.py CC-date segment output-dir warc_file_type warc_file_range lm_shrink_prefix cdate_shrink_prefix ''' import re, warc, sys, glob, codecs, os.path import cython, typing @@ -39,7 +39,7 @@ return '\\U%08X'%c def java_unicode_encode(ude): - '''like backslashreplace but use uppercase and \ u00NN instead of \ xnn''' + '''like backslashreplace but use uppercase and \\ u00NN instead of \\ xnn''' return (''.join(_u_esc(ord(c)) for c in ude.object[ude.start:ude.end]), ude.end) @@ -126,7 +126,7 @@ LM_ERA_L = len(era) C_MONTH = month.encode('utf8') C_MONTH_L = len(month) - infile_pat='bash -c "ls /beegfs/common_crawl/CC-MAIN-%s/*.%s/orig/%s/*00%s.warc.gz | sort -k8"'%( + infile_pat='bash -c "ls $CCC/CC-MAIN-%s/*.%s/orig/%s/*00%s.warc.gz | sort -k8"'%( CCdate, segment, subdir, ("???" if fpat is None else ( (("{%s..%s}"%tuple(fpat.split(','))) if ',' in fpat else fpat))))