Mercurial > hg > cc > cirrus_work
changeset 342:4759d6ac4625 trim
update ROOT, fix deprecated patterns
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Tue, 10 Mar 2026 21:17:17 +0000 |
| parents | 6d0e29e22b12 |
| children | 9ecf193718c1 |
| files | lib/python/cc/ix.py |
| diffstat | 1 files changed, 8 insertions(+), 5 deletions(-) [+] |
line wrap: on
line diff
--- a/lib/python/cc/ix.py Wed Mar 04 21:14:15 2026 +0000 +++ b/lib/python/cc/ix.py Tue Mar 10 21:17:17 2026 +0000 @@ -13,13 +13,15 @@ from subprocess import Popen, PIPE #import asyncio -HACK_USAGE=regex.compile('\[-x\]\n\s*\[length\] \[offset\] \[filename\]') +HACK_USAGE=regex.compile(r'\[-x\]\n\s*\[length\] \[offset\] \[filename\]') BINOUT=sys.stdout.buffer FPAT="/%s/%s/orig/%s/%s" CMD_PROC=None TMPFILENAME=None +ROOT="/work/dc007/shared/pod12/common_crawl" + class HackFormat(argparse.RawDescriptionHelpFormatter): def format_help(self): global FOO @@ -93,9 +95,10 @@ toWrite -= BINOUT.write(buf) def process1(options,buf,filename,offset,length,whole): + global ROOT root=options.root rfn=root+filename - if root!="/beegfs/common_crawl": + if root!=ROOT: # Support using ramdisk or other local disk as a faster cached if not os.path.exists(rfn): if not os.path.exists(os.path.dirname(rfn)): @@ -207,7 +210,7 @@ return def main(): - global _output,TMPFILE,TMPFILENAME,tempfile + global _output,TMPFILE,TMPFILENAME,tempfile,ROOT parser = argparse.ArgumentParser( description='''Extract records from warc files given length, offset and file triples. Input one triple on command line, or @@ -241,7 +244,7 @@ default=FPAT) parser.add_argument('-r','--root',nargs='?', help='File path root, create a copy there if necessary', - default='/beegfs/common_crawl'), + default=ROOT), parser.add_argument('-z','--zipped', help="output raw gzipped record, ignored if any of -bhw supplied", action='store_true') @@ -284,7 +287,7 @@ launch(pa.cmd) # three different ways to process if pa.index: - CDX=regex.compile('length": "([0-9]*)", "offset": "([0-9]*)", "filename": "crawl-data/([^/]*)/segments/([^/]*)/(warc|crawldiagnostics|robotstxt)/(.*\.gz)"') # robotstxt works? + CDX=regex.compile(r'length": "([0-9]*)", "offset": "([0-9]*)", "filename": "crawl-data/([^/]*)/segments/([^/]*)/(warc|crawldiagnostics|robotstxt)/(.*\.gz)"') # robotstxt works? for l in sys.stdin: m=CDX.search(l) if m is None: