# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1746475066 -3600
# Node ID fdec28613df3aade09011bb118be8c6e3a42845d
# Parent  5e08e6db47ad421a665477101b8c5fa47a016efd
robotstxt now working?

diff -r 5e08e6db47ad -r fdec28613df3 lib/python/cc/ix.py
--- a/lib/python/cc/ix.py	Mon May 05 20:57:30 2025 +0100
+++ b/lib/python/cc/ix.py	Mon May 05 20:57:46 2025 +0100
@@ -284,7 +284,7 @@
       launch(pa.cmd)
   # three different ways to process
   if pa.index:
-    CDX=regex.compile('length": "([0-9]*)", "offset": "([0-9]*)", "filename": "crawl-data/([^/]*)/segments/([^/]*)/(warc|crawldiagnostics)/(.*\.gz)"') # no robotstxt yet...
+    CDX=regex.compile('length": "([0-9]*)", "offset": "([0-9]*)", "filename": "crawl-data/([^/]*)/segments/([^/]*)/(warc|crawldiagnostics|robotstxt)/(.*\.gz)"') # robotstxt works?
     for l in sys.stdin:
       m=CDX.search(l)
       if m is None: