Mercurial > hg > cc > azure

comparison master/src/wecu/wecu.py @ 62:892e1c0240e1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added more robust (I hope) error handling, got reducer working with support for choosing dict or tsv output
author Henry S. Thompson <ht@markup.co.uk>
date Tue, 02 Jun 2020 17:35:07 +0000
parents cfaf5223b071
children d46c8b12fc04
comparison
equal deleted inserted replaced
61:cfaf5223b071 62:892e1c0240e1
103 103
104 patterns_str = ' '.join(['"{}"'.format(x) for x in args.pattern]) 104 patterns_str = ' '.join(['"{}"'.format(x) for x in args.pattern])
105 105
106 cores_per_worker = num_cores(args) 106 cores_per_worker = num_cores(args)
107 107
108 os.system('run_sac.sh {} {} {} {} {} {} {} {}'.format( 108 os.system('run_sac.sh {} {} {} {} {} {} {} {} {}'.format(
109 cores_per_worker, 109 cores_per_worker,
110 HOSTS_FILEPATH, 110 HOSTS_FILEPATH,
111 WORK_DIR, 111 WORK_DIR,
112 ('sac_mapper.py' if args.mapper is None 112 ('sac_mapper.py' if args.mapper is None
113 else args.mapper), 113 else args.mapper),
114 ('' if args.filter is None 114 ('' if args.filter is None
115 else "-f '%s'"%args.filter), 115 else "-f '%s'"%args.filter),
116 ('' if args.numKeys is None
117 else "-k %s"%args.numKeys),
116 ('by-file' if args.by_file 118 ('by-file' if args.by_file
117 else 'aggregate'), 119 else 'dict' if args.dict else 'aggregate'),
118 regex_str, 120 regex_str,
119 patterns_str)) 121 patterns_str))
120 122
121 def generate_handler(args): 123 def generate_handler(args):
122 import generate_file_list 124 import generate_file_list
170 # Scan-and-count parser 172 # Scan-and-count parser
171 sac_list = subparsers.add_parser('sac', help='Execute scan-and-count (SAC) workloads directly from the command line') 173 sac_list = subparsers.add_parser('sac', help='Execute scan-and-count (SAC) workloads directly from the command line')
172 sac_list.add_argument('pattern', type=str, nargs='+') 174 sac_list.add_argument('pattern', type=str, nargs='+')
173 sac_list.add_argument('--regex', action="store_true", help="Provide this flag to indicate that the provided strings should be treated as regular expressions") 175 sac_list.add_argument('--regex', action="store_true", help="Provide this flag to indicate that the provided strings should be treated as regular expressions")
174 sac_list.add_argument('--by-file', action="store_true", help="Provide this flag to indicate that the output should not be aggregated and displayed per file instead") 176 sac_list.add_argument('--by-file', action="store_true", help="Provide this flag to indicate that the output should not be aggregated and displayed per file instead")
177 sac_list.add_argument('--dict', action="store_true", help="Provide this flag to indicate that the output should aggregated and displayed in the form 'res={dict}'")
175 sac_list.add_argument('--mapper', type=str, help="Supply a bespoke mapper for use in place of sac_mapper.py") 178 sac_list.add_argument('--mapper', type=str, help="Supply a bespoke mapper for use in place of sac_mapper.py")
176 sac_list.add_argument('--filter', type=str, help="Supply a filter on the unzipped warc file ahead of the mapper") 179 sac_list.add_argument('--filter', type=str, help="Supply a filter on the unzipped warc file ahead of the mapper")
180 sac_list.add_argument('--numKeys', type=int, help="Depth of key list, default 1")
177 sac_list.add_argument('--jobs-per-worker', type=int, help="By deafult the number of concurrent tasks is set to the number of available logical cores. Provide this flag to set a different number of concurrent tasks.") 181 sac_list.add_argument('--jobs-per-worker', type=int, help="By deafult the number of concurrent tasks is set to the number of available logical cores. Provide this flag to set a different number of concurrent tasks.")
178 sac_list.set_defaults(handler=sac_handler) 182 sac_list.set_defaults(handler=sac_handler)
179 183
180 # Generate sample parser 184 # Generate sample parser
181 generate_parser = subparsers.add_parser('generate-sample', help='Generate a sample of a chosen Common Crawl month') 185 generate_parser = subparsers.add_parser('generate-sample', help='Generate a sample of a chosen Common Crawl month')