Mercurial > hg > cc > azure

comparison master/src/wecu/wecu.py @ 61:cfaf5223b071

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
trying to get my own mapper working
author Henry S. Thompson <ht@markup.co.uk>
date Sun, 31 May 2020 12:06:44 +0000
parents 5fdca5baa4e9
children 892e1c0240e1
comparison
equal deleted inserted replaced
60:5fdca5baa4e9 61:cfaf5223b071
103 103
104 patterns_str = ' '.join(['"{}"'.format(x) for x in args.pattern]) 104 patterns_str = ' '.join(['"{}"'.format(x) for x in args.pattern])
105 105
106 cores_per_worker = num_cores(args) 106 cores_per_worker = num_cores(args)
107 107
108 os.system('run_sac.sh {} {} {} {} {} {} {}'.format( 108 os.system('run_sac.sh {} {} {} {} {} {} {} {}'.format(
109 cores_per_worker, 109 cores_per_worker,
110 HOSTS_FILEPATH, 110 HOSTS_FILEPATH,
111 WORK_DIR, 111 WORK_DIR,
112 ('sac_mapper.py' if args.mapper is None 112 ('sac_mapper.py' if args.mapper is None
113 else args.mapper), 113 else args.mapper),
114 ('' if args.filter is None
115 else "-f '%s'"%args.filter),
114 ('by-file' if args.by_file 116 ('by-file' if args.by_file
115 else 'aggregate'), 117 else 'aggregate'),
116 regex_str, 118 regex_str,
117 patterns_str)) 119 patterns_str))
118 120
169 sac_list = subparsers.add_parser('sac', help='Execute scan-and-count (SAC) workloads directly from the command line') 171 sac_list = subparsers.add_parser('sac', help='Execute scan-and-count (SAC) workloads directly from the command line')
170 sac_list.add_argument('pattern', type=str, nargs='+') 172 sac_list.add_argument('pattern', type=str, nargs='+')
171 sac_list.add_argument('--regex', action="store_true", help="Provide this flag to indicate that the provided strings should be treated as regular expressions") 173 sac_list.add_argument('--regex', action="store_true", help="Provide this flag to indicate that the provided strings should be treated as regular expressions")
172 sac_list.add_argument('--by-file', action="store_true", help="Provide this flag to indicate that the output should not be aggregated and displayed per file instead") 174 sac_list.add_argument('--by-file', action="store_true", help="Provide this flag to indicate that the output should not be aggregated and displayed per file instead")
173 sac_list.add_argument('--mapper', type=str, help="Supply a bespoke mapper for use in place of sac_mapper.py") 175 sac_list.add_argument('--mapper', type=str, help="Supply a bespoke mapper for use in place of sac_mapper.py")
176 sac_list.add_argument('--filter', type=str, help="Supply a filter on the unzipped warc file ahead of the mapper")
174 sac_list.add_argument('--jobs-per-worker', type=int, help="By deafult the number of concurrent tasks is set to the number of available logical cores. Provide this flag to set a different number of concurrent tasks.") 177 sac_list.add_argument('--jobs-per-worker', type=int, help="By deafult the number of concurrent tasks is set to the number of available logical cores. Provide this flag to set a different number of concurrent tasks.")
175 sac_list.set_defaults(handler=sac_handler) 178 sac_list.set_defaults(handler=sac_handler)
176 179
177 # Generate sample parser 180 # Generate sample parser
178 generate_parser = subparsers.add_parser('generate-sample', help='Generate a sample of a chosen Common Crawl month') 181 generate_parser = subparsers.add_parser('generate-sample', help='Generate a sample of a chosen Common Crawl month')