Mercurial > hg > cc > azure
comparison master/src/wecu/wecu.py @ 63:d46c8b12fc04
support multiple approaches to key combination, use local files to collect results
| author | Henry S. Thompson <ht@markup.co.uk> |
|---|---|
| date | Wed, 03 Jun 2020 16:40:34 +0000 |
| parents | 892e1c0240e1 |
| children | b91e44355bbf |
comparison
equal
deleted
inserted
replaced
| 62:892e1c0240e1 | 63:d46c8b12fc04 |
|---|---|
| 109 cores_per_worker, | 109 cores_per_worker, |
| 110 HOSTS_FILEPATH, | 110 HOSTS_FILEPATH, |
| 111 WORK_DIR, | 111 WORK_DIR, |
| 112 ('sac_mapper.py' if args.mapper is None | 112 ('sac_mapper.py' if args.mapper is None |
| 113 else args.mapper), | 113 else args.mapper), |
| 114 ('' if args.keyHandler is None | |
| 115 else "-h %s"%args.keyHandler) | |
| 114 ('' if args.filter is None | 116 ('' if args.filter is None |
| 115 else "-f '%s'"%args.filter), | 117 else "-f '%s'"%args.filter), |
| 116 ('' if args.numKeys is None | 118 ('' if args.numKeys is None |
| 117 else "-k %s"%args.numKeys), | 119 else "-k %s"%args.numKeys), |
| 118 ('by-file' if args.by_file | 120 ('by-file' if args.by_file |
| 175 sac_list.add_argument('--regex', action="store_true", help="Provide this flag to indicate that the provided strings should be treated as regular expressions") | 177 sac_list.add_argument('--regex', action="store_true", help="Provide this flag to indicate that the provided strings should be treated as regular expressions") |
| 176 sac_list.add_argument('--by-file', action="store_true", help="Provide this flag to indicate that the output should not be aggregated and displayed per file instead") | 178 sac_list.add_argument('--by-file', action="store_true", help="Provide this flag to indicate that the output should not be aggregated and displayed per file instead") |
| 177 sac_list.add_argument('--dict', action="store_true", help="Provide this flag to indicate that the output should aggregated and displayed in the form 'res={dict}'") | 179 sac_list.add_argument('--dict', action="store_true", help="Provide this flag to indicate that the output should aggregated and displayed in the form 'res={dict}'") |
| 178 sac_list.add_argument('--mapper', type=str, help="Supply a bespoke mapper for use in place of sac_mapper.py") | 180 sac_list.add_argument('--mapper', type=str, help="Supply a bespoke mapper for use in place of sac_mapper.py") |
| 179 sac_list.add_argument('--filter', type=str, help="Supply a filter on the unzipped warc file ahead of the mapper") | 181 sac_list.add_argument('--filter', type=str, help="Supply a filter on the unzipped warc file ahead of the mapper") |
| 182 sac_list.add_argument('--keyHandler', type=int, help="Key handler, default for nested dicts, 1 for 4-tuple, 2 for concat") | |
| 180 sac_list.add_argument('--numKeys', type=int, help="Depth of key list, default 1") | 183 sac_list.add_argument('--numKeys', type=int, help="Depth of key list, default 1") |
| 181 sac_list.add_argument('--jobs-per-worker', type=int, help="By deafult the number of concurrent tasks is set to the number of available logical cores. Provide this flag to set a different number of concurrent tasks.") | 184 sac_list.add_argument('--jobs-per-worker', type=int, help="By deafult the number of concurrent tasks is set to the number of available logical cores. Provide this flag to set a different number of concurrent tasks.") |
| 182 sac_list.set_defaults(handler=sac_handler) | 185 sac_list.set_defaults(handler=sac_handler) |
| 183 | 186 |
| 184 # Generate sample parser | 187 # Generate sample parser |