<commit_before>import logging
import sys
from optparse import OptionParser

from netort.resource import manager as resource_manager
from yandextank.core.consoleworker import TankWorker
from yandextank.validator.validator import ValidationError


def main():
    parser = OptionParser()
    parser.add_option(
        '-c',
        '--config',
        action='append',
        help="Path to INI file containing run options, multiple options accepted",
        default=[]
    )
    parser.add_option(
        '-f',
        '--fail-lock',
        action='store_true',
        dest='lock_fail',
        help="Don't wait for lock to release, fail test instead")
    parser.add_option(
        '-i',
        '--ignore-lock',
        action='store_true',
        dest='ignore_lock',
        help="Ignore lock files from concurrent instances, has precedence before --lock-fail"
    )
    parser.add_option(
        '-k',
        '--lock-dir',
        action='store',
        dest='lock_dir',
        type="string",
        help="Directory for lock file")
    parser.add_option(
        '-l',
        '--log',
        action='store',
        default="tank.log",
        help="Tank log file location")
    parser.add_option(
        '--error_log',
        action='store',
        dest='error_log',
        default="tank_errors.log",
        help="Tank errors log file location")
    parser.add_option(
        '-m',
        '--manual-start',
        action='store_true',
        dest='manual_start',
        help="Wait for Enter key to start the test")
    parser.add_option(
        '-n',
        '--no-rc',
        action='store_true',
        dest='no_rc',
        help="Don't load config files from /etc/yandex-tank and ~/.yandex-tank")
    parser.add_option(
        '-o',
        '--option',
        action='append',
        help="Set config option, multiple options accepted, example: -o 'shellexec.start=pwd'"
    )
    parser.add_option(
        '-q',
        '--quiet',
        action='store_true',
        help="Less console output, only errors and warnings")
    parser.add_option(
        '-s',
        '--scheduled-start',
        action='store',
        dest='scheduled_start',
        help="Start test at specified time, format 'YYYY-MM-DD hh:mm:ss', date part is optional"
    )
    parser.add_option(
        '-v',
        '--verbose',
        action='store_true',
        help="More console output, +debug messages")
    parser.add_option(
        '-p',
        '--patch-cfg',
        action='append',
        help='Patch config with yaml snippet (similar to -o, but has full compatibility to\
        and the exact scheme of yaml format config)',
        dest='patches'
    )

    # FIXME: restore completion helper
    # completion_helper = CompletionHelperOptionParser()
    # completion_helper.handle_request(parser)

    options, ammofiles = parser.parse_args()
    ammofile = ammofiles[0] if len(ammofiles) > 0 else None

    init_logging(options.error_log, options.verbose, options.quiet)

    cli_kwargs = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {}
    if options.ignore_lock:
        cli_kwargs.setdefault('core', {})['ignore_lock'] = options.ignore_lock

    if ammofile:
        logging.debug("Ammofile: %s", ammofile)
        cli_kwargs['phantom'] = {
            'use_caching': False,
            'ammofile': ammofile
        }
    try:
        worker = TankWorker([resource_manager.resource_filename(cfg) for cfg in options.config],
                            options.option,
                            options.patches,
                            [cli_kwargs],
                            options.no_rc,
                            ammo_file=ammofile if ammofile else None)
    except ValidationError as e:
        logging.error('Config validation error:\n{}'.format(e.errors))
        return
    worker.start()
    try:
        worker.join()
    except KeyboardInterrupt:
        worker.stop()
        worker.join()
    # try:
    #     worker.configure()
    #     rc = worker.perform_test()
    #     sys.exit(rc)
    # except Exception as ex:
    #     worker.core._collect_artifacts()
    #     logging.error("Exception: %s", ex)
    #     logging.debug("Exception: %s", traceback.format_exc(ex))
    #     sys.exit(1)


def init_logging(events_log_fname, verbose, quiet):
    """ Set up logging, as it is very important for console tool """
    logger = logging.getLogger('')
    logger.setLevel(logging.DEBUG)
    # log_filename = log
    # events_log_fname = self.options.error_log

    # create file handler which logs even debug messages
    # if log_filename:
    #     file_handler = logging.FileHandler(log_filename)
    #     file_handler.setLevel(logging.DEBUG)
    #     file_handler.setFormatter(
    #         logging.Formatter(
    #             "%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d\t%(message)s"
    #         ))
    #     logger.addHandler(file_handler)

    # create file handler which logs error messages

    if events_log_fname:
        err_file_handler = logging.FileHandler(events_log_fname)
        err_file_handler.setLevel(logging.WARNING)
        err_file_handler.setFormatter(
            logging.Formatter(
                "%(asctime)s\t%(message)s"
            ))
        logger.addHandler(err_file_handler)

    # create console handler with a higher log level
    console_handler = logging.StreamHandler(sys.stdout)
    stderr_hdl = logging.StreamHandler(sys.stderr)

    fmt_verbose = logging.Formatter(<commit_msg>rm commented lines<commit_after>import logging
import sys
from optparse import OptionParser

from netort.resource import manager as resource_manager
from yandextank.core.consoleworker import TankWorker
from yandextank.validator.validator import ValidationError


def main():
    parser = OptionParser()
    parser.add_option(
        '-c',
        '--config',
        action='append',
        help="Path to INI file containing run options, multiple options accepted",
        default=[]
    )
    parser.add_option(
        '-f',
        '--fail-lock',
        action='store_true',
        dest='lock_fail',
        help="Don't wait for lock to release, fail test instead")
    parser.add_option(
        '-i',
        '--ignore-lock',
        action='store_true',
        dest='ignore_lock',
        help="Ignore lock files from concurrent instances, has precedence before --lock-fail"
    )
    parser.add_option(
        '-k',
        '--lock-dir',
        action='store',
        dest='lock_dir',
        type="string",
        help="Directory for lock file")
    parser.add_option(
        '-l',
        '--log',
        action='store',
        default="tank.log",
        help="Tank log file location")
    parser.add_option(
        '--error_log',
        action='store',
        dest='error_log',
        default="tank_errors.log",
        help="Tank errors log file location")
    parser.add_option(
        '-m',
        '--manual-start',
        action='store_true',
        dest='manual_start',
        help="Wait for Enter key to start the test")
    parser.add_option(
        '-n',
        '--no-rc',
        action='store_true',
        dest='no_rc',
        help="Don't load config files from /etc/yandex-tank and ~/.yandex-tank")
    parser.add_option(
        '-o',
        '--option',
        action='append',
        help="Set config option, multiple options accepted, example: -o 'shellexec.start=pwd'"
    )
    parser.add_option(
        '-q',
        '--quiet',
        action='store_true',
        help="Less console output, only errors and warnings")
    parser.add_option(
        '-s',
        '--scheduled-start',
        action='store',
        dest='scheduled_start',
        help="Start test at specified time, format 'YYYY-MM-DD hh:mm:ss', date part is optional"
    )
    parser.add_option(
        '-v',
        '--verbose',
        action='store_true',
        help="More console output, +debug messages")
    parser.add_option(
        '-p',
        '--patch-cfg',
        action='append',
        help='Patch config with yaml snippet (similar to -o, but has full compatibility to\
        and the exact scheme of yaml format config)',
        dest='patches'
    )

    # FIXME: restore completion helper
    # completion_helper = CompletionHelperOptionParser()
    # completion_helper.handle_request(parser)

    options, ammofiles = parser.parse_args()
    ammofile = ammofiles[0] if len(ammofiles) > 0 else None

    init_logging(options.error_log, options.verbose, options.quiet)

    cli_kwargs = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {}
    if options.ignore_lock:
        cli_kwargs.setdefault('core', {})['ignore_lock'] = options.ignore_lock

    if ammofile:
        logging.debug("Ammofile: %s", ammofile)
        cli_kwargs['phantom'] = {
            'use_caching': False,
            'ammofile': ammofile
        }
    try:
        worker = TankWorker([resource_manager.resource_filename(cfg) for cfg in options.config],
                            options.option,
                            options.patches,
                            [cli_kwargs],
                            options.no_rc,
                            ammo_file=ammofile if ammofile else None)
    except ValidationError as e:
        logging.error('Config validation error:\n{}'.format(e.errors))
        return
    worker.start()
    try:
        worker.join()
    except KeyboardInterrupt:
        worker.stop()
        worker.join()
    # try:
    #     worker.configure()
    #     rc = worker.perform_test()
    #     sys.exit(rc)
    # except Exception as ex:
    #     worker.core._collect_artifacts()
    #     logging.error("Exception: %s", ex)
    #     logging.debug("Exception: %s", traceback.format_exc(ex))
    #     sys.exit(1)


def init_logging(events_log_fname, verbose, quiet):
    """ Set up logging, as it is very important for console tool """
    logger = logging.getLogger('')
    logger.setLevel(logging.DEBUG)

    # create file handler which logs error messages
    if events_log_fname:
        err_file_handler = logging.FileHandler(events_log_fname)
        err_file_handler.setLevel(logging.WARNING)
        err_file_handler.setFormatter(
            logging.Formatter(
                "%(asctime)s\t%(message)s"
            ))
        logger.addHandler(err_file_handler)

    # create console handler with a higher log level
    console_handler = logging.StreamHandler(sys.stdout)
    stderr_hdl = logging.StreamHandler(sys.stderr)

    fmt_verbose = logging.Formatter(
{'fraction_non_alphanumeric': 0.11787503544758483, 'fraction_numerical': 0.0005671613574061821, 'mean_word_length': 2.2070324340709306, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4001798', 'n_tokens_mistral': 3117, 'n_tokens_neox': 3047, 'n_words': 703}