<commit_before>import imp
import os
import re

import rhc.config as config_file
from rhc.micro_fsm.fsm_micro import create as create_machine

import logging
log = logging.getLogger(__name__)


def to_args(line):
    args = []
    kwargs = {}
    for tok in line.split():
        nvp = re.split(r'(?<!\\)=', tok, 1)  # split by first non-escaped equal sign
        if len(nvp) == 2:
            n, v = nvp
            n = n.replace('\=', '=')
            v = v.replace('\=', '=')
            kwargs[n] = v
        else:
            args.append(tok.replace('\=', '='))
    return args, kwargs


def _prepare_import(import_fname):
    if '.' in import_fname and os.path.sep not in import_fname:  # path is dot separated
        parts = import_fname.split('.')
        extension = ''
        if parts[-1] == 'micro':
            parts = parts[:-1]  # save '.micro' extension if exists
            extension = '.micro'
        sink, path, sink = imp.find_module(parts[0])  # use module-based location
        import_fname = os.path.join(path, *parts[1:]) + extension
    elif not import_fname.startswith(os.path.sep):  # path is relative
        import_fname = os.path.join(os.getcwd(), import_fname)
    return import_fname


def load(micro='micro', files=None, lines=None):

    if files is None:
        files = []
    if lines is None:
        lines = []

    if isinstance(micro, str):
        if micro in files:
            raise Exception('a micro file (in this case, %s) cannot be recursively imported' % micro)
        files.append(micro)
        micro = open(micro).readlines()
    elif isinstance(micro, list):
        files.append('dict')
    else:
        raise Exception('Invalid micro file type %s' % type(micro))

    fname = files[-1]

    for num, line in enumerate(micro, start=1):
        line = line.split('#', 1)[0].strip()
        if len(line):
            line = line.split(' ', 1)
            if len(line) == 1:
                raise Exception('too few tokens, file=%s, line=%d' % (fname, num))
            if line[0].lower() == 'import':
                import_fname = _prepare_import(line[1])
                load(import_fname, files, lines)
            else:
                lines.append((fname, num, line[0], line[1]))
    return lines


class Parser(object):

    def __init__(self):
        self.fsm = create_machine(
            add_config=self.act_add_config,
            add_config_server=self.act_add_config_server,
            add_connection=self.act_add_connection,
            add_header=self.act_add_header,
            add_method=self.act_add_method,
            add_old_server=self.act_add_old_server,
            add_optional=self.act_add_optional,
            add_required=self.act_add_required,
            add_resource=self.act_add_resource,
            add_resource_header=self.act_add_resource_header,
            add_route=self.act_add_route,
            add_server=self.act_add_server,
            add_setup=self.act_add_setup,
            add_teardown=self.act_add_teardown,
        )
        self.error = None
        self.fsm.state = 'init'
        self.config = config_file.Config()
        self.setup = None
        self.teardown = None
        self.connections = {}
        self._config_servers = {}
        self.servers = {}

    @property
    def is_new(self):
        return len(self._config_servers) == 0

    @classmethod
    def parse(cls, micro='micro'):
        parser = cls()
        for fname, num, parser.event, parser.line in load(micro):
            parser.args, parser.kwargs = to_args(parser.line)
            if not parser.fsm.handle(parser.event.lower()):
                raise Exception("Unexpected directive '%s', file=%s, line=%d" % (parser.event, fname, num))
            if parser.error:
                raise Exception('%s, line=%d' % (parser.error, num))
        return parser

    def _add_config(self, name, **kwargs):
        self.config._define(name, **kwargs)

    def act_add_config(self):
        if self.kwargs.get('validate') is not None:
            try:
                self.kwargs['validate'] = {
                    'int': config_file.validate_int,
                    'bool': config_file.validate_bool,
                    'file': config_file.validate_file,
                }[self.kwargs['validate']]
            except KeyError:
                raise Exception("validate must be one of 'int', 'bool', 'file'")
        config = Config(*self.args, **self.kwargs)
        self._add_config(config.name, **config.kwargs)

    def act_add_config_server(self):
        name, port = self.args
        self._config_servers[name] = port

    def act_add_connection(self):
        connection = Connection(*self.args, **self.kwargs)
        if connection.name in self.connections:
            self.error = 'duplicate CONNECTION name: %s' % connection.name
        elif connection.url is None and connection.code is None:
            self.error = 'connection must have an url or code defined: %s' % connection.name
        else:
            self.connections[connection.name] = connection
            self.connection = connection
            if connection.url is not None:
                self._add_config('connection.%s.url' % connection.name, value=connection.url)
            self._add_config('connection.%s.is_active' % connection.name, value=True, validator=config_file.validate_bool)<commit_msg>process default config values with validator<commit_after>import imp
import os
import re

import rhc.config as config_file
from rhc.micro_fsm.fsm_micro import create as create_machine

import logging
log = logging.getLogger(__name__)


def to_args(line):
    args = []
    kwargs = {}
    for tok in line.split():
        nvp = re.split(r'(?<!\\)=', tok, 1)  # split by first non-escaped equal sign
        if len(nvp) == 2:
            n, v = nvp
            n = n.replace('\=', '=')
            v = v.replace('\=', '=')
            kwargs[n] = v
        else:
            args.append(tok.replace('\=', '='))
    return args, kwargs


def _prepare_import(import_fname):
    if '.' in import_fname and os.path.sep not in import_fname:  # path is dot separated
        parts = import_fname.split('.')
        extension = ''
        if parts[-1] == 'micro':
            parts = parts[:-1]  # save '.micro' extension if exists
            extension = '.micro'
        sink, path, sink = imp.find_module(parts[0])  # use module-based location
        import_fname = os.path.join(path, *parts[1:]) + extension
    elif not import_fname.startswith(os.path.sep):  # path is relative
        import_fname = os.path.join(os.getcwd(), import_fname)
    return import_fname


def load(micro='micro', files=None, lines=None):

    if files is None:
        files = []
    if lines is None:
        lines = []

    if isinstance(micro, str):
        if micro in files:
            raise Exception('a micro file (in this case, %s) cannot be recursively imported' % micro)
        files.append(micro)
        micro = open(micro).readlines()
    elif isinstance(micro, list):
        files.append('dict')
    else:
        raise Exception('Invalid micro file type %s' % type(micro))

    fname = files[-1]

    for num, line in enumerate(micro, start=1):
        line = line.split('#', 1)[0].strip()
        if len(line):
            line = line.split(' ', 1)
            if len(line) == 1:
                raise Exception('too few tokens, file=%s, line=%d' % (fname, num))
            if line[0].lower() == 'import':
                import_fname = _prepare_import(line[1])
                load(import_fname, files, lines)
            else:
                lines.append((fname, num, line[0], line[1]))
    return lines


class Parser(object):

    def __init__(self):
        self.fsm = create_machine(
            add_config=self.act_add_config,
            add_config_server=self.act_add_config_server,
            add_connection=self.act_add_connection,
            add_header=self.act_add_header,
            add_method=self.act_add_method,
            add_old_server=self.act_add_old_server,
            add_optional=self.act_add_optional,
            add_required=self.act_add_required,
            add_resource=self.act_add_resource,
            add_resource_header=self.act_add_resource_header,
            add_route=self.act_add_route,
            add_server=self.act_add_server,
            add_setup=self.act_add_setup,
            add_teardown=self.act_add_teardown,
        )
        self.error = None
        self.fsm.state = 'init'
        self.config = config_file.Config()
        self.setup = None
        self.teardown = None
        self.connections = {}
        self._config_servers = {}
        self.servers = {}

    @property
    def is_new(self):
        return len(self._config_servers) == 0

    @classmethod
    def parse(cls, micro='micro'):
        parser = cls()
        for fname, num, parser.event, parser.line in load(micro):
            parser.args, parser.kwargs = to_args(parser.line)
            if not parser.fsm.handle(parser.event.lower()):
                raise Exception("Unexpected directive '%s', file=%s, line=%d" % (parser.event, fname, num))
            if parser.error:
                raise Exception('%s, line=%d' % (parser.error, num))
        return parser

    def _add_config(self, name, **kwargs):
        self.config._define(name, **kwargs)

    def act_add_config(self):
        if self.kwargs.get('validate') is not None:
            try:
                self.kwargs['validate'] = {
                    'int': config_file.validate_int,
                    'bool': config_file.validate_bool,
                    'file': config_file.validate_file,
                }[self.kwargs['validate']]
            except KeyError:
                raise Exception("validate must be one of 'int', 'bool', 'file'")
        config = Config(*self.args, **self.kwargs)
        if config.default and config.validate:
            config.default = config.validate(config.default)
        self._add_config(config.name, **config.kwargs)

    def act_add_config_server(self):
        name, port = self.args
        self._config_servers[name] = port

    def act_add_connection(self):
        connection = Connection(*self.args, **self.kwargs)
        if connection.name in self.connections:
            self.error = 'duplicate CONNECTION name: %s' % connection.name
        elif connection.url is None and connection.code is None:
            self.error = 'connection must have an url or code defined: %s' % connection.name
        else:
            self.connections[connection.name] = connection
            self.connection = connection
            if connection.url is not None:
                self._add_config('connection.%s.url' % connection.name, value=connection.url)
            self._add_config('connection.%s.is_active' % connection.name, value=True, validator=config_file.validate_bool)
{'fraction_non_alphanumeric': 0.11780369112110917, 'fraction_numerical': 0.003121843724175925, 'mean_word_length': 2.1885245901639343, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6238957', 'n_tokens_mistral': 3290, 'n_tokens_neox': 3190, 'n_words': 750}