<commit_before>from __future__ import with_statement

from behave import model, i18n

DEFAULT_LANGUAGE = 'en'


def parse_file(filename, language=None):
    with open(filename, 'rb') as f:
        # file encoding is assumed to be utf8. Oh, yes.
        data = f.read().decode('utf8')
    return parse_feature(data, language, filename)


def parse_feature(data, language=None, filename=None):
    # ALL data operated on by the parser MUST be unicode
    assert isinstance(data, unicode)

    try:
        result = Parser(language).parse(data, filename)
    except ParserError, e:
        e.filename = filename
        raise

    return result


class ParserError(Exception):
    def __init__(self, message, line, filename=None):
        if line:
            message += ' at line %d' % line
        super(ParserError, self).__init__(message)
        self.line = line
        self.filename = filename

    def __str__(self):
        if self.filename:
            return 'Failed to parse "%s": %s' % (self.filename, self.args[0])
        return 'Failed to parse <string>: %s' % self.args[0]


class Parser(object):
    def __init__(self, language=None):
        self.language = language
        self.reset()

    def reset(self):
        # This can probably go away.
        if self.language:
            self.keywords = i18n.languages[self.language]
        else:
            self.keywords = None

        self.state = 'init'
        self.line = 0
        self.last_step = None
        self.multiline_start = None
        self.multiline_leading = None
        self.multiline_terminator = None

        self.filename = None
        self.feature = None
        self.statement = None
        self.tags = []
        self.lines = []
        self.table = None
        self.examples = None

    def parse(self, data, filename=None):
        self.reset()

        self.filename = filename

        for line in data.split('\n'):
            self.line += 1
            if not line.strip() and not self.state == 'multiline':
                continue
            self.action(line)

        if self.table:
            self.action_table('')

        feature = self.feature
        feature.parser = self
        self.reset()
        return feature

    def action(self, line):
        if line.strip().startswith('#') and not self.state == 'multiline':
            if self.keywords or self.state != 'init' or self.tags:
                return

            line = line.strip()[1:].strip()
            if line.lstrip().lower().startswith('language:'):
                language = line[9:].strip()
                self.language = language
                self.keywords = i18n.languages[language]
            return

        func = getattr(self, 'action_' + self.state, None)
        if func is None:
            raise ParserError('Parser in unknown state ' + self.state,
                              self.line)
        if not func(line):
            raise ParserError("Parser failure in state " + self.state,
                              self.line)

    def action_init(self, line):
        line = line.strip()

        if line.startswith('@'):
            self.tags.extend([model.Tag(tag.strip(), self.line)
                              for tag in line[1:].split('@')])
            return True
        feature_kwd = self.match_keyword('feature', line)
        if feature_kwd:
            name = line[len(feature_kwd) + 1:].strip()
            self.feature = model.Feature(self.filename, self.line, feature_kwd,
                                         name, tags=self.tags)
            self.tags = []
            self.state = 'feature'
            return True
        return False

    def action_feature(self, line):
        line = line.strip()

        if line.startswith('@'):
            self.tags.extend([model.Tag(tag.strip(), self.line)
                              for tag in line[1:].split('@')])
            return True

        background_kwd = self.match_keyword('background', line)
        if background_kwd:
            name = line[len(background_kwd) + 1:].strip()
            self.statement = model.Background(self.filename, self.line,
                                              background_kwd, name)
            self.feature.background = self.statement
            self.state = 'steps'
            return True

        scenario_kwd = self.match_keyword('scenario', line)
        if scenario_kwd:
            name = line[len(scenario_kwd) + 1:].strip()
            self.statement = model.Scenario(self.filename, self.line,
                                            scenario_kwd, name, tags=self.tags)
            self.tags = []
            self.feature.add_scenario(self.statement)
            self.state = 'steps'
            return True

        scenario_outline_kwd = self.match_keyword('scenario_outline', line)
        if scenario_outline_kwd:
            name = line[len(scenario_outline_kwd) + 1:].strip()
            self.statement = model.ScenarioOutline(self.filename, self.line,
                                                   scenario_outline_kwd, name,
                                                   tags=self.tags)
            self.tags = []
            self.feature.add_scenario(self.statement)
            self.state = 'steps'
            return True

        self.feature.description.append(line)
        return True

    def action_steps(self, line):
        stripped = line.lstrip()
        if stripped.startswith('"""') or stripped.startswith("'''"):
            self.state = 'multiline'
            self.multiline_start = self.line
            self.multiline_terminator = stripped[:3]
            self.multiline_leading = line.index(stripped[0])
            return True

        line = line.strip()
<commit_msg>Added support for descriptions in scenarios and scenario outlines so that the parser does not raise an exception if they are encountered<commit_after>from __future__ import with_statement

from behave import model, i18n

DEFAULT_LANGUAGE = 'en'


def parse_file(filename, language=None):
    with open(filename, 'rb') as f:
        # file encoding is assumed to be utf8. Oh, yes.
        data = f.read().decode('utf8')
    return parse_feature(data, language, filename)


def parse_feature(data, language=None, filename=None):
    # ALL data operated on by the parser MUST be unicode
    assert isinstance(data, unicode)

    try:
        result = Parser(language).parse(data, filename)
    except ParserError, e:
        e.filename = filename
        raise

    return result


class ParserError(Exception):
    def __init__(self, message, line, filename=None):
        if line:
            message += ' at line %d' % line
        super(ParserError, self).__init__(message)
        self.line = line
        self.filename = filename

    def __str__(self):
        if self.filename:
            return 'Failed to parse "%s": %s' % (self.filename, self.args[0])
        return 'Failed to parse <string>: %s' % self.args[0]


class Parser(object):
    def __init__(self, language=None):
        self.language = language
        self.reset()

    def reset(self):
        # This can probably go away.
        if self.language:
            self.keywords = i18n.languages[self.language]
        else:
            self.keywords = None

        self.state = 'init'
        self.line = 0
        self.last_step = None
        self.multiline_start = None
        self.multiline_leading = None
        self.multiline_terminator = None

        self.filename = None
        self.feature = None
        self.statement = None
        self.tags = []
        self.lines = []
        self.table = None
        self.examples = None

    def parse(self, data, filename=None):
        self.reset()

        self.filename = filename

        for line in data.split('\n'):
            self.line += 1
            if not line.strip() and not self.state == 'multiline':
                continue
            self.action(line)

        if self.table:
            self.action_table('')

        feature = self.feature
        feature.parser = self
        self.reset()
        return feature

    def action(self, line):
        if line.strip().startswith('#') and not self.state == 'multiline':
            if self.keywords or self.state != 'init' or self.tags:
                return

            line = line.strip()[1:].strip()
            if line.lstrip().lower().startswith('language:'):
                language = line[9:].strip()
                self.language = language
                self.keywords = i18n.languages[language]
            return

        func = getattr(self, 'action_' + self.state, None)
        if func is None:
            raise ParserError('Parser in unknown state ' + self.state,
                              self.line)
        if not func(line):
            raise ParserError("Parser failure in state " + self.state,
                              self.line)

    def action_init(self, line):
        line = line.strip()

        if line.startswith('@'):
            self.tags.extend([model.Tag(tag.strip(), self.line)
                              for tag in line[1:].split('@')])
            return True
        feature_kwd = self.match_keyword('feature', line)
        if feature_kwd:
            name = line[len(feature_kwd) + 1:].strip()
            self.feature = model.Feature(self.filename, self.line, feature_kwd,
                                         name, tags=self.tags)
            self.tags = []
            self.state = 'feature'
            return True
        return False

    def action_feature(self, line):
        line = line.strip()

        if line.startswith('@'):
            self.tags.extend([model.Tag(tag.strip(), self.line)
                              for tag in line[1:].split('@')])
            return True

        background_kwd = self.match_keyword('background', line)
        if background_kwd:
            name = line[len(background_kwd) + 1:].strip()
            self.statement = model.Background(self.filename, self.line,
                                              background_kwd, name)
            self.feature.background = self.statement
            self.state = 'steps'
            return True

        scenario_kwd = self.match_keyword('scenario', line)
        if scenario_kwd:
            name = line[len(scenario_kwd) + 1:].strip()
            self.statement = model.Scenario(self.filename, self.line,
                                            scenario_kwd, name, tags=self.tags)
            self.tags = []
            self.state = 'scenario'
            return True

        scenario_outline_kwd = self.match_keyword('scenario_outline', line)
        if scenario_outline_kwd:
            name = line[len(scenario_outline_kwd) + 1:].strip()
            self.statement = model.ScenarioOutline(self.filename, self.line,
                                                   scenario_outline_kwd, name,
                                                   tags=self.tags)
            self.tags = []
            self.state = 'scenario'
            return True

        self.feature.description.append(line)
        return True

    def action_scenario(self, line):
        line = line.strip()

        step = self.parse_step(line)
        if step:
            self.feature.add_scenario(self.statement)
            self.state = 'steps'
            self.statement.steps.append(step)
            return True

        self.statement.description.append(line)
        return True

    def action_steps(self, line):
        stripped = line.lstrip()
        if stripped.startswith('"""') or stripped.startswith("'''"):
            self.state = 'multiline'
            self.multiline_start = self.line
            self.multiline_terminator = stripped[:3]
            self.multiline_leading = line.index(stripped[0])
            return True

        line = line.strip()

{'fraction_non_alphanumeric': 0.10445494431319609, 'fraction_numerical': 0.003712453594330071, 'mean_word_length': 1.7024623803009575, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4691242', 'n_tokens_mistral': 3255, 'n_tokens_neox': 3142, 'n_words': 773}