<commit_before>
from ..nodes import ParseTree
# from ..sl_exceptions import TyrianSyntaxError
from .grammar_parser import GrammarParser


class Parser(object):
    """
    At the moment simply a wrapper for GrammarParser,
    though in the future will perform parsing duties
    """
    def __init__(self,
                 token_defs=None,
                 raw_grammar=None,
                 grammar_mapping=None,
                 nodes=None,
                 settings=None):
        self.grammar_parser = GrammarParser(
            raw_grammar=raw_grammar,
            token_defs=token_defs,
            grammar_mapping=grammar_mapping,
            nodes=nodes,
            settings=settings
        )

    def parse(self, lexed):
        return ParseTree(self._parse(lexed))

    def _parse(self, lexed):
        start_token = self.grammar_parser.settings['start_token']

        base_grammar = self.grammar_parser.grammars[start_token.upper()]

        index = 0
        results = []

        while index < len(lexed):
            result = base_grammar.check(lexed[index:], '<list>')

            assert result['result'], (result, ' '.join([x['token'] for x in lexed[index:]]))
            del result['tokens']

            what_was_consumed = ' '.join([x['token'] for x in lexed[index:index+result['consumed']]])
            results.append((result, what_was_consumed))

            index += result['consumed']

        return results


    # def _parse(self, tokens):
    #     nodes = []

    #     tokens = list(filter(bool, tokens))

    #     while tokens:
    #         token = tokens.pop(0)

    #         possible = self.determine_possible(tokens)
    #         if possible:
    #             logger.debug('possible: {}'.format(possible))
    #         raise Exception()

    #         if token[0] == 'OPEN_BRACKET':
    #             logger.debug("LISTNODE START")
    #             nodes.append(self._parse(tokens))
    #         elif token[0] == 'CLOSE_BRACKET':
    #             logger.debug('LISTNODE END')
    #             return ListNode(nodes)

    #         elif token[0] == 'ID':
    #             logger.debug('IDNODE')
    #             nodes.append(IDNode(token[1]))

    #         elif token[0] == 'NUMBER':
    #             logger.debug("NUMBER")
    #             nodes.append(NumberNode(token[1]))
    #         else:
    #             raise TyrianSyntaxError(token)

    #     return ListNode(nodes)
<commit_msg>The parser now works :D<commit_after>
from ..utils import reduce
from ..nodes import ParseTree, ContainerNode
# from ..sl_exceptions import TyrianSyntaxError
from .grammar_parser import GrammarParser


class Parser(object):
    """

    """
    def __init__(self,
                 token_defs=None,
                 raw_grammar=None,
                 grammar_mapping=None,
                 nodes=None,
                 settings=None):
        self.grammar_parser = GrammarParser(
            raw_grammar=raw_grammar,
            token_defs=token_defs,
            grammar_mapping=grammar_mapping,
            nodes=nodes,
            settings=settings
        )

    def parse(self, lexed):
        start_token = self.grammar_parser.settings['start_token']

        base_grammar = self.grammar_parser.grammars[start_token.upper()]

        index = 0
        results = []

        while index < len(lexed):
            result = base_grammar.check(lexed[index:], '<list>')

            assert result['result'], (result, ' '.join([x['token'] for x in lexed[index:]]))
            del result['tokens']

            results.append(result)

            index += result['consumed']

        processed = self._process(results)
        return ParseTree(processed)

    def _process(self, parsed):
        processed = []
        for result in parsed:
            parse_tree = result['parse_tree']
            parse_tree = reduce(parse_tree, can_return_single=True)

            parse_tree = ContainerNode(parse_tree)

            processed.append(parse_tree)
        return ContainerNode(processed, strip=False)

{'fraction_non_alphanumeric': 0.1092707045735476, 'fraction_numerical': 0.0022249690976514215, 'mean_word_length': 1.964102564102564, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4408539', 'n_tokens_mistral': 1161, 'n_tokens_neox': 1119, 'n_words': 225}