<commit_before>import copy
import pprint

from thinglang.lexer.tokens import LexicalGroupEnd
from thinglang.lexer.tokens.base import LexicalIndent
from thinglang.parser.patterns import REPLACEMENT_PASSES
from thinglang.parser.symbols import RootSymbol
from thinglang.parser.symbols.classes import ThingDefinition


def parse(lexical_groups):
    """
    The parser takes a list<list<lexical-token>>, where each inner list (list<lexical-token>) is referred to as a lexical group (why? because).
    Each lexical group is parsed independently, through a process that reduces the group iteratively until no further reductions can be made.
    Then, lexical groups are processed in order, creating a tree structure based on indentation.
    Each TokenNode in the tree contains n-children, where each child is also a token node.
    The root node of this tree is returned
    """
    stack = [RootSymbol()]
    processed_groups = [x for x in [parse_group(group) for group in lexical_groups] if x]

    for idx, node in enumerate(processed_groups):

        if isinstance(node, ThingDefinition):
            stack = [stack[0]]
<commit_msg>Another patch to deal with method definitions following an open block<commit_after>import copy
import pprint

from thinglang.lexer.tokens import LexicalGroupEnd
from thinglang.lexer.tokens.base import LexicalIndent
from thinglang.parser.patterns import REPLACEMENT_PASSES
from thinglang.parser.symbols import RootSymbol
from thinglang.parser.symbols.classes import ThingDefinition, MethodDefinition


def parse(lexical_groups):
    """
    The parser takes a list<list<lexical-token>>, where each inner list (list<lexical-token>) is referred to as a lexical group (why? because).
    Each lexical group is parsed independently, through a process that reduces the group iteratively until no further reductions can be made.
    Then, lexical groups are processed in order, creating a tree structure based on indentation.
    Each TokenNode in the tree contains n-children, where each child is also a token node.
    The root node of this tree is returned
    """
    stack = [RootSymbol()]
    processed_groups = [x for x in [parse_group(group) for group in lexical_groups] if x]

    for idx, node in enumerate(processed_groups):

        if isinstance(node, ThingDefinition):
            stack = [stack[0]]

        if isinstance(node, MethodDefinition):
            stack = [stack[0], stack[1]]

{'fraction_non_alphanumeric': 0.06463565253190613, 'fraction_numerical': 0.0016467682173734047, 'mean_word_length': 4.148305084745763, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '211612', 'n_tokens_mistral': 657, 'n_tokens_neox': 628, 'n_words': 271}