<commit_before>from thinglang.lexer.tokens.base import LexicalIdentifier
from thinglang.parser.symbols import Transient
from thinglang.parser.symbols.arithmetic import ArithmeticOperation
from thinglang.parser.symbols.base import AssignmentOperation
from thinglang.parser.symbols.functions import MethodCall, Access, ArgumentList
from thinglang.parser.symbols.logic import IterativeLoop, Loop
from thinglang.parser.symbols.types import CastOperation
from thinglang.utils.tree_utils import TreeTraversal, inspects
from thinglang.utils.union_types import POTENTIALLY_OBTAINABLE


class Simplifier(TreeTraversal):


    @inspects(IterativeLoop)
    def unwrap_iterative_loops(self, node):
        generator_id, generator_assignment = self.create_transient(node.generator, node, LexicalIdentifier('Range'))
        initial_assignment = AssignmentOperation.create(node.name, MethodCall([Access([generator_id, LexicalIdentifier('next')]), ArgumentList()]), LexicalIdentifier('number')).contextify(node.parent)
        iterative_assignment = AssignmentOperation.create(node.name, MethodCall(
            [Access([generator_id, LexicalIdentifier('next')]), ArgumentList()]))

        node.insert_before(generator_assignment)
        node.insert_before(initial_assignment)


        node.children.append(iterative_assignment)
        loop = Loop([None, node.name]).contextify(node.parent).populate(node.children)

        node.insert_before(loop)
        node.remove()



    @inspects(predicate=lambda x: isinstance(getattr(x, 'value', None), POTENTIALLY_OBTAINABLE))
    def inspect_obtainable_operations(self, node):
        return self.unwrap_method_calls(node.value, node)

    def unwrap_method_calls(self, method_call, node, parent_call=None):
        if not isinstance(method_call, POTENTIALLY_OBTAINABLE):
            return

        for argument in method_call.arguments:
            if isinstance(argument, (MethodCall, CastOperation)):
                self.unwrap_method_calls(argument, node, parent_call=method_call)
            if isinstance(argument, ArithmeticOperation):
                for x in argument.arguments:
                    self.unwrap_method_calls(x, node, parent_call=argument)

        if parent_call is not None:
            id, assignment = self.create_transient(method_call, node)
            node.insert_before(assignment)
            parent_call.replace(method_call, id)

    @classmethod
    def is_compound(cls, node):
        if not node:
            return False
        return (isinstance(node, MethodCall) and any(isinstance(arg, MethodCall) for arg in node.arguments.value)) or \
               (isinstance(node, AssignmentOperation) and cls.is_compound(node.value))

    @staticmethod
    def create_transient(value, parent, type=None):
        local_id = Transient().contextify(parent.context)
        return local_id, AssignmentOperation([type, local_id, None, value]).contextify(parent.parent)
<commit_msg>Changed create_transient to return id, declaration and assignment ops<commit_after>from thinglang.lexer.tokens.base import LexicalIdentifier
from thinglang.parser.symbols import Transient
from thinglang.parser.symbols.arithmetic import ArithmeticOperation
from thinglang.parser.symbols.base import AssignmentOperation
from thinglang.parser.symbols.functions import MethodCall, Access, ArgumentList
from thinglang.parser.symbols.logic import IterativeLoop, Loop
from thinglang.parser.symbols.types import CastOperation
from thinglang.utils.tree_utils import TreeTraversal, inspects
from thinglang.utils.union_types import POTENTIALLY_OBTAINABLE


class Simplifier(TreeTraversal):


    @inspects(IterativeLoop)
    def unwrap_iterative_loops(self, node):
        generator_id, generator_declaration, generator_assignment = self.create_transient(node.generator, node, LexicalIdentifier('Range'))
        initial_assignment = AssignmentOperation.create(node.name, MethodCall([Access([generator_id, LexicalIdentifier('next')]), ArgumentList()]), LexicalIdentifier('number')).contextify(node.parent)
        iterative_assignment = AssignmentOperation.create(node.name, MethodCall(
            [Access([generator_id, LexicalIdentifier('next')]), ArgumentList()]))

        node.insert_before(generator_assignment)
        node.insert_before(initial_assignment)


        node.children.append(iterative_assignment)
        loop = Loop([None, node.name]).contextify(node.parent).populate(node.children)

        node.insert_before(loop)
        node.remove()



    @inspects(predicate=lambda x: isinstance(getattr(x, 'value', None), POTENTIALLY_OBTAINABLE))
    def inspect_obtainable_operations(self, node):
        return self.unwrap_method_calls(node.value, node)

    def unwrap_method_calls(self, method_call, node, parent_call=None):
        if not isinstance(method_call, POTENTIALLY_OBTAINABLE):
            return

        for argument in method_call.arguments:
            if isinstance(argument, (MethodCall, CastOperation)):
                self.unwrap_method_calls(argument, node, parent_call=method_call)
            if isinstance(argument, ArithmeticOperation):
                for x in argument.arguments:
                    self.unwrap_method_calls(x, node, parent_call=argument)

        if parent_call is not None:
            id, declaration, assignment = self.create_transient(method_call, node)
            node.insert_before(declaration)
            parent_call.replace(method_call, id)

    @classmethod
    def is_compound(cls, node):
        if not node:
            return False
        return (isinstance(node, MethodCall) and any(isinstance(arg, MethodCall) for arg in node.arguments.value)) or \
               (isinstance(node, AssignmentOperation) and cls.is_compound(node.value))

    @staticmethod
    def create_transient(value, parent, type=None):
        local_id = Transient().contextify(parent.context)
        return local_id, AssignmentOperation([type, local_id, None, value]).contextify(parent.parent), AssignmentOperation([local_id, None, value]).contextify(parent.parent)

{'fraction_non_alphanumeric': 0.09017341040462427, 'fraction_numerical': 0.0, 'mean_word_length': 4.132203389830509, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '753548', 'n_tokens_mistral': 1699, 'n_tokens_neox': 1654, 'n_words': 339}