<commit_before># -*- coding: utf-8 -*-
# See LICENSE file for copyright and license details


import copy
import ast
import datatype


class Variable(object):
    def __init__(self, name, datatype):
        self.name = name
        self.datatype = datatype


class Constant(object):
    def __init__(self, value, datatype):
        self.value = value
        self.datatype = datatype


class Function(object):
    def __init__(
            self,
            name,
            interface,
            block_list,
            constant_list,
            variable_list,
            expression_list):
        self.name = name
        self.interface = interface
        self.block_list = block_list
        self.constant_list = constant_list
        self.variable_list = variable_list
        self.expression_list = expression_list


class IfStatement(object):
    def __init__(self, expression_id, if_branch_id, else_branch_id=None):
        self.expression_id = expression_id
        self.if_branch_id = if_branch_id
        self.else_branch_id = else_branch_id


class ForStatement(object):
    def __init__(self, expression_id, branch_id):
        self.expression_id = expression_id
        self.branch_id = branch_id


class VariableDeclarationStatement(object):
    def __init__(self, variable_id, expression_id):
        self.variable_id = variable_id
        self.expression_id = expression_id


class AssignStatement(object):
    def __init__(self, variable_id, expression_id):
        self.variable_id = variable_id
        self.expression_id = expression_id


class ReturnStatement(object):
    def __init__(self, expression_id):
        self.expression_id = expression_id


class FunctionCallStatement(object):
    def __init__(self, expression_id):
        self.expression_id = expression_id


class FunctionCallExpression(object):
    def __init__(self, name, argument_id_list, result_id):
        self.name = name
        self.argument_id_list = argument_id_list
        self.result_id = result_id


class LinkToNumberConstant(object):
    def __init__(self, id):
        self.id = id


class LinkToVariable(object):
    def __init__(self, id):
        self.id = id


class LinkToParameter(object):
    def __init__(self, name):
        self.name = name


class LinkToFunctionCall(object):
    def __init__(self, id):
        self.id = id


class Table(object):

    def __init__(
            self,
            declaration_list,
            identifier_list,
            import_list):
        self.declaration_list = declaration_list
        self.identifier_list = identifier_list
        self.import_list = import_list

    def _parse_number(self, number):
        assert isinstance(number, ast.Number)
        self.declaration_list[-1].constant_list.append(
            Constant(
                datatype=datatype.SimpleDataType('int'),
                value=number.value,
            ),
        )
        return LinkToNumberConstant(
            id=len(self.declaration_list[-1].constant_list) - 1)

    def _parse_identifier(self, identidier_node):
        assert isinstance(identidier_node, ast.Identifier)
        # TODO: find this identifier
        return LinkToParameter(name=identidier_node.name)

    def _parse_function_call(self, function_call_node):
        assert isinstance(function_call_node, ast.FunctionCall)
        last_declaration = self.declaration_list[-1]
        varname = 'tmp_' + str(len(last_declaration.variable_list))
        # TODO: get datatype from some global symtable
        # return_type = 'int'
        return_type = datatype.SimpleDataType(name='int')
        last_declaration.variable_list.append(
            Variable(name=varname, datatype=return_type))
        result_id = LinkToVariable(
            id=len(last_declaration.variable_list) - 1)
        argument_id_list = []
        for argument in function_call_node.argument_list:
            id = self._parse_expression(argument)
            argument_id_list.append(id)
        # TODO: parse function_call_node.expression!
        assert isinstance(function_call_node.expression, ast.Identifier)
        # function_name = function_call_node.expression.value
        function_name = function_call_node.expression.name
        last_declaration.expression_list.append(
            FunctionCallExpression(
                name=function_name,
                argument_id_list=argument_id_list,
                result_id=result_id,
            )
        )
        return LinkToFunctionCall(
            id=len(last_declaration.expression_list) - 1)<commit_msg>table.py: _parse_function_call: Removed commented-out line<commit_after># -*- coding: utf-8 -*-
# See LICENSE file for copyright and license details


import copy
import ast
import datatype


class Variable(object):
    def __init__(self, name, datatype):
        self.name = name
        self.datatype = datatype


class Constant(object):
    def __init__(self, value, datatype):
        self.value = value
        self.datatype = datatype


class Function(object):
    def __init__(
            self,
            name,
            interface,
            block_list,
            constant_list,
            variable_list,
            expression_list):
        self.name = name
        self.interface = interface
        self.block_list = block_list
        self.constant_list = constant_list
        self.variable_list = variable_list
        self.expression_list = expression_list


class IfStatement(object):
    def __init__(self, expression_id, if_branch_id, else_branch_id=None):
        self.expression_id = expression_id
        self.if_branch_id = if_branch_id
        self.else_branch_id = else_branch_id


class ForStatement(object):
    def __init__(self, expression_id, branch_id):
        self.expression_id = expression_id
        self.branch_id = branch_id


class VariableDeclarationStatement(object):
    def __init__(self, variable_id, expression_id):
        self.variable_id = variable_id
        self.expression_id = expression_id


class AssignStatement(object):
    def __init__(self, variable_id, expression_id):
        self.variable_id = variable_id
        self.expression_id = expression_id


class ReturnStatement(object):
    def __init__(self, expression_id):
        self.expression_id = expression_id


class FunctionCallStatement(object):
    def __init__(self, expression_id):
        self.expression_id = expression_id


class FunctionCallExpression(object):
    def __init__(self, name, argument_id_list, result_id):
        self.name = name
        self.argument_id_list = argument_id_list
        self.result_id = result_id


class LinkToNumberConstant(object):
    def __init__(self, id):
        self.id = id


class LinkToVariable(object):
    def __init__(self, id):
        self.id = id


class LinkToParameter(object):
    def __init__(self, name):
        self.name = name


class LinkToFunctionCall(object):
    def __init__(self, id):
        self.id = id


class Table(object):

    def __init__(
            self,
            declaration_list,
            identifier_list,
            import_list):
        self.declaration_list = declaration_list
        self.identifier_list = identifier_list
        self.import_list = import_list

    def _parse_number(self, number):
        assert isinstance(number, ast.Number)
        self.declaration_list[-1].constant_list.append(
            Constant(
                datatype=datatype.SimpleDataType('int'),
                value=number.value,
            ),
        )
        return LinkToNumberConstant(
            id=len(self.declaration_list[-1].constant_list) - 1)

    def _parse_identifier(self, identidier_node):
        assert isinstance(identidier_node, ast.Identifier)
        # TODO: find this identifier
        return LinkToParameter(name=identidier_node.name)

    def _parse_function_call(self, function_call_node):
        assert isinstance(function_call_node, ast.FunctionCall)
        last_declaration = self.declaration_list[-1]
        varname = 'tmp_' + str(len(last_declaration.variable_list))
        # TODO: get datatype from some global symtable
        # return_type = 'int'
        return_type = datatype.SimpleDataType(name='int')
        last_declaration.variable_list.append(
            Variable(name=varname, datatype=return_type))
        result_id = LinkToVariable(
            id=len(last_declaration.variable_list) - 1)
        argument_id_list = []
        for argument in function_call_node.argument_list:
            id = self._parse_expression(argument)
            argument_id_list.append(id)
        # TODO: parse function_call_node.expression!
        assert isinstance(function_call_node.expression, ast.Identifier)
        function_name = function_call_node.expression.name
        last_declaration.expression_list.append(
            FunctionCallExpression(
                name=function_name,
                argument_id_list=argument_id_list,
                result_id=result_id,
            )
        )
        return LinkToFunctionCall(
            id=len(last_declaration.expression_list) - 1)
{'fraction_non_alphanumeric': 0.07693153093746566, 'fraction_numerical': 0.001538630618749313, 'mean_word_length': 2.615415176797775, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2779503', 'n_tokens_mistral': 2563, 'n_tokens_neox': 2505, 'n_words': 509}