<commit_before>from collections import namedtuple, OrderedDict
import casadi as ca
import numpy as np
import itertools
import logging
import re

from pymoca import ast
from .alias_relation import AliasRelation

logger = logging.getLogger("pymoca")

CASADI_COMPARISON_DEPTH = 100
CASADI_ATTRIBUTES = [attr for attr in ast.Symbol.ATTRIBUTES if not attr == 'unit']


class Variable:
    def __init__(self, symbol, python_type=float, aliases=None):
        if aliases is None:
            aliases = []
        self.symbol = symbol
        self.python_type = python_type
        self.aliases = aliases

        # Default attribute values
        self.value = np.nan
        self.start = 0
        self.min = -np.inf
        self.max = np.inf
        self.nominal = 1
        self.fixed = False

    def __str__(self):
        return self.symbol.name()

    def __repr__(self):
        return '{}[{},{}]:{}'.format(self.symbol.name(), self.symbol.size1(), self.symbol.size2(), self.python_type.__name__)

    def to_dict(self):
        d = {}
        d['name'] = self.symbol.name()
        d['shape'] = (self.symbol.size1(), self.symbol.size2())
        d['python_type'] = self.python_type
        d['aliases'] = self.aliases
        return d

    @classmethod
    def from_dict(cls, d):
        variable = cls(ca.MX.sym(d['name'], *d['shape']), d['python_type'])
        variable.aliases = d['aliases']
        return variable


DelayArgument = namedtuple('DelayArgument', ['expr', 'duration'])


# noinspection PyUnresolvedReferences
class Model:

    def __init__(self):
        self.states = []
        self.der_states = []
        self.alg_states = []
        self.inputs = []
        self.outputs = []
        self.constants = []
        self.parameters = []
        self.equations = []
        self.initial_equations = []
        self.time = ca.MX.sym('time')
        self.delay_states = []
        self.delay_arguments = []
        self._expand_mx_func = lambda x: x

    def __str__(self):
        r = ""
        r += "Model\n"
        r += "time: " + str(self.time) + "\n"
        r += "states: " + str(self.states) + "\n"
        r += "der_states: " + str(self.der_states) + "\n"
        r += "alg_states: " + str(self.alg_states) + "\n"
        r += "inputs: " + str(self.inputs) + "\n"
        r += "outputs: " + str(self.outputs) + "\n"
        r += "constants: " + str(self.constants) + "\n"
        r += "parameters: " + str(self.parameters) + "\n"
        r += "equations: " + str(self.equations) + "\n"
        r += "initial equations: " + str(self.initial_equations) + "\n"
        return r

    def check_balanced(self):
        n_states = sum(v.symbol.size1() * v.symbol.size2() for v in itertools.chain(self.states, self.alg_states))
        n_equations = sum(e.size1() * e.size2() for e in self.dae_residual_function.mx_out())
        if n_states == n_equations:
            logger.info("System is balanced.")
        else:
            logger.warning(
                "System is not balanced.  "
                "Number of states is {}, number of equations is {}.".format(
                    n_states, n_equations))

    @staticmethod
    def _symbols(l):
        return [v.symbol for v in l]

    def _substitute_delay_arguments(self, delay_arguments, symbols, values):
        exprs = ca.substitute([ca.MX(argument.expr) for argument in delay_arguments], symbols, values)
        durations = ca.substitute([ca.MX(argument.duration) for argument in delay_arguments], symbols, values)
        return [DelayArgument(expr, duration) for expr, duration in zip(exprs, durations)]

    def simplify(self, options):
        if options.get('replace_parameter_expressions', False):
            logger.info("Replacing parameter expressions")

            simple_parameters, symbols, values = [], [], []
            for p in self.parameters:
                value = ca.MX(p.value)
                if value.is_constant():
                    simple_parameters.append(p)
                else:
                    symbols.append(p.symbol)
                    values.append(value)

            self.parameters = simple_parameters

            if len(values) > 0:
                # Resolve expressions that include other, non-simple parameter
                # expressions.
                converged = False
                while not converged:
                    new_values = ca.substitute(values, symbols, values)
                    converged = ca.is_equal(ca.veccat(*values), ca.veccat(*new_values), CASADI_COMPARISON_DEPTH)
                    values = new_values

                if len(self.equations) > 0:
                    self.equations = ca.substitute(self.equations, symbols, values)
                if len(self.initial_equations) > 0:
                    self.initial_equations = ca.substitute(self.initial_equations, symbols, values)
                if len(self.delay_arguments) > 0:
                    self.delay_arguments = self._substitute_delay_arguments(self.delay_arguments, symbols, values)

                # Replace parameter expressions in metadata
                for variable in itertools.chain(self.states, self.alg_states, self.inputs, self.parameters, self.constants):
                    for attribute in CASADI_ATTRIBUTES:
                        value = getattr(variable, attribute)
                        if isinstance(value, ca.MX) and not value.is_constant():
                            [value] = ca.substitute([value], symbols, values)
                            setattr(variable, attribute, value)

        if options.get('replace_constant_expressions', False):
            logger.info("Replacing constant expressions")

            simple_constants, symbols, values = [], [], []
            for c in self.constants:
                value = ca.MX(c.value)
                if value.is_constant():
                    simple_constants.append(c)
                else:
                    symbols.append(c.symbol)
                    values.append(value)

            self.constants = simple_constants

            if len(values) > 0:
                # Resolve expressions that include other, non-simple parameter
                # expressions.
                converged = False
                while not converged:
                    new_values = ca.substitute(values, symbols, values)
                    converged = ca.is_equal(ca.veccat(*values), ca.veccat(*new_values), CASADI_COMPARISON_DEPTH)
                    values = new_values

                if len(self.equations) > 0:
                    self.equations = ca.substitute(self.equations, symbols, values)
                if len(self.initial_equations) > 0:
                    self.initial_equations = ca.substitute(self.initial_equations, symbols, values)
                if len(self.delay_arguments) > 0:
                    self.delay_arguments = self._substitute_delay_arguments(self.delay_arguments, symbols, values)

                # Replace constant expressions in metadata
                for variable in itertools.chain(self.states, self.alg_states, self.inputs, self.parameters, self.constants):
                    for attribute in CASADI_ATTRIBUTES:
                        value = getattr(variable, attribute)
                        if isinstance(value, ca.MX) and not value.is_constant():
                            [value] = ca.substitute([value], symbols, values)
                            setattr(variable, attribute, value)

        if options.get('eliminate_constant_assignments', False):
            logger.info("Elimating constant variable assignments")

            alg_states = OrderedDict([(s.symbol.name(), s) for s in self.alg_states])

            reduced_equations = []
            for eq in self.equations:
                if eq.is_symbolic() and eq.name() in alg_states:
                    constant = alg_states.pop(eq.name())
                    constant.value = 0.0

                    self.constants.append(constant)

                    # Skip this equation
                    continue

                if eq.n_dep() == 2 and (eq.is_op(ca.OP_SUB) or eq.is_op(ca.OP_ADD)):
                    if eq.dep(0).is_symbolic() and eq.dep(0).name() in alg_states and eq.dep(1).is_constant():
                        variable = eq.dep(0)
                        value = eq.dep(1)
                    elif eq.dep(1).is_symbolic() and eq.dep(1).name() in alg_states and eq.dep(0).is_constant():
                        variable = eq.dep(1)
                        value = eq.dep(0)
                    else:
                        variable = None
                        value = None

                    if variable is not None:
                        constant = alg_states.pop(variable.name())

                        if eq.is_op(ca.OP_SUB):
                            constant.value = value
                        else:
                            constant.value = -value

                        self.constants.append(constant)

                        # Skip this equation
                        continue

                # Keep this equation
                reduced_equations.append(eq)

            # Eliminate alias variables
            self.alg_states = list(alg_states.values())
            self.equations = reduced_equations

        if options.get('replace_parameter_values', False):
            logger.info("Replacing parameter values")

            # N.B. Any parameter expression elimination must be done first.
            unspecified_parameters, symbols, values = [], [], []
            for p in self.parameters:
                if ca.MX(p.value).is_constant() and ca.MX(p.value).is_regular():
                    symbols.append(p.symbol)
                    values.append(p.value)
                else:
                    unspecified_parameters.append(p)

            if len(self.equations) > 0:
                self.equations = ca.substitute(self.equations, symbols, values)
            if len(self.initial_equations) > 0:
                self.initial_equations = ca.substitute(self.initial_equations, symbols, values)
            self.parameters = unspecified_parameters

            # Replace parameter values in metadata
            for variable in itertools.chain(self.states, self.alg_states, self.inputs, self.parameters, self.constants):
                for attribute in CASADI_ATTRIBUTES:
                    value = getattr(variable, attribute)
                    if isinstance(value, ca.MX) and not value.is_constant():
                        [value] = ca.substitute([value], symbols, values)
                        setattr(variable, attribute, value)

        if options.get('replace_constant_values', False):
            logger.info("Replacing constant values")

            # N.B. Any parameter expression elimination must be done first.
            symbols = self._symbols(self.constants)
            values = [v.value for v in self.constants]
            if len(self.equations) > 0:
                self.equations = ca.substitute(self.equations, symbols, values)
            if len(self.initial_equations) > 0:
                self.initial_equations = ca.substitute(self.initial_equations, symbols, values)
            if len(self.delay_arguments) > 0:
                self.delay_arguments = self._substitute_delay_arguments(self.delay_arguments, symbols, values)
            self.constants = []

            # Replace constant values in metadata
            for variable in itertools.chain(self.states, self.alg_states, self.inputs, self.parameters, self.constants):
                for attribute in CASADI_ATTRIBUTES:
                    value = getattr(variable, attribute)
                    if isinstance(value, ca.MX) and not value.is_constant():
                        [value] = ca.substitute([value], symbols, values)
                        setattr(variable, attribute, value)

        if options.get('eliminable_variable_expression', None) is not None:
            logger.info("Elimating variables that match the regular expression {}".format(options['eliminable_variable_expression']))

            p = re.compile(options['eliminable_variable_expression'])

            alg_states = OrderedDict([(s.symbol.name(), s) for s in self.alg_states])

            variables = []
            values = []

            reduced_equations = []
            for eq in self.equations:
                if eq.is_symbolic() and eq.name() in alg_states and p.match(eq.name()):
                    variables.append(eq)
                    values.append(0.0)
                    del alg_states[eq.name()]
                    # Skip this equation
                    continue

                if eq.n_dep() == 2 and (eq.is_op(ca.OP_SUB) or eq.is_op(ca.OP_ADD)):
                    if eq.dep(0).is_symbolic() and eq.dep(0).name() in alg_states and p.match(eq.dep(0).name()):
                        variable = eq.dep(0)
                        value = eq.dep(1)
                    elif eq.dep(1).is_symbolic() and eq.dep(1).name() in alg_states and p.match(eq.dep(1).name()):
                        variable = eq.dep(1)
                        value = eq.dep(0)
                    else:
                        variable = None
                        value = None

                    if variable is not None:
                        del alg_states[variable.name()]

                        variables.append(variable)
                        if eq.is_op(ca.OP_SUB):
                            values.append(value)
                        else:
                            values.append(-value)

                        # Skip this equation
                        continue

                # Keep this equation
                reduced_equations.append(eq)

            # Eliminate alias variables
            self.alg_states = list(alg_states.values())
            self.equations = reduced_equations

            if len(self.equations) > 0:
                self.equations = ca.substitute(self.equations, variables, values)
            if len(self.initial_equations) > 0:
                self.initial_equations = ca.substitute(self.initial_equations, variables, values)
            if len(self.delay_arguments) > 0:
                self.delay_arguments = self._substitute_delay_arguments(self.delay_arguments, variables, values)

        if options.get('expand_vectors', False):
            logger.info("Expanding vectors")

            symbols = []
            values = []

            for l in ['states', 'der_states', 'alg_states', 'inputs', 'parameters', 'constants']:
                old_vars = getattr(self, l)
                new_vars = []
                for old_var in old_vars:
                    if old_var.symbol.numel() > 1:
                        rows = []
                        for i in range(old_var.symbol.size1()):
                            cols = []
                            for j in range(old_var.symbol.size2()):
                                if old_var.symbol.size1() > 1 and old_var.symbol.size2() > 1:
                                    component_symbol = ca.MX.sym('{}[{},{}]'.format(old_var.symbol.name(), i, j))
                                elif old_var.symbol.size1() > 1:
                                    component_symbol = ca.MX.sym('{}[{}]'.format(old_var.symbol.name(), i))
                                elif old_var.symbol.size2() > 1:
                                    component_symbol = ca.MX.sym('{}[{}]'.format(old_var.symbol.name(), j))
                                else:
                                    raise AssertionError<commit_msg>Use 1-based indexing when expanding vectors<commit_after>from collections import namedtuple, OrderedDict
import casadi as ca
import numpy as np
import itertools
import logging
import re

from pymoca import ast
from .alias_relation import AliasRelation

logger = logging.getLogger("pymoca")

CASADI_COMPARISON_DEPTH = 100
CASADI_ATTRIBUTES = [attr for attr in ast.Symbol.ATTRIBUTES if not attr == 'unit']


class Variable:
    def __init__(self, symbol, python_type=float, aliases=None):
        if aliases is None:
            aliases = []
        self.symbol = symbol
        self.python_type = python_type
        self.aliases = aliases

        # Default attribute values
        self.value = np.nan
        self.start = 0
        self.min = -np.inf
        self.max = np.inf
        self.nominal = 1
        self.fixed = False

    def __str__(self):
        return self.symbol.name()

    def __repr__(self):
        return '{}[{},{}]:{}'.format(self.symbol.name(), self.symbol.size1(), self.symbol.size2(), self.python_type.__name__)

    def to_dict(self):
        d = {}
        d['name'] = self.symbol.name()
        d['shape'] = (self.symbol.size1(), self.symbol.size2())
        d['python_type'] = self.python_type
        d['aliases'] = self.aliases
        return d

    @classmethod
    def from_dict(cls, d):
        variable = cls(ca.MX.sym(d['name'], *d['shape']), d['python_type'])
        variable.aliases = d['aliases']
        return variable


DelayArgument = namedtuple('DelayArgument', ['expr', 'duration'])


# noinspection PyUnresolvedReferences
class Model:

    def __init__(self):
        self.states = []
        self.der_states = []
        self.alg_states = []
        self.inputs = []
        self.outputs = []
        self.constants = []
        self.parameters = []
        self.equations = []
        self.initial_equations = []
        self.time = ca.MX.sym('time')
        self.delay_states = []
        self.delay_arguments = []
        self._expand_mx_func = lambda x: x

    def __str__(self):
        r = ""
        r += "Model\n"
        r += "time: " + str(self.time) + "\n"
        r += "states: " + str(self.states) + "\n"
        r += "der_states: " + str(self.der_states) + "\n"
        r += "alg_states: " + str(self.alg_states) + "\n"
        r += "inputs: " + str(self.inputs) + "\n"
        r += "outputs: " + str(self.outputs) + "\n"
        r += "constants: " + str(self.constants) + "\n"
        r += "parameters: " + str(self.parameters) + "\n"
        r += "equations: " + str(self.equations) + "\n"
        r += "initial equations: " + str(self.initial_equations) + "\n"
        return r

    def check_balanced(self):
        n_states = sum(v.symbol.size1() * v.symbol.size2() for v in itertools.chain(self.states, self.alg_states))
        n_equations = sum(e.size1() * e.size2() for e in self.dae_residual_function.mx_out())
        if n_states == n_equations:
            logger.info("System is balanced.")
        else:
            logger.warning(
                "System is not balanced.  "
                "Number of states is {}, number of equations is {}.".format(
                    n_states, n_equations))

    @staticmethod
    def _symbols(l):
        return [v.symbol for v in l]

    def _substitute_delay_arguments(self, delay_arguments, symbols, values):
        exprs = ca.substitute([ca.MX(argument.expr) for argument in delay_arguments], symbols, values)
        durations = ca.substitute([ca.MX(argument.duration) for argument in delay_arguments], symbols, values)
        return [DelayArgument(expr, duration) for expr, duration in zip(exprs, durations)]

    def simplify(self, options):
        if options.get('replace_parameter_expressions', False):
            logger.info("Replacing parameter expressions")

            simple_parameters, symbols, values = [], [], []
            for p in self.parameters:
                value = ca.MX(p.value)
                if value.is_constant():
                    simple_parameters.append(p)
                else:
                    symbols.append(p.symbol)
                    values.append(value)

            self.parameters = simple_parameters

            if len(values) > 0:
                # Resolve expressions that include other, non-simple parameter
                # expressions.
                converged = False
                while not converged:
                    new_values = ca.substitute(values, symbols, values)
                    converged = ca.is_equal(ca.veccat(*values), ca.veccat(*new_values), CASADI_COMPARISON_DEPTH)
                    values = new_values

                if len(self.equations) > 0:
                    self.equations = ca.substitute(self.equations, symbols, values)
                if len(self.initial_equations) > 0:
                    self.initial_equations = ca.substitute(self.initial_equations, symbols, values)
                if len(self.delay_arguments) > 0:
                    self.delay_arguments = self._substitute_delay_arguments(self.delay_arguments, symbols, values)

                # Replace parameter expressions in metadata
                for variable in itertools.chain(self.states, self.alg_states, self.inputs, self.parameters, self.constants):
                    for attribute in CASADI_ATTRIBUTES:
                        value = getattr(variable, attribute)
                        if isinstance(value, ca.MX) and not value.is_constant():
                            [value] = ca.substitute([value], symbols, values)
                            setattr(variable, attribute, value)

        if options.get('replace_constant_expressions', False):
            logger.info("Replacing constant expressions")

            simple_constants, symbols, values = [], [], []
            for c in self.constants:
                value = ca.MX(c.value)
                if value.is_constant():
                    simple_constants.append(c)
                else:
                    symbols.append(c.symbol)
                    values.append(value)

            self.constants = simple_constants

            if len(values) > 0:
                # Resolve expressions that include other, non-simple parameter
                # expressions.
                converged = False
                while not converged:
                    new_values = ca.substitute(values, symbols, values)
                    converged = ca.is_equal(ca.veccat(*values), ca.veccat(*new_values), CASADI_COMPARISON_DEPTH)
                    values = new_values

                if len(self.equations) > 0:
                    self.equations = ca.substitute(self.equations, symbols, values)
                if len(self.initial_equations) > 0:
                    self.initial_equations = ca.substitute(self.initial_equations, symbols, values)
                if len(self.delay_arguments) > 0:
                    self.delay_arguments = self._substitute_delay_arguments(self.delay_arguments, symbols, values)

                # Replace constant expressions in metadata
                for variable in itertools.chain(self.states, self.alg_states, self.inputs, self.parameters, self.constants):
                    for attribute in CASADI_ATTRIBUTES:
                        value = getattr(variable, attribute)
                        if isinstance(value, ca.MX) and not value.is_constant():
                            [value] = ca.substitute([value], symbols, values)
                            setattr(variable, attribute, value)

        if options.get('eliminate_constant_assignments', False):
            logger.info("Elimating constant variable assignments")

            alg_states = OrderedDict([(s.symbol.name(), s) for s in self.alg_states])

            reduced_equations = []
            for eq in self.equations:
                if eq.is_symbolic() and eq.name() in alg_states:
                    constant = alg_states.pop(eq.name())
                    constant.value = 0.0

                    self.constants.append(constant)

                    # Skip this equation
                    continue

                if eq.n_dep() == 2 and (eq.is_op(ca.OP_SUB) or eq.is_op(ca.OP_ADD)):
                    if eq.dep(0).is_symbolic() and eq.dep(0).name() in alg_states and eq.dep(1).is_constant():
                        variable = eq.dep(0)
                        value = eq.dep(1)
                    elif eq.dep(1).is_symbolic() and eq.dep(1).name() in alg_states and eq.dep(0).is_constant():
                        variable = eq.dep(1)
                        value = eq.dep(0)
                    else:
                        variable = None
                        value = None

                    if variable is not None:
                        constant = alg_states.pop(variable.name())

                        if eq.is_op(ca.OP_SUB):
                            constant.value = value
                        else:
                            constant.value = -value

                        self.constants.append(constant)

                        # Skip this equation
                        continue

                # Keep this equation
                reduced_equations.append(eq)

            # Eliminate alias variables
            self.alg_states = list(alg_states.values())
            self.equations = reduced_equations

        if options.get('replace_parameter_values', False):
            logger.info("Replacing parameter values")

            # N.B. Any parameter expression elimination must be done first.
            unspecified_parameters, symbols, values = [], [], []
            for p in self.parameters:
                if ca.MX(p.value).is_constant() and ca.MX(p.value).is_regular():
                    symbols.append(p.symbol)
                    values.append(p.value)
                else:
                    unspecified_parameters.append(p)

            if len(self.equations) > 0:
                self.equations = ca.substitute(self.equations, symbols, values)
            if len(self.initial_equations) > 0:
                self.initial_equations = ca.substitute(self.initial_equations, symbols, values)
            self.parameters = unspecified_parameters

            # Replace parameter values in metadata
            for variable in itertools.chain(self.states, self.alg_states, self.inputs, self.parameters, self.constants):
                for attribute in CASADI_ATTRIBUTES:
                    value = getattr(variable, attribute)
                    if isinstance(value, ca.MX) and not value.is_constant():
                        [value] = ca.substitute([value], symbols, values)
                        setattr(variable, attribute, value)

        if options.get('replace_constant_values', False):
            logger.info("Replacing constant values")

            # N.B. Any parameter expression elimination must be done first.
            symbols = self._symbols(self.constants)
            values = [v.value for v in self.constants]
            if len(self.equations) > 0:
                self.equations = ca.substitute(self.equations, symbols, values)
            if len(self.initial_equations) > 0:
                self.initial_equations = ca.substitute(self.initial_equations, symbols, values)
            if len(self.delay_arguments) > 0:
                self.delay_arguments = self._substitute_delay_arguments(self.delay_arguments, symbols, values)
            self.constants = []

            # Replace constant values in metadata
            for variable in itertools.chain(self.states, self.alg_states, self.inputs, self.parameters, self.constants):
                for attribute in CASADI_ATTRIBUTES:
                    value = getattr(variable, attribute)
                    if isinstance(value, ca.MX) and not value.is_constant():
                        [value] = ca.substitute([value], symbols, values)
                        setattr(variable, attribute, value)

        if options.get('eliminable_variable_expression', None) is not None:
            logger.info("Elimating variables that match the regular expression {}".format(options['eliminable_variable_expression']))

            p = re.compile(options['eliminable_variable_expression'])

            alg_states = OrderedDict([(s.symbol.name(), s) for s in self.alg_states])

            variables = []
            values = []

            reduced_equations = []
            for eq in self.equations:
                if eq.is_symbolic() and eq.name() in alg_states and p.match(eq.name()):
                    variables.append(eq)
                    values.append(0.0)
                    del alg_states[eq.name()]
                    # Skip this equation
                    continue

                if eq.n_dep() == 2 and (eq.is_op(ca.OP_SUB) or eq.is_op(ca.OP_ADD)):
                    if eq.dep(0).is_symbolic() and eq.dep(0).name() in alg_states and p.match(eq.dep(0).name()):
                        variable = eq.dep(0)
                        value = eq.dep(1)
                    elif eq.dep(1).is_symbolic() and eq.dep(1).name() in alg_states and p.match(eq.dep(1).name()):
                        variable = eq.dep(1)
                        value = eq.dep(0)
                    else:
                        variable = None
                        value = None

                    if variable is not None:
                        del alg_states[variable.name()]

                        variables.append(variable)
                        if eq.is_op(ca.OP_SUB):
                            values.append(value)
                        else:
                            values.append(-value)

                        # Skip this equation
                        continue

                # Keep this equation
                reduced_equations.append(eq)

            # Eliminate alias variables
            self.alg_states = list(alg_states.values())
            self.equations = reduced_equations

            if len(self.equations) > 0:
                self.equations = ca.substitute(self.equations, variables, values)
            if len(self.initial_equations) > 0:
                self.initial_equations = ca.substitute(self.initial_equations, variables, values)
            if len(self.delay_arguments) > 0:
                self.delay_arguments = self._substitute_delay_arguments(self.delay_arguments, variables, values)

        if options.get('expand_vectors', False):
            logger.info("Expanding vectors")

            symbols = []
            values = []

            for l in ['states', 'der_states', 'alg_states', 'inputs', 'parameters', 'constants']:
                old_vars = getattr(self, l)
                new_vars = []
                for old_var in old_vars:
                    if old_var.symbol.numel() > 1:
                        rows = []
                        for i in range(old_var.symbol.size1()):
                            cols = []
                            for j in range(old_var.symbol.size2()):
                                if old_var.symbol.size1() > 1 and old_var.symbol.size2() > 1:
                                    component_symbol = ca.MX.sym('{}[{},{}]'.format(old_var.symbol.name(), i+1, j+1))
                                elif old_var.symbol.size1() > 1:
                                    component_symbol = ca.MX.sym('{}[{}]'.format(old_var.symbol.name(), i+1))
                                elif old_var.symbol.size2() > 1:
                                    component_symbol = ca.MX.sym('{}[{}]'.format(old_var.symbol.name(), j+1))
                                else:
                                    raise AssertionError
{'fraction_non_alphanumeric': 0.10382075774088083, 'fraction_numerical': 0.004391307135072761, 'mean_word_length': 1.6286123515039177, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5813507', 'n_tokens_mistral': 8584, 'n_tokens_neox': 8093, 'n_words': 1852}