<commit_before>import os
import random
import zlib
from collections import MutableSequence, Iterable
from numbers import Real
import operator
from functools import reduce

import numpy as np
import bottleneck as bn

from .instance import *
from Orange.data import io

class RowInstance(Instance):
    def __init__(self, table, row_index):
        super().__init__(table.domain)
        self._x = table._X[row_index]
        self._y = table._Y[row_index]
        self._values = np.hstack((self._x, self._y))
        self._metas = table._metas[row_index]
        self.row_index = row_index
        self.table = table

    def get_class(self):
        self._check_single_class()
        if self.table.domain.class_var:
            return Value(self.table.domain.class_var, self._y[0])

    def set_class(self, value):
        self._check_single_class()
        if not isinstance(value, Real):
            self._y[0] = self.table.domain.class_var.to_val(value)
        else:
            self._y[0] = value
        self._values[len(self.table.domain.attributes)] = self._y[0]

    def get_classes(self):
        return (Value(var, value) for var, value in
            zip(self.table.domain.class_vars, self._y))

    def set_weight(self, weight):
        if self.table._W is None:
            self.table.set_weights()
        self.table._W[self.row_index] = weight

    def get_weight(self):
        if not self.table._W:
            raise ValueError("Instances in the referenced table have no weights")
        return self.table._W[self.row_index]

    def __setitem__(self, key, value):
        if not isinstance(key, int):
            key = self.domain.index(key)
        if isinstance(value, str):
            var = self.domain[key]
            value = var.to_val(value)
        if key >= 0:
            if not isinstance(value, Real):
                raise TypeError("Expected primitive value, got '%s'" %
                                type(value).__name__)
            if key < len(self._x):
                self._values[key] = self._x[key] = value
            else:
                self._values[key] = self._y[key - len(self._x)] = value
        else:
            self._metas[-1-key] = value


class Columns:
    def __init__(self, domain):
        for v in domain:
            setattr(self, v.name.replace(" ", "_"), v)
        for v in domain.metas:
            setattr(self, v.name.replace(" ", "_"), v)

class Table(MutableSequence):
    def __new__(cls, *args, **argkw):
        self = None
        if not args:
            if not args and not argkw:
                self = super().__new__(cls)
            elif "filename" in argkw:
                self = cls.read_data(argkw["filename"])
        elif len(args) == 1:
            arg = args[0]
            if isinstance(arg, str):
                self = Table.read_data(args[0])
            elif isinstance(arg, Domain):
                self = Table.new_from_domain(arg)
        elif 2 <= len(args) <= 4:
            if isinstance(args[0], Domain) and \
               all(isinstance(arg, np.ndarray) for arg in args[1:]):
                self = Table.new_from_numpy(*args)
        if self is None:
            raise ValueError("Invalid arguments for Table.__new__")
        self.clear_cache()
        return self

    @staticmethod
    def new_from_domain(domain, n_rows=0, weights=True):
        self = Table.__new__(Table)
        self.domain = domain
        self.n_rows = n_rows
        self._X = np.zeros((n_rows, len(domain.attributes)))
        self._Y = np.zeros((n_rows, len(domain.class_vars)))
        if weights:
            self._W = np.ones(n_rows)
        else:
            self._W = np.empty((n_rows, 0))
        self._metas = np.empty((n_rows, len(self.domain.metas)), object)
        return self


    @staticmethod
    def compose_cols_from(table, row_indices, src_cols, n_rows):
        if not len(src_cols):
            return np.zeros((n_rows, 0), dtype=table._X.dtype)

        n_src_attrs = len(table.domain.attributes)
        if all(0 <= x < n_src_attrs for x in src_cols):
            return table._X[row_indices, src_cols]
        if all(x < 0 for x in src_cols):
            return table._metas[row_indices, [-1-x for x in src_cols]]
        if all(x >= n_src_attrs for x in src_cols):
            return table._Y[row_indices, [x-n_src_attrs for x in src_cols]]

        a = np.empty((n_rows, len(src_cols)), dtype=table._X.dtype)
        for i, col in enumerate(src_cols):
            if col < 0:
                a[:, i] = table._metas[row_indices, -1 - col]
            elif col < n_src_attrs:
                a[:, i] = table._X[row_indices, col]
            else:
                a[:, i] = table._Y[row_indices, col - n_src_attrs]
        return a


    @staticmethod
    def new_from_table(domain, table, row_indices=...):
        if domain == table.domain:
            return Table.new_from_table_rows(table, row_indices)

        if isinstance(row_indices, slice):
            start, stop, stride = row_indices.indices(len(table._X))
            n_rows = (stop - start) / stride
        elif row_indices is ...:
            n_rows = len(table._X)
        else:
            n_rows = len(row_indices)

        self = Table.__new__(Table)
        self.domain = domain
        conversion = domain.get_conversion(table.domain)
        self._X = Table.compose_cols_from(
            table, row_indices, conversion.attributes, n_rows)
        self._Y = Table.compose_cols_from(
            table, row_indices, conversion.classes, n_rows)
        self._metas = Table.compose_cols_from(
            table, row_indices, conversion.metas, n_rows)
        self._W = np.array(table._W[row_indices])
        return self


    @staticmethod
    def new_from_table_rows(table, row_indices):
        self = Table.__new__(Table)
        self.domain = table.domain
        self._X = table._X[row_indices]
        self._Y = table._Y[row_indices]
        self._metas = table._metas[row_indices]
        self._W = table._W[row_indices]
        return self

    @staticmethod
    def new_from_numpy(domain, X, Y=None, metas=None, W=None):
        if Y is None:
            Y = X[:, len(domain.attributes):]
            X = X[:, :len(domain.attributes)]
        if X.shape[1] != len(domain.attributes):
            raise ValueError("Invalid number of variable columns ({} != {}".
                format(X.shape[1], len(domain.attributes)))
        if Y.shape[1] != len(domain.class_vars):
            raise ValueError("Invalid number of class columns ({} != {}".
                format(Y.shape[1], len(domain.class_vars)))
        if metas is not None and metas.shape[1] != len(domain.metas):
            raise ValueError("Invalid number of meta attribute columns ({} != {}".
                format(metas.shape[1], len(domain.metas)))
        if metas is None:
            metas = np.empty((X.shape[0], 0), object)
        if W is None:
            W = np.empty((X.shape[0], 0))<commit_msg>Refactored new_from_numpy.<commit_after>import os
import random
import zlib
from collections import MutableSequence, Iterable
from numbers import Real
import operator
from functools import reduce

import numpy as np
import bottleneck as bn

from .instance import *
from Orange.data import io

class RowInstance(Instance):
    def __init__(self, table, row_index):
        super().__init__(table.domain)
        self._x = table._X[row_index]
        self._y = table._Y[row_index]
        self._values = np.hstack((self._x, self._y))
        self._metas = table._metas[row_index]
        self.row_index = row_index
        self.table = table

    def get_class(self):
        self._check_single_class()
        if self.table.domain.class_var:
            return Value(self.table.domain.class_var, self._y[0])

    def set_class(self, value):
        self._check_single_class()
        if not isinstance(value, Real):
            self._y[0] = self.table.domain.class_var.to_val(value)
        else:
            self._y[0] = value
        self._values[len(self.table.domain.attributes)] = self._y[0]

    def get_classes(self):
        return (Value(var, value) for var, value in
            zip(self.table.domain.class_vars, self._y))

    def set_weight(self, weight):
        if self.table._W is None:
            self.table.set_weights()
        self.table._W[self.row_index] = weight

    def get_weight(self):
        if not self.table._W:
            raise ValueError("Instances in the referenced table have no weights")
        return self.table._W[self.row_index]

    def __setitem__(self, key, value):
        if not isinstance(key, int):
            key = self.domain.index(key)
        if isinstance(value, str):
            var = self.domain[key]
            value = var.to_val(value)
        if key >= 0:
            if not isinstance(value, Real):
                raise TypeError("Expected primitive value, got '%s'" %
                                type(value).__name__)
            if key < len(self._x):
                self._values[key] = self._x[key] = value
            else:
                self._values[key] = self._y[key - len(self._x)] = value
        else:
            self._metas[-1-key] = value


class Columns:
    def __init__(self, domain):
        for v in domain:
            setattr(self, v.name.replace(" ", "_"), v)
        for v in domain.metas:
            setattr(self, v.name.replace(" ", "_"), v)

class Table(MutableSequence):
    def __new__(cls, *args, **argkw):
        self = None
        if not args:
            if not args and not argkw:
                self = super().__new__(cls)
            elif "filename" in argkw:
                self = cls.read_data(argkw["filename"])
        elif len(args) == 1:
            arg = args[0]
            if isinstance(arg, str):
                self = Table.read_data(args[0])
            elif isinstance(arg, Domain):
                self = Table.new_from_domain(arg)
        elif 2 <= len(args) <= 4:
            if isinstance(args[0], Domain) and \
               all(isinstance(arg, np.ndarray) for arg in args[1:]):
                self = Table.new_from_numpy(*args)
        if self is None:
            raise ValueError("Invalid arguments for Table.__new__")
        self.clear_cache()
        return self

    @staticmethod
    def new_from_domain(domain, n_rows=0, weights=True):
        self = Table.__new__(Table)
        self.domain = domain
        self.n_rows = n_rows
        self._X = np.zeros((n_rows, len(domain.attributes)))
        self._Y = np.zeros((n_rows, len(domain.class_vars)))
        if weights:
            self._W = np.ones(n_rows)
        else:
            self._W = np.empty((n_rows, 0))
        self._metas = np.empty((n_rows, len(self.domain.metas)), object)
        return self


    @staticmethod
    def compose_cols_from(table, row_indices, src_cols, n_rows):
        if not len(src_cols):
            return np.zeros((n_rows, 0), dtype=table._X.dtype)

        n_src_attrs = len(table.domain.attributes)
        if all(0 <= x < n_src_attrs for x in src_cols):
            return table._X[row_indices, src_cols]
        if all(x < 0 for x in src_cols):
            return table._metas[row_indices, [-1-x for x in src_cols]]
        if all(x >= n_src_attrs for x in src_cols):
            return table._Y[row_indices, [x-n_src_attrs for x in src_cols]]

        a = np.empty((n_rows, len(src_cols)), dtype=table._X.dtype)
        for i, col in enumerate(src_cols):
            if col < 0:
                a[:, i] = table._metas[row_indices, -1 - col]
            elif col < n_src_attrs:
                a[:, i] = table._X[row_indices, col]
            else:
                a[:, i] = table._Y[row_indices, col - n_src_attrs]
        return a


    @staticmethod
    def new_from_table(domain, table, row_indices=...):
        if domain == table.domain:
            return Table.new_from_table_rows(table, row_indices)

        if isinstance(row_indices, slice):
            start, stop, stride = row_indices.indices(len(table._X))
            n_rows = (stop - start) / stride
        elif row_indices is ...:
            n_rows = len(table._X)
        else:
            n_rows = len(row_indices)

        self = Table.__new__(Table)
        self.domain = domain
        conversion = domain.get_conversion(table.domain)
        self._X = Table.compose_cols_from(
            table, row_indices, conversion.attributes, n_rows)
        self._Y = Table.compose_cols_from(
            table, row_indices, conversion.classes, n_rows)
        self._metas = Table.compose_cols_from(
            table, row_indices, conversion.metas, n_rows)
        self._W = np.array(table._W[row_indices])
        return self


    @staticmethod
    def new_from_table_rows(table, row_indices):
        self = Table.__new__(Table)
        self.domain = table.domain
        self._X = table._X[row_indices]
        self._Y = table._Y[row_indices]
        self._metas = table._metas[row_indices]
        self._W = table._W[row_indices]
        return self

    @staticmethod
    def new_from_numpy(domain, X, Y=None, metas=None, W=None):
        if Y is None:
            Y = X[:, len(domain.attributes):]
            X = X[:, :len(domain.attributes)]
        if metas is None:
            metas = np.empty((X.shape[0], 0), object)
        if W is None:
            W = np.empty((X.shape[0], 0))

        if X.shape[1] != len(domain.attributes):
            raise ValueError("Invalid number of variable columns ({} != {}".
                format(X.shape[1], len(domain.attributes)))
        if Y.shape[1] != len(domain.class_vars):
            raise ValueError("Invalid number of class columns ({} != {}".
                format(Y.shape[1], len(domain.class_vars)))
        if metas.shape[1] != len(domain.metas):
            raise ValueError("Invalid number of meta attribute columns ({} != {}".
                format(metas.shape[1], len(domain.metas)))
        if not X.shape[0] == Y.shape[0] == metas.shape[0] == W.shape[0]:
            raise ValueError("Parts of data contain different numbers of rows.")

{'fraction_non_alphanumeric': 0.1164972634870993, 'fraction_numerical': 0.004691164972634871, 'mean_word_length': 2.039533376539209, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7179371', 'n_tokens_mistral': 4353, 'n_tokens_neox': 4272, 'n_words': 1034}