<commit_before>import numpy as np
from nose.tools import raises

import simpleml.transform as tf


DATA1 = np.array(
    [[ 0.3, -2.6,  0.7, -1.3],
     [ 0.4, -1.8,  1.9, -0.4],
     [-0.2,  0.5, -0.4, -1.7],
     [-0.4,  3.5,  1.3, -1.3],
     [ 0.1, -0.6,  1.3, -1.3],
     [ 0.4, -2.8,  1.3,  0.8],
     [ 0.2, -1.5,  0.2, -1.5],
     [-0.3, -1.6,  1.5, -0.5]]
)

DATA2 = np.array(
    [[-0.4,  0.2, -0.6,  0.9],
     [ 1.6,  0.6, -1.1,  0.1],
     [-0.9, -0.7, -0.2,  0.9],
     [ 1.2,  0.5,  0.1,  0.3],
     [ 0.2, -0.6, -0.2, -0.9],
     [-0.3,  0.2,  1. , -0.5]]
)


class TestStandardize:
    def test_clean_data(self):
        data_std = tf.standardize(DATA1)

        assert np.allclose(np.mean(data_std, 0), np.zeros(DATA1.shape[1]))
        assert np.allclose(np.std(data_std, 0), np.ones(DATA1.shape[1]))

    def test_vec(self):
        data_std = tf.standardize(np.array([1, -1, 2, -2, 3, -3]))
        assert np.isclose(np.mean(data_std), 0)
        assert np.isclose(np.std(data_std), 1)

    @raises(ValueError)
    def test_missing(self):
        tf.standardize(np.array([1, -1, np.nan, -2, 3, -3]))

    @raises(ValueError)
    def test_inf(self):
        tf.standardize(np.array([1, -1, np.inf, -2, 3, -3]))

    @raises(ValueError)
    def test_const_mat(self):
        tf.standardize(np.ones((6, 4)))

    @raises(ValueError)
    def test_const_vec(self):
        tf.standardize(np.array([2, 2, 2]))


class TestPCA:
    def test_init_nonum(self):
        pca = tf.PCA().fit(DATA1)
        assert pca.num_comp == DATA1.shape[1]

    def test_init_withnum(self):
        pca = tf.PCA(num_comp=2)
        assert pca.num_comp == 2

    @raises(ValueError)
    def test_init_vec(self):
        tf.PCA().fit(np.array([1, 2, 3]))
<commit_msg>Testing dummy transformation functions.<commit_after>import numpy as np
from nose.tools import raises

import simpleml.transform as tf


DATA1 = np.array(
    [[ 0.3, -2.6,  0.7, -1.3],
     [ 0.4, -1.8,  1.9, -0.4],
     [-0.2,  0.5, -0.4, -1.7],
     [-0.4,  3.5,  1.3, -1.3],
     [ 0.1, -0.6,  1.3, -1.3],
     [ 0.4, -2.8,  1.3,  0.8],
     [ 0.2, -1.5,  0.2, -1.5],
     [-0.3, -1.6,  1.5, -0.5]]
)

DATA2 = np.array(
    [[-0.4,  0.2, -0.6,  0.9],
     [ 1.6,  0.6, -1.1,  0.1],
     [-0.9, -0.7, -0.2,  0.9],
     [ 1.2,  0.5,  0.1,  0.3],
     [ 0.2, -0.6, -0.2, -0.9],
     [-0.3,  0.2,  1. , -0.5]]
)


class TestStandardize:
    def test_clean_data(self):
        data_std = tf.standardize(DATA1)

        assert np.allclose(np.mean(data_std, 0), np.zeros(DATA1.shape[1]))
        assert np.allclose(np.std(data_std, 0), np.ones(DATA1.shape[1]))

    def test_vec(self):
        data_std = tf.standardize(np.array([1, -1, 2, -2, 3, -3]))
        assert np.isclose(np.mean(data_std), 0)
        assert np.isclose(np.std(data_std), 1)

    @raises(ValueError)
    def test_missing(self):
        tf.standardize(np.array([1, -1, np.nan, -2, 3, -3]))

    @raises(ValueError)
    def test_inf(self):
        tf.standardize(np.array([1, -1, np.inf, -2, 3, -3]))

    @raises(ValueError)
    def test_const_mat(self):
        tf.standardize(np.ones((6, 4)))

    @raises(ValueError)
    def test_const_vec(self):
        tf.standardize(np.array([2, 2, 2]))


class TestDummies:
    original = np.array([0, 1, 1, 0])
    replaced = np.array([2, 4, 4, 2])
    dummied = np.array([[1, 0], [0, 1], [0, 1], [1, 0]])

    def test_to_dummies(self):
        assert np.allclose(tf.to_dummies(self.original), self.dummied)
        assert np.allclose(tf.to_dummies(self.replaced), self.dummied)

    @raises(ValueError)
    def test_to_dummies_all_unique(self):
        tf.to_dummies(np.array([1, 2, 3, 4]))

    @raises(ValueError)
    def test_to_dummies_nan(self):
        tf.to_dummies(np.array([1, 3, 3, 1, np.nan]))

    def test_from_dummies(self):
        assert np.allclose(tf.from_dummies(self.dummied), self.original)

    def test_from_dummies_values(self):
        assert np.allclose(tf.from_dummies(self.dummied, values=[2, 4]),
                           self.replaced)

class TestPCA:
    def test_init_nonum(self):
        pca = tf.PCA().fit(DATA1)
        assert pca.num_comp == DATA1.shape[1]

    def test_init_withnum(self):
        pca = tf.PCA(num_comp=2)
        assert pca.num_comp == 2

    @raises(ValueError)
    def test_init_vec(self):
        tf.PCA().fit(np.array([1, 2, 3]))

{'fraction_non_alphanumeric': 0.2018327605956472, 'fraction_numerical': 0.07514318442153493, 'mean_word_length': 2.1798980335032776, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3088381', 'n_tokens_mistral': 2054, 'n_tokens_neox': 1965, 'n_words': 363}