<commit_before>import doctest
import os
import warnings
from datetime import timedelta

import numpy as np
import pandas as pd
import sklearn
import category_encoders.tests.helpers as th
from sklearn.utils.estimator_checks import check_transformer_general, check_transformers_unfitted
from sklearn.compose import ColumnTransformer
from unittest2 import TestSuite, TextTestRunner, TestCase  # or `from unittest import ...` if on Python 3.4+

import category_encoders as encoders

__author__ = 'willmcginnis'


# data definitions
np_X = th.create_array(n_rows=100)
np_X_t = th.create_array(n_rows=50, extras=True)
np_y = np.random.randn(np_X.shape[0]) > 0.5
np_y_t = np.random.randn(np_X_t.shape[0]) > 0.5
X = th.create_dataset(n_rows=100)
X_t = th.create_dataset(n_rows=50, extras=True)
y = pd.DataFrame(np_y)
y_t = pd.DataFrame(np_y_t)


# turn warnings like division by zero into errors
np.seterr(all='raise')

# turn non-Numpy warnings into errors
warnings.filterwarnings('error')


# this class utilises parametrised tests where we loop over different encoders
class TestEncoders(TestCase):

    def test_np(self):
        for encoder_name in encoders.__all__:
            with self.subTest(encoder_name=encoder_name):

                # Encode a numpy array
                enc = getattr(encoders, encoder_name)()
                enc.fit(np_X, np_y)
                th.verify_numeric(enc.transform(np_X_t))

    def test_classification(self):
        for encoder_name in encoders.__all__:
            with self.subTest(encoder_name=encoder_name):
                cols = ['unique_str', 'underscore', 'extra', 'none', 'invariant', 321, 'categorical', 'na_categorical']

                enc = getattr(encoders, encoder_name)(cols=cols)
                enc.fit(X, np_y)
                th.verify_numeric(enc.transform(X_t))

                enc = getattr(encoders, encoder_name)(verbose=1)
                enc.fit(X, np_y)
                th.verify_numeric(enc.transform(X_t))

                enc = getattr(encoders, encoder_name)(drop_invariant=True)
                enc.fit(X, np_y)
                th.verify_numeric(enc.transform(X_t))

                enc = getattr(encoders, encoder_name)(return_df=False)
                enc.fit(X, np_y)
                self.assertEqual(enc.transform(X_t), np.ndarray, str(type(enc.transform(X_t))))
                self.assertTrue(isinstance(enc.transform(X_t), np.ndarray))
                self.assertEqual(enc.transform(X_t).shape[0], X_t.shape[0], 'Row count must not change')

                # documented in issue #122
                # when we use the same encoder on two different datasets, it should not explode
                # X_a = pd.DataFrame(data=['1', '2', '2', '2', '2', '2'], columns=['col_a'])
                # X_b = pd.DataFrame(data=['1', '1', '1', '2', '2', '2'], columns=['col_b']) # different values and name
                # y_dummy = [True, False, True, False, True, False]
                # enc = getattr(encoders, encoder_name)()
                # enc.fit(X_a, y_dummy)
                # enc.fit(X_b, y_dummy)
                # verify_numeric(enc.transform(X_b))
<commit_msg>Revert "test type of transform"<commit_after>import doctest
import os
import warnings
from datetime import timedelta

import numpy as np
import pandas as pd
import sklearn
import category_encoders.tests.helpers as th
from sklearn.utils.estimator_checks import check_transformer_general, check_transformers_unfitted
from sklearn.compose import ColumnTransformer
from unittest2 import TestSuite, TextTestRunner, TestCase  # or `from unittest import ...` if on Python 3.4+

import category_encoders as encoders

__author__ = 'willmcginnis'


# data definitions
np_X = th.create_array(n_rows=100)
np_X_t = th.create_array(n_rows=50, extras=True)
np_y = np.random.randn(np_X.shape[0]) > 0.5
np_y_t = np.random.randn(np_X_t.shape[0]) > 0.5
X = th.create_dataset(n_rows=100)
X_t = th.create_dataset(n_rows=50, extras=True)
y = pd.DataFrame(np_y)
y_t = pd.DataFrame(np_y_t)


# turn warnings like division by zero into errors
np.seterr(all='raise')

# turn non-Numpy warnings into errors
warnings.filterwarnings('error')


# this class utilises parametrised tests where we loop over different encoders
class TestEncoders(TestCase):

    def test_np(self):
        for encoder_name in encoders.__all__:
            with self.subTest(encoder_name=encoder_name):

                # Encode a numpy array
                enc = getattr(encoders, encoder_name)()
                enc.fit(np_X, np_y)
                th.verify_numeric(enc.transform(np_X_t))

    def test_classification(self):
        for encoder_name in encoders.__all__:
            with self.subTest(encoder_name=encoder_name):
                cols = ['unique_str', 'underscore', 'extra', 'none', 'invariant', 321, 'categorical', 'na_categorical']

                enc = getattr(encoders, encoder_name)(cols=cols)
                enc.fit(X, np_y)
                th.verify_numeric(enc.transform(X_t))

                enc = getattr(encoders, encoder_name)(verbose=1)
                enc.fit(X, np_y)
                th.verify_numeric(enc.transform(X_t))

                enc = getattr(encoders, encoder_name)(drop_invariant=True)
                enc.fit(X, np_y)
                th.verify_numeric(enc.transform(X_t))

                enc = getattr(encoders, encoder_name)(return_df=False)
                enc.fit(X, np_y)
                self.assertTrue(isinstance(enc.transform(X_t), np.ndarray))
                self.assertEqual(enc.transform(X_t).shape[0], X_t.shape[0], 'Row count must not change')

                # documented in issue #122
                # when we use the same encoder on two different datasets, it should not explode
                # X_a = pd.DataFrame(data=['1', '2', '2', '2', '2', '2'], columns=['col_a'])
                # X_b = pd.DataFrame(data=['1', '1', '1', '2', '2', '2'], columns=['col_b']) # different values and name
                # y_dummy = [True, False, True, False, True, False]
                # enc = getattr(encoders, encoder_name)()
                # enc.fit(X_a, y_dummy)
                # enc.fit(X_b, y_dummy)
                # verify_numeric(enc.transform(X_b))

{'fraction_non_alphanumeric': 0.11261261261261261, 'fraction_numerical': 0.01287001287001287, 'mean_word_length': 2.749698431845597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1069915', 'n_tokens_mistral': 2127, 'n_tokens_neox': 1985, 'n_words': 453}