<commit_before>"""
Tests for pandas wrappers
"""

from unittest import TestCase

import pandas as pd
import numpy as np
from datetime import datetime

from cate.ops import pandas_fillna
from cate.core.op import OP_REGISTRY
from cate.util.misc import object_to_qualified_name


class TestFillna(TestCase):
    """
    Test fillna operation
    """
    def test_nominal(self):
        """
        Test nominal operation
        """
        # Test na filling using a given method
        data = {'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6],
                'B': [5, 6, 8, 7, 5, np.nan, np.nan, np.nan, 1, 2, 7, 6],
                'time': [datetime(2000, x, 1) for x in range(1, 13)]}
        expected = {'A': [1, 2, 3, 3, 4, 9, 9, 9, 1, 0, 4, 6],
                    'B': [5, 6, 8, 7, 5, 5, 5, 5, 1, 2, 7, 6],
                    'time': [datetime(2000, x, 1) for x in range(1, 13)]}
        expected = pd.DataFrame.from_dict(expected, dtype=float)
        expected = expected.set_index('time')
        df = pd.DataFrame.from_dict(data, dtype=float)
        df = df.set_index('time')

        actual = pandas_fillna(df, method='ffill')
        self.assertTrue(actual.equals(expected))

        # Test na filling using a given value
        actual = pandas_fillna(df, value=3.14)
        expected = {'A': [1, 2, 3, 3.14, 4, 9, 3.14, 3.14, 1, 0, 4, 6],
                    'B': [5, 6, 8, 7, 5, 3.14, 3.14, 3.14, 1, 2, 7, 6],
                    'time': [datetime(2000, x, 1) for x in range(1, 13)]}
        expected = pd.DataFrame.from_dict(expected, dtype=float)
        expected = expected.set_index('time')
        self.assertTrue(actual.equals(expected))

    def test_registered(self):
        """
        Test operation when run as a registered operation
        """
        reg_op = OP_REGISTRY.get_op(object_to_qualified_name(pandas_fillna))
        # Test na filling using a given method
        data = {'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6],
                'B': [5, 6, 8, 7, 5, np.nan, np.nan, np.nan, 1, 2, 7, 6],
                'time': [datetime(2000, x, 1) for x in range(1, 13)]}
        expected = {'A': [1, 2, 3, 3, 4, 9, 9, 9, 1, 0, 4, 6],
                    'B': [5, 6, 8, 7, 5, 5, 5, 5, 1, 2, 7, 6],
                    'time': [datetime(2000, x, 1) for x in range(1, 13)]}
        expected = pd.DataFrame.from_dict(expected, dtype=float)
        expected = expected.set_index('time')
        df = pd.DataFrame.from_dict(data, dtype=float)
        df = df.set_index('time')

        actual = reg_op(df=df, method='ffill')
        self.assertTrue(actual.equals(expected))<commit_msg>Use shorter form of DataFrame instantiation<commit_after>"""
Tests for pandas wrappers
"""

from unittest import TestCase

import pandas as pd
import numpy as np
from datetime import datetime

from cate.ops import pandas_fillna
from cate.core.op import OP_REGISTRY
from cate.util.misc import object_to_qualified_name


class TestFillna(TestCase):
    """
    Test fillna operation
    """
    def test_nominal(self):
        """
        Test nominal operation
        """
        # Test na filling using a given method
        data = {'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6],
                'B': [5, 6, 8, 7, 5, np.nan, np.nan, np.nan, 1, 2, 7, 6]}
        expected = {'A': [1, 2, 3, 3, 4, 9, 9, 9, 1, 0, 4, 6],
                    'B': [5, 6, 8, 7, 5, 5, 5, 5, 1, 2, 7, 6]}
        time = pd.date_range('2000-01-01', freq='MS', periods=12)

        expected = pd.DataFrame(data=expected, index=time, dtype=float)
        df = pd.DataFrame(data=data, index=time, dtype=float)

        actual = pandas_fillna(df, method='ffill')
        self.assertTrue(actual.equals(expected))

        # Test na filling using a given value
        actual = pandas_fillna(df, value=3.14)
        expected = {'A': [1, 2, 3, 3.14, 4, 9, 3.14, 3.14, 1, 0, 4, 6],
                    'B': [5, 6, 8, 7, 5, 3.14, 3.14, 3.14, 1, 2, 7, 6]}
        expected = pd.DataFrame(data=expected, index=time, dtype=float)
        self.assertTrue(actual.equals(expected))

    def test_registered(self):
        """
        Test operation when run as a registered operation
        """
        reg_op = OP_REGISTRY.get_op(object_to_qualified_name(pandas_fillna))
        # Test na filling using a given method
        data = {'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6],
                'B': [5, 6, 8, 7, 5, np.nan, np.nan, np.nan, 1, 2, 7, 6]}
        expected = {'A': [1, 2, 3, 3, 4, 9, 9, 9, 1, 0, 4, 6],
                    'B': [5, 6, 8, 7, 5, 5, 5, 5, 1, 2, 7, 6]}
        time = pd.date_range('2000-01-01', freq='MS', periods=12)

        expected = pd.DataFrame(data=expected, index=time, dtype=float)
        df = pd.DataFrame(data=data, index=time, dtype=float)

        actual = reg_op(df=df, method='ffill')
        self.assertTrue(actual.equals(expected))
{'fraction_non_alphanumeric': 0.15873340143003065, 'fraction_numerical': 0.06251276813074566, 'mean_word_length': 2.266177451634423, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5968066', 'n_tokens_mistral': 2027, 'n_tokens_neox': 1723, 'n_words': 548}