<commit_before>import numpy as np
from numpy import nanmin, nanmax


class DataHandler(object):
    """
    -DONE Verify input.
    -DONE Determine whether data is 1D or 2D.
    -DONE Set very large or small values to nan.
    -DONE Investigate whether data are linspaces.
    -DONE Sort and flip data that is linspace.
    -DONE Determine whether data is suitable for imshow. imshow_eligible
    """
    def __init__(self, x, y):
        self.clip_min = -1e25
        self.clip_max = 1e25
        self.x = x
        self.y = y
        self.data = [self.x, self.y]
        self.data_is_valid = False

    def set_data_validity(self):
        """
        Depends on
        self.data
        self.data_dim
        """
        data_is_valid = True
        try:
            for data in self.data:
                assert data.ndim == self.data_dim
                assert data.shape == self.data[0].shape
        except Exception:
            data_is_valid = False
        self.data_is_valid = data_is_valid

    def clip_tdata_to_nan(self):
        for tdata in self.tdata:
            if tdata is None:
                continue
            tdata = self.clip_to_nan(tdata, self.clip_min, self.clip_max)

    def get_extent_of_data_dim(self, dim):
        """
        Depends on _set_tdata
        """
        assert dim in range(self.n_data_arrs)
        arr = self.tdata[dim]
        if self.data_is_linear[dim] and dim != 2:
            # Return first and last elements of array since we know it's
            # sorted.
            first = (0,) * arr.ndim
            last = (-1,) * arr.ndim
            return [arr[first], arr[last]]
        else:
            return [nanmin(arr), nanmax(arr)]

    @staticmethod
    def clip_to_nan(arr, clip_min, clip_max):
        assert type(arr) is np.ndarray
        # By default Numpy gives a RuntimeWarning when a nan is generated. We
        # are explicitly generating nans here so we don't want to see the
        # warning.
        with np.errstate(invalid='ignore'):
            arr[arr < clip_min] = np.nan
            arr[arr > clip_max] = np.nan
        return arr

    @staticmethod
    def is_linear(arr):
        """
        'linear' in this context also implies 'not constant'.
        """
        assert arr.ndim == 1
        arr_diff = np.diff(arr)
        # Check that there are non-zero elements in arr_diff.
        # Otherwise arr is constant.
        if not any(arr_diff):
            return False
        # Check that the elements are the same,
        # i.e., the slope of arr is constant.
        if not np.allclose(arr_diff, arr_diff[0]):
            return False
        return True

    def is_linear_on_axis(self, arr):
        assert arr.ndim == 2
        for axis in (0,1):
            idx = [0,0]
            idx[axis] = slice(None)
            arr_1D = arr[idx]
            if not self.is_linear(arr_1D):
                continue
            # Check that arr consists solely of copies of arr_1D. First,
            # insert an np.newaxis in arr_1D so you can subtract it from arr.
            if axis == 0:
                arr_1D = arr_1D[...,np.newaxis]
            if not np.allclose(arr, arr_1D):
                continue
            return axis
        return False

    @staticmethod
    def reverse_axis(arr, axis):
        assert axis in range(arr.ndim)
        idx = [slice(None)] * arr.ndim
        idx[axis] = slice(None, None, -1)
        return arr[idx]


class Transformed2DData(DataHandler):<commit_msg>Fix issue where data was incorrectly labeled as linear<commit_after>import numpy as np
from numpy import nanmin, nanmax


class DataHandler(object):
    """
    -DONE Verify input.
    -DONE Determine whether data is 1D or 2D.
    -DONE Set very large or small values to nan.
    -DONE Investigate whether data are linspaces.
    -DONE Sort and flip data that is linspace.
    -DONE Determine whether data is suitable for imshow. imshow_eligible
    """
    def __init__(self, x, y):
        self.clip_min = -1e25
        self.clip_max = 1e25
        self.x = x
        self.y = y
        self.data = [self.x, self.y]
        self.data_is_valid = False

    def set_data_validity(self):
        """
        Depends on
        self.data
        self.data_dim
        """
        data_is_valid = True
        try:
            for data in self.data:
                assert data.ndim == self.data_dim
                assert data.shape == self.data[0].shape
        except Exception:
            data_is_valid = False
        self.data_is_valid = data_is_valid

    def clip_tdata_to_nan(self):
        for tdata in self.tdata:
            if tdata is None:
                continue
            tdata = self.clip_to_nan(tdata, self.clip_min, self.clip_max)

    def get_extent_of_data_dim(self, dim):
        """
        Depends on _set_tdata
        """
        assert dim in range(self.n_data_arrs)
        arr = self.tdata[dim]
        return [nanmin(arr), nanmax(arr)]

    @staticmethod
    def clip_to_nan(arr, clip_min, clip_max):
        assert type(arr) is np.ndarray
        # By default Numpy gives a RuntimeWarning when a nan is generated. We
        # are explicitly generating nans here so we don't want to see the
        # warning.
        with np.errstate(invalid='ignore'):
            arr[arr < clip_min] = np.nan
            arr[arr > clip_max] = np.nan
        return arr

    @staticmethod
    def is_linear(arr):
        """
        'linear' in this context also implies 'not constant'.
        """
        assert arr.ndim == 1
        arr_diff = np.diff(arr)
        # Check that there are non-zero elements in arr_diff.
        # Otherwise arr is constant.
        if not any(arr_diff):
            return False
        # Check that the elements are the same,
        # i.e., the slope of arr is constant.
        if not np.allclose(arr_diff, arr_diff[0], atol=1e-15):
            return False
        return True

    def is_linear_on_axis(self, arr):
        assert arr.ndim == 2
        for axis in (0,1):
            idx = [0,0]
            idx[axis] = slice(None)
            arr_1D = arr[idx]
            if not self.is_linear(arr_1D):
                continue
            # Check that arr consists solely of copies of arr_1D. First,
            # insert an np.newaxis in arr_1D so you can subtract it from arr.
            if axis == 0:
                arr_1D = arr_1D[...,np.newaxis]
            if not np.allclose(arr, arr_1D):
                continue
            return axis
        return False

    @staticmethod
    def reverse_axis(arr, axis):
        assert axis in range(arr.ndim)
        idx = [slice(None)] * arr.ndim
        idx[axis] = slice(None, None, -1)
        return arr[idx]


class Transformed2DData(DataHandler):
{'fraction_non_alphanumeric': 0.08475329678470885, 'fraction_numerical': 0.008593865757890057, 'mean_word_length': 1.7283751010509296, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2587330', 'n_tokens_mistral': 2082, 'n_tokens_neox': 2022, 'n_words': 635}