<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@author: efourrier

Purpose : This is a framework for Modeling with pandas, numpy and skicit-learn.
The Goal of this module is to rely on a dataframe structure for modelling g

"""


#########################################################
# Import modules and global helpers
#########################################################

import pandas as pd
import numpy as np
from numpy.random import permutation


def cserie(serie):
    return serie[serie].index.tolist()


class DataExploration(object):
    """
    This class is designed to provide infos about the dataset such as
    number of missing values, number of unique values, constant columns,
    long strings ...

    For the most useful methods it will store the result into a attributes

    When you used a method the output will be stored in a instance attribute so you
    don't have to compute the result again.

        """

    def __init__(self, data):
        """
        Parameters
        ----------
        data : DataFrame
                the data you want explore

        Examples
        --------
        explorer = DataExploration(data = your_DataFrame)
        explorer.structure() : global structure of your DataFrame
        explorer.psummary() to get the a global snapchat of the different stuff detected
        data_cleaned = explorer.basic_cleaning() to clean your data.
        """
        assert isinstance(data, pd.DataFrame)
        self.data = data
        # if not self.label:
        # 	print("""the label column is empty the data will be considered
        # 		as a dataset of predictors""")
        self._nrow = len(self.data.index)
        self._ncol = len(self.data.columns)
        self._dfnumi = (self.data.dtypes == float) | (
            self.data.dtypes == int)
        self._dfnum = cserie(self._dfnumi)
        self._dfchari = (self.data.dtypes == object)
        self._dfchar = cserie(self._dfchari)
        self._nacolcount = pd.DataFrame()
        self._narowcount = pd.DataFrame()
        self._count_unique = pd.DataFrame()
        self._constantcol = []
        self._manymissingcol = []
        self._manymissingrow = []
        self._dupcol = []
        self._nearzerovar = pd.DataFrame()
        self._corrcolumns = []
        self._dict_info = {}
        self._structure = pd.DataFrame()
        self._string_info = ""

    # def get_label(self):
    # 	""" return the Serie of label you want predict """
    # 	if not self.label:
    # 		print("""the label column is empty the data will be considered
    # 			as a dataset of predictors""")
    # 	return self.data[self.label]

    def count_unique(self):
        """ Return a serie with the number of unique value per columns """
        if len(self._count_unique):
            return self._count_unique
        self._count_unique = self.data.apply(lambda x: x.nunique(), axis=0)
        return self._count_unique

    def sample_df(self, pct=0.05, nr=10, threshold=None):
        """ sample a number of rows of a dataframe = min(max(0.05*nrow(self,nr),threshold)"""
        a = max(int(pct * float(len(self.data.index))), nr)
        if threshold:
            a = min(a, threshold)
        return self.data.loc[permutation(self.data.index)[:a]]

    def nacolcount(self):
        """ count the number of missing values per columns """
        if len(self._nacolcount):
            return self._nacolcount
        self._nacolcount = self.data.isnull().sum(axis=0)
        self._nacolcount = pd.DataFrame(self._nacolcount, columns=['Nanumber'])
        self._nacolcount['Napercentage'] = self._nacolcount[
            'Nanumber'] / (self._nrow)
        return self._nacolcount

    def narowcount(self):
        """ count the number of missing values per columns """
        if len(self._narowcount):
            return self._narowcount
        self._narowcount = self.data.isnull().sum(axis=1)
        self._narowcount = pd.DataFrame(
            self._narowcount, columns=['Nanumber'])
        self._narowcount['Napercentage'] = self._narowcount[
            'Nanumber'] / (self._nrow)
        return self._narowcount

    def manymissing(self, a=0.9, row=False):
        """ identify columns of a dataframe with many missing values ( >= a), if
        row = True row either.
        - the output is a list """
        if row:
            self._manymissingrow = self.narowcount()
            self._manymissingrow = cserie(
                self._manymissingrow['Napercentage'] >= a)
            return self._manymissingrow
        else:
            self._manymissingcol = self.nacolcount()
            self._manymissingcol = cserie(
                self._manymissingcol['Napercentage'] >= a)
            return self._manymissingcol

    def df_len_string(self):
        """ Return a Series with the max of the length of the string of string-type columns """
        return self.data.drop(self._dfnum, axis=1).apply(lambda x: np.max(x.str.len()), axis=0)

    def detectkey(self, index_format=False, pct=0.15, dropna=False, **kwargs):
        """ identify id or key columns as an index if index_format = True or
        as a list if index_format = False """
        if not dropna:
            col_to_keep = self.sample_df(
                pct=pct, **kwargs).apply(lambda x: len(x.unique()) == len(x), axis=0)
            if len(col_to_keep) == 0:
                return []
            is_key_index = col_to_keep
            is_key_index[is_key_index] == self.data.loc[:, is_key_index].apply(
                lambda x: len(x.unique()) == len(x), axis=0)
            if index_format:
                return is_key_index
            else:
                return cserie(is_key_index)
        else:
            col_to_keep = self.sample_df(
                pct=pct, **kwargs).apply(lambda x: x.nunique() == len(x.dropna()), axis=0)
            if len(col_to_keep) == 0:
                return []
            is_key_index = col_to_keep
            is_key_index[is_key_index] == self.data.loc[:, is_key_index].apply(
                lambda x: x.nunique() == len(x.dropna()), axis=0)
            if index_format:
                return is_key_index
            else:
                return cserie(is_key_index)

    def constantcol(self, **kwargs):
        """ identify constant columns """
        # sample to reduce computation time
        if len(self._constantcol):
            return self._constantcol
        col_to_keep = self.sample_df(
            **kwargs).apply(lambda x: len(x.unique()) == 1, axis=0)
        if len(cserie(col_to_keep)) == 0:
            return []
        self._constantcol = cserie(self.data.loc[:, col_to_keep].apply(
            lambda x: len(x.unique()) == 1, axis=0))
        return self._constantcol

        def constantcol2(self, **kwargs):
            """ identify constant columns """
            # sample to reduce computation time
            return cserie((self.data == self.data.ix[0]).all())

    def factors(self, nb_max_levels=10, threshold_value=None, index=False):
        """ return a list of the detected factor variable, detection is based on
        ther percentage of unicity perc_unique = 0.05 by default.
        We follow here the definition of R factors variable considering that a
        factor variable is a character variable that take value in a list a levels

        this is a bad implementation


        Arguments
        ----------
        nb_max_levels: the mac nb of levels you fix for a categorical variable
        threshold_value : the nb of of unique value in percentage of the dataframe length
        index : if you want the result as an index or a list

         """
        if threshold_value:
            max_levels = max(nb_max_levels, threshold_value * self._nrow)
        else:
            max_levels = nb_max_levels

        def helper_factor(x, num_var=self._dfnum):
            unique_value = set()
            if x.name in num_var:
                return False
            else:
                for e in x.values:
                    if len(unique_value) >= max_levels:
                        return False
                    else:<commit_msg>Fix indent typo in constantcol2<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@author: efourrier

Purpose : This is a framework for Modeling with pandas, numpy and skicit-learn.
The Goal of this module is to rely on a dataframe structure for modelling g

"""


#########################################################
# Import modules and global helpers
#########################################################

import pandas as pd
import numpy as np
from numpy.random import permutation


def cserie(serie):
    return serie[serie].index.tolist()


class DataExploration(object):
    """
    This class is designed to provide infos about the dataset such as
    number of missing values, number of unique values, constant columns,
    long strings ...

    For the most useful methods it will store the result into a attributes

    When you used a method the output will be stored in a instance attribute so you
    don't have to compute the result again.

        """

    def __init__(self, data):
        """
        Parameters
        ----------
        data : DataFrame
                the data you want explore

        Examples
        --------
        explorer = DataExploration(data = your_DataFrame)
        explorer.structure() : global structure of your DataFrame
        explorer.psummary() to get the a global snapchat of the different stuff detected
        data_cleaned = explorer.basic_cleaning() to clean your data.
        """
        assert isinstance(data, pd.DataFrame)
        self.data = data
        # if not self.label:
        # 	print("""the label column is empty the data will be considered
        # 		as a dataset of predictors""")
        self._nrow = len(self.data.index)
        self._ncol = len(self.data.columns)
        self._dfnumi = (self.data.dtypes == float) | (
            self.data.dtypes == int)
        self._dfnum = cserie(self._dfnumi)
        self._dfchari = (self.data.dtypes == object)
        self._dfchar = cserie(self._dfchari)
        self._nacolcount = pd.DataFrame()
        self._narowcount = pd.DataFrame()
        self._count_unique = pd.DataFrame()
        self._constantcol = []
        self._manymissingcol = []
        self._manymissingrow = []
        self._dupcol = []
        self._nearzerovar = pd.DataFrame()
        self._corrcolumns = []
        self._dict_info = {}
        self._structure = pd.DataFrame()
        self._string_info = ""

    # def get_label(self):
    # 	""" return the Serie of label you want predict """
    # 	if not self.label:
    # 		print("""the label column is empty the data will be considered
    # 			as a dataset of predictors""")
    # 	return self.data[self.label]

    def count_unique(self):
        """ Return a serie with the number of unique value per columns """
        if len(self._count_unique):
            return self._count_unique
        self._count_unique = self.data.apply(lambda x: x.nunique(), axis=0)
        return self._count_unique

    def sample_df(self, pct=0.05, nr=10, threshold=None):
        """ sample a number of rows of a dataframe = min(max(0.05*nrow(self,nr),threshold)"""
        a = max(int(pct * float(len(self.data.index))), nr)
        if threshold:
            a = min(a, threshold)
        return self.data.loc[permutation(self.data.index)[:a]]

    def nacolcount(self):
        """ count the number of missing values per columns """
        if len(self._nacolcount):
            return self._nacolcount
        self._nacolcount = self.data.isnull().sum(axis=0)
        self._nacolcount = pd.DataFrame(self._nacolcount, columns=['Nanumber'])
        self._nacolcount['Napercentage'] = self._nacolcount[
            'Nanumber'] / (self._nrow)
        return self._nacolcount

    def narowcount(self):
        """ count the number of missing values per columns """
        if len(self._narowcount):
            return self._narowcount
        self._narowcount = self.data.isnull().sum(axis=1)
        self._narowcount = pd.DataFrame(
            self._narowcount, columns=['Nanumber'])
        self._narowcount['Napercentage'] = self._narowcount[
            'Nanumber'] / (self._nrow)
        return self._narowcount

    def manymissing(self, a=0.9, row=False):
        """ identify columns of a dataframe with many missing values ( >= a), if
        row = True row either.
        - the output is a list """
        if row:
            self._manymissingrow = self.narowcount()
            self._manymissingrow = cserie(
                self._manymissingrow['Napercentage'] >= a)
            return self._manymissingrow
        else:
            self._manymissingcol = self.nacolcount()
            self._manymissingcol = cserie(
                self._manymissingcol['Napercentage'] >= a)
            return self._manymissingcol

    def df_len_string(self):
        """ Return a Series with the max of the length of the string of string-type columns """
        return self.data.drop(self._dfnum, axis=1).apply(lambda x: np.max(x.str.len()), axis=0)

    def detectkey(self, index_format=False, pct=0.15, dropna=False, **kwargs):
        """ identify id or key columns as an index if index_format = True or
        as a list if index_format = False """
        if not dropna:
            col_to_keep = self.sample_df(
                pct=pct, **kwargs).apply(lambda x: len(x.unique()) == len(x), axis=0)
            if len(col_to_keep) == 0:
                return []
            is_key_index = col_to_keep
            is_key_index[is_key_index] == self.data.loc[:, is_key_index].apply(
                lambda x: len(x.unique()) == len(x), axis=0)
            if index_format:
                return is_key_index
            else:
                return cserie(is_key_index)
        else:
            col_to_keep = self.sample_df(
                pct=pct, **kwargs).apply(lambda x: x.nunique() == len(x.dropna()), axis=0)
            if len(col_to_keep) == 0:
                return []
            is_key_index = col_to_keep
            is_key_index[is_key_index] == self.data.loc[:, is_key_index].apply(
                lambda x: x.nunique() == len(x.dropna()), axis=0)
            if index_format:
                return is_key_index
            else:
                return cserie(is_key_index)

    def constantcol(self, **kwargs):
        """ identify constant columns """
        # sample to reduce computation time
        if len(self._constantcol):
            return self._constantcol
        col_to_keep = self.sample_df(
            **kwargs).apply(lambda x: len(x.unique()) == 1, axis=0)
        if len(cserie(col_to_keep)) == 0:
            return []
        self._constantcol = cserie(self.data.loc[:, col_to_keep].apply(
            lambda x: len(x.unique()) == 1, axis=0))
        return self._constantcol

    def constantcol2(self, **kwargs):
        """ identify constant columns """
        return cserie((self.data == self.data.ix[0]).all())

    def factors(self, nb_max_levels=10, threshold_value=None, index=False):
        """ return a list of the detected factor variable, detection is based on
        ther percentage of unicity perc_unique = 0.05 by default.
        We follow here the definition of R factors variable considering that a
        factor variable is a character variable that take value in a list a levels

        this is a bad implementation


        Arguments
        ----------
        nb_max_levels: the mac nb of levels you fix for a categorical variable
        threshold_value : the nb of of unique value in percentage of the dataframe length
        index : if you want the result as an index or a list

         """
        if threshold_value:
            max_levels = max(nb_max_levels, threshold_value * self._nrow)
        else:
            max_levels = nb_max_levels

        def helper_factor(x, num_var=self._dfnum):
            unique_value = set()
            if x.name in num_var:
                return False
            else:
                for e in x.values:
                    if len(unique_value) >= max_levels:
                        return False
                    else:
{'fraction_non_alphanumeric': 0.11539407352760359, 'fraction_numerical': 0.004610844706750277, 'mean_word_length': 2.143381642512077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1952440', 'n_tokens_mistral': 4756, 'n_tokens_neox': 4619, 'n_words': 1334}