<jupyter_start><jupyter_text>Function to build the data quality reportsHere we create function that help us to easily build the data quality reports and various plot to help visualize the data.<jupyter_code>import matplotlib.pyplot as plt_func
def createQuantitativeReport(df):
    """
    Build a data quality report for quantitative variables.
    
    Such a report should have the quantitative variables as rows with 
    columns describing various summary statistics of these variables.
    The statistics for each variable should include the total number of 
    instances ('Count'), the percentage of missing values ('Miss. (%)'),
    the cardinality ('Card.'), the minimum ('Min.'), the 1st quartile 
    ('1st Qrt.'), the mean ('Mean'), the median ('Median'), the 3rd 
    quartile ('3rd Qrt.'), the maximum ('Max.') and the standard 
    deviation ('Std. Dev.'). Note that many of these statistics can 
    also summarized by calling the pandas.DataFrame.info and
    pd.DataFrame.describe methods.
    
    In this context, cardinality represents the number of distinct 
    values per variable.
    
    Args:
        df: pandas.core.frame.DataFrame
            The DataFrame should hold the data that we want to 
            investigate. Each row in the DataFrame represents an 
            individual sample point, with the successive columns 
            corresponding to the variables and their specific values 
            for that sample point.

    Returns:
        pandas.core.frame.DataFrame: The aforementioned data 
            quality report for quantitative variables.
    """
    df_qr = pd.DataFrame(0, index=df.columns, 
                            columns=['Count', 'Miss. (%)', 'Card.', 'Min.', '1st Qrt.', 
                                        'Mean', 'Median', '3rd Qrt.', 'Max.', 'Std. Dev.'])
#     return 'test'
    df_qr['Count'] = df.shape[0]
    df_qr['Miss. (%)'] = (df.isna().sum() / df.shape[0]) * 100
    df_qr['Card.'] = df.nunique(dropna=True)
    df_qr['Min.'] = df.min(skipna=True) 
    df_qr['1st Qrt.'] = df.quantile(q=0.25, interpolation='linear')
    df_qr['Mean'] = df.mean(skipna=True)
    df_qr['Median'] = df.quantile(q=0.5, interpolation='linear')
    df_qr['3rd Qrt.'] = df.quantile(q=0.75, interpolation='linear')
    df_qr['Max.'] = df.max(skipna=True)
    df_qr['Std. Dev.'] = df.std(ddof=1, skipna=True)
    df_qr.name = 'Quantitative Qualilty Report'
    return df_qr
def createCategoricalReport(df):
    """
    Build a data quality report for categorical variables.
    
    Such a report should have the categorical variables as rows with 
    columns describing various summary statistics of these variables.
    The statistics for each variable should include the total number of 
    instances ('Count'), the percentage of missing values ('Miss. (%)'),
    the cardinality ('Card.'), the first and second mode ('Mode' and 
    '2nd Mode'), the frequency of the first mode ('Mode Freq.' and '2nd 
    Mode Freq.') and the percentage of the first mode ('Mode (%)' and 
    '2nd Mode (%)'). Note that many of these statistics can also 
    summarized by calling the pandas.DataFrame.info and
    pd.DataFrame.describe (with the include argument set to 
    'category') methods.
    
    The same as for the data quality report for quantitative variables,
    cardinality represents the number of distinct values per variable.
    The first and second mode refer, respectively, to the first and 
    second most frequent levels for each categorical variable.
    
    Args:
        df: pandas.core.frame.DataFrame
            The DataFrame should hold the data that we want to 
            investigate. Each row in the DataFrame represents an 
            individual sample point, with the successive columns 
            corresponding to the variables and their specific values 
            for that sample point.

    Returns:
        pandas.core.frame.DataFrame: The aforementioned data 
            quality report for categorical variables.
    """
    df_qr = pd.DataFrame(0, index=df.columns, 
                            columns=['Count', 'Miss. (%)', 'Card.', 'Mode', 'Mode Freq.', 
                                        'Mode (%)', '2nd Mode', '2nd Mode Freq.', '2nd Mode (%)'])
    df_qr['Count'] = df.shape[0]
    df_qr['Miss. (%)'] = (df.isna().sum() / df.shape[0]) * 100
    df_qr['Card.'] = df.nunique(dropna=True)
    for variable, row in df_qr.iterrows():
        df_qr.loc[variable, 'Mode'] = df[variable].value_counts(dropna=True).index[0]
        df_qr.loc[variable, 'Mode Freq.'] = df[variable].value_counts(dropna=True).iloc[0]
        df_qr.loc[variable, 'Mode (%)'] = (df_qr.loc[variable, 'Mode Freq.'] / df[variable].count()) * 100
        try:
            df_qr.loc[variable, '2nd Mode'] = df[variable].value_counts(dropna=True).index[1]
            df_qr.loc[variable, '2nd Mode Freq.'] = df[variable].value_counts(dropna=True).iloc[1]
            df_qr.loc[variable,'2nd Mode (%)'] = (df_qr.loc[variable, '2nd Mode Freq.'] / df[variable].count()) * 100
        except IndexError:
            df_qr.loc[variable, '2nd Mode'] = None
            df_qr.loc[variable, '2nd Mode Freq.'] = None
            df_qr.loc[variable,'2nd Mode (%)'] = None
            
    df_qr.name = 'Categorical Qualilty Report'
    return df_qr
def createDataQualityReports(df):
    """ 
    Return both the quantitative and categorical reports that
    are created by the createQuantitativeReport() and 
    createCategoricalReport() functions, respectively.
    
    Args:
        df: pandas.core.frame.DataFrame
            The DataFrame should hold the data that we want to 
            investigate. Each row in the DataFrame represents an 
            individual sample point, with the successive columns 
            corresponding to the variables and their specific values 
            for that sample point. Furthermore, the variables must be 
            separated by 'categorical' and 'quantitative' indexes
            using hierarchical indexing.

    Returns:
        pandas.core.frame.DataFrame, pandas.core.frame.DataFrame: A
            tuple containing the two aforementioned data quality reports
            for quantitative and categorical variables. See the
            createQuantitativeReport() and reateCategoricalReport() 
            functions for more details.
    """
    index = df.columns.tolist()
    if any('quantitative' == el[0] for el in index):
        df_qr_quantitative = createQuantitativeReport(df.xs('quantitative', axis=1))
    else:
        df_qr_quantitative = 'There are no quantitative variables present in this DataFrame.'
        
    if any('categorical' == el[0] for el in index):
        df_qr_categorical = createCategoricalReport(df.xs('categorical', axis=1))
    else:
        df_qr_categorical = 'There are no categorical variables present in this DataFrame.'
    return df_qr_quantitative, df_qr_categorical
def plotQuantitativeVariables(df, width=15, height=6, approach='equal-width', bins=10):
    """
    This function returns frequency plots for each quantitative variable
    in the pandas DataFrame. 
    
    For variables with cardinality (ie, the number of distinct values per 
    variable) less than 10, we use bar plots. Otherwise, we use histograms.
    These histograms are created using either equal-width or equal-
    frequency binning.
    
    Args:
        df: pandas.core.frame.DataFrame
            The DataFrame should hold the data that we want to 
            investigate. Each row in the DataFrame represents an 
            individual sample point, with the successive columns 
            corresponding to the variables and their specific values 
            for that sample point. 
        width: float, default 15
            The width of each subplot.
        height: float, default 6
            The height of each subplot.
        approach: str {'equal-width', 'equal-frequency'}, default 'equal-width'
            If 'equal-width', then the histograms are created using equal-width
            binning. If 'equal-frequency', then the histograms are created 
            using equal-frequency binning.
        bins: int, default 10
            Number of bins used to create the histograms.
    """
    assert approach in {'equal-width', 'equal-frequency'}, f'approach = {approach}'
    fontdict = {'fontsize': 18, 'fontweight': 'medium', 'color': 'orange'}
    numberOfSubplots = df.shape[1]
    fig, axes = plt_func.subplots(numberOfSubplots, 1, constrained_layout=True, figsize=(width, height*numberOfSubplots))
    columns = list(df.columns)
    sr_cardinality = df.nunique(dropna=True)
    for index, column in enumerate(columns):
        cardinality = sr_cardinality.loc[column]
        axes[index].tick_params(axis='both', colors='orange')
        if cardinality < 10:
            if not df_con[column].value_counts().empty:
                df[column].value_counts().sort_index().plot(kind='bar', ax=axes[index])
            axes[index].set_title(f'Bar plot of the \'{column}\' variable', fontdict=fontdict)
            axes[index].tick_params(axis="x", labelrotation=0)
        elif approach == 'equal-width':
            df[column].plot(kind='hist', bins=bins, ax=axes[index],
                                                figsize=(width, height*numberOfSubplots))
            axes[index].set_ylabel(None)         
            axes[index].set_title(f'Equal-width histogram of the \'{column}\' variable', fontdict=fontdict)
        else:
            buckets = pd.qcut(df['weight'], q=bins).value_counts().sort_index()
            buckets.plot(kind='bar', ax=axes[index],
                         figsize=(width, height*numberOfSubplots))
            axes[index].set_title(f'Equal-frequency histogram of the \'{column}\' variable', fontdict=fontdict)
            axes[index].tick_params(axis="x", labelsize=10, labelrotation=45)
def plotCategoricalVariables(df, width=15, height=4):
    """
    This function returns bar plots for each categorical variable
    in the pandas DataFrame. 
    
    Args:
        df: pandas.core.frame.DataFrame
            The DataFrame should hold the data that we want to 
            investigate. Each row in the DataFrame represents an 
            individual sample point, with the successive columns 
            corresponding to the variables and their specific values 
            for that sample point.
        width: float, default 15
            The width of each subplot.
        height: float, default 4
            The height of each subplot.
    """
    fontdict = {'fontsize': 18, 'fontweight': 'medium', 'color': 'orange'}
    numberOfSubplots = df.shape[1]
    fig, axes = plt_func.subplots(numberOfSubplots, 1, constrained_layout=True, figsize=(width, height*numberOfSubplots))
    columns = list(df.columns)
    if numberOfSubplots == 1:
        column = columns[0]
        df[column].value_counts().sort_index().plot(kind='bar', ax=axes)
        axes.tick_params(axis='both', colors='orange')
        axes.set_title(f'Bar plot of the \'{column}\' variable', fontdict=fontdict)
        axes.tick_params(axis="x", labelrotation=0)
    else:
        for index, column in enumerate(columns):
            df[column].value_counts().sort_index().plot(kind='bar', ax=axes[index])
            axes[index].tick_params(axis='both', colors='orange')
            axes[index].set_title(f'Bar plot of the \'{column}\' variable', fontdict=fontdict)
            axes[index].tick_params(axis="x", labelrotation=0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10500779760873333, 'fraction_numerical': 0.008057529024432507, 'mean_word_length': 2.7211476466795617, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '316586', 'license': 'MIT', 'n_tokens_mistral': 3375, 'n_tokens_neox': 3108, 'n_words': 1030, 'path': 'customModules/DataQualityReports.ipynb', 'repo_name': 'HerrBroekman/Applied-solution-of-An-Introduction-to-Statistical-Learning-written-in-Python', 'size': '11542'}