<jupyter_start><jupyter_text>Pandas PipesCleaner Data Analysis with Pandas Using Pipes  https://towardsdatascience.com/cleaner-data-analysis-with-pandas-using-pipes-4d73770fbf3c Great idea for a standard set of utility cleanup functions in a libraryData:  https://www.kaggle.com/yoghurtpatil/direct-marketing?select=DirectMarketing.csv<jupyter_code>import numpy as np
import pandas as pd
marketing = pd.read_csv("./data/DirectMarketing.csv")
marketing.head()<jupyter_output><empty_output><jupyter_text>Functions used by PIPE<jupyter_code>def drop_missing(df):
    thresh = len(df) * 0.6
    df.dropna(axis=1, thresh=thresh, inplace=True)
    return df

def remove_outliers(df, column_name):
    low = np.quantile(df[column_name], 0.05)
    high = np.quantile(df[column_name], 0.95)
    return df[df[column_name].between(low, high, inclusive=True)]

def to_category(df):
    cols = df.select_dtypes(include='object').columns
    for col in cols:
        ratio = len(df[col].value_counts()) / len(df)
        if ratio < 0.05:
            df[col] = df[col].astype('category')
    return df

def copy_df(df):
   return df.copy()<jupyter_output><empty_output><jupyter_text>Pipe to clean a copy of the data<jupyter_code>marketing_cleaned = (marketing.
                       pipe(copy_df).
                       pipe(drop_missing).
                       pipe(remove_outliers, 'Salary').
                       pipe(to_category))
print(marketing.shape)
marketing.dtypes
print(marketing_cleaned.shape)
marketing_cleaned.dtypes<jupyter_output>(900, 10)
{'fraction_non_alphanumeric': 0.11521035598705502, 'fraction_numerical': 0.015533980582524271, 'mean_word_length': 3.83125, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 12, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '40722', 'license': 'MIT', 'n_tokens_mistral': 556, 'n_tokens_neox': 531, 'n_words': 98, 'path': '{{ cookiecutter.repo_name }}/notebooks/20-PandasPipes.ipynb', 'repo_name': 'kellington/cookiecutter-data-science', 'size': '1545'}