<jupyter_start><jupyter_text>Example of pipelining with Pandas with `pdpipe` Dr. <NAME>, Fremont, CA, Nov 2019<jupyter_code>import pandas as pd
import numpy as np
import pdpipe as pdp
df = pd.read_csv("USA_Housing.csv")
round(df.sample(5),2)
df.shape
df.columns
round(df.describe().T,2)
def size(n):
    if n<=4:
        return 'Small'
    elif 4<n<=6:
        return 'Medium'
    else:
        return 'Big'

df['House_size']=df['Avg. Area Number of Rooms'].apply(size)
df['House_size']=df['Avg. Area Number of Rooms'].apply(size)
round(df.sample(5),2)<jupyter_output><empty_output><jupyter_text>Drop a column<jupyter_code>drop_age = pdp.ColDrop('Avg. Area House Age')
df2 = drop_age(df)
round(df2.sample(5))<jupyter_output><empty_output><jupyter_text>Chaining stages by adding them up<jupyter_code>pipeline = pdp.ColDrop('Avg. Area House Age')
pipeline+= pdp.OneHotEncode('House_size')
df3 = pipeline(df)
round(df3.sample(5))
def price_tag(x):
    if x>250000:
        return 'keep'
    else:
        return 'drop'
pipeline = pdp.ColDrop('Avg. Area House Age')
pipeline+= pdp.OneHotEncode('House_size')
pipeline+=pdp.ApplyByCols('Price',price_tag,'Price_tag',drop=False)
df4 = pipeline(df)
df4.shape
round(df4.sample(5),2)
pipeline = pdp.ColDrop('Avg. Area House Age')
pipeline+= pdp.OneHotEncode('House_size')
pipeline+=pdp.ApplyByCols('Price',price_tag,'Price_tag',drop=False)
pipeline+=pdp.ValDrop(['drop'],'Price_tag')
pipeline+= pdp.ColDrop('Price_tag')
df5 = pipeline(df)
df5.shape<jupyter_output><empty_output><jupyter_text>Scikit-learn scaling<jupyter_code>pipeline_scale = pdp.Scale('StandardScaler',exclude_columns=['House_size_Medium','House_size_Small'])
df6 = pipeline_scale(df5)
round(df6.sample(5),3)<jupyter_output><empty_output><jupyter_text>NLTK stages<jupyter_code>pipeline_tokenize=pdp.TokenizeWords('Address')
df7 = pipeline_tokenize(df6)
df7.sample(5)
def extract_state(token):
    return str(token[-2])
pipeline_state = pdp.ApplyByCols('Address',extract_state,result_columns='State')
df8=pipeline_state(df7)
round(df8.sample(5),3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15355086372360843, 'fraction_numerical': 0.022072936660268713, 'mean_word_length': 7.407258064516129, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '424587', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 821, 'n_tokens_neox': 808, 'n_words': 129, 'path': 'Pandas and Numpy/pdpipe-example.ipynb', 'repo_name': 'thirupathi-chintu/Machine-Learning-with-Python', 'size': '2096'}