<jupyter_start><jupyter_text>Import utilities<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
sns.set_style("whitegrid")
from itertools import combinations
from sklearn.preprocessing import PolynomialFeatures<jupyter_output><empty_output><jupyter_text>Import Data<jupyter_code>from Ingestion.kivadataloader import KivaDataLoader
import pandas as pd

m = KivaDataLoader()
cleaneduploans = m.get_clean_dataframe()<jupyter_output>Connected to kivateam.cnd5idfvxmld.us-east-1.rds.amazonaws.com:5432/kiva
The process takes about 5 minutes to run.<jupyter_text>Add Interactions/two-way<jupyter_code>def add_interactions(cleaneduploans):
    # Get feature names
    combos = list(combinations(list(cleaneduploans.columns),2))
    colnames = list(cleaneduploans.columns) + ['_'.join(x) for x in combos]
    
    # Find Interactions
    poly = PolynomialFeatures(interaction_only=True, include_bias=False)
    cleaneduploans = poly.fit_transform(cleaneduploans)
    cleaneduploans = pd.DataFrame(cleaneduploans)
    cleaneduploans.columns = colnames
    
    # Remove interaction terms with all 0 values
    noint_indices = [i for i, x in enumerate(list((cleaneduploans == 0).all())) if x]
    cleaneduploans = cleaneduploans.drop(cleaneduploans.columns[noint_indices], axis=1)
    
    return cleaneduploans
    
# From Yamil - our columns excluding dates
X = cleaneduploans.drop(['posted_to_raised_days', 'posted_to_raised_bins', 'disburse_to_posted_days','disbursed_to_raised_days','partner_id'], axis=1)
# From Yamil - Using numeric values only
X = X.select_dtypes(exclude=['object','datetime64[ns, UTC]'])
# From Yamil - dropping NaN values
X.dropna(axis=1, inplace=True)
X.shape<jupyter_output><empty_output><jupyter_text>Don't run code below<jupyter_code># Two-way interaction our features. As it currently stands, the 39 features will crearte C(39,2) = 741 new features.
# We should not run this code until we decide on the features. 
# We will utilize this if some of our original features produce weak relationship with target.
X = add_interactions(X)
print(X.head(5))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09225776541492814, 'fraction_numerical': 0.011126564673157162, 'mean_word_length': 6.052287581699346, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '528458', 'license': 'MIT', 'n_tokens_mistral': 743, 'n_tokens_neox': 730, 'n_words': 171, 'path': 'JupyterInvestigations/feature_engineering_feature_interaction.ipynb', 'repo_name': 'yavah/Kiva', 'size': '2157'}