<jupyter_start><jupyter_text>Pandas Interoperability<jupyter_code>#code adapted from https://github.com/thomasjpfan/ml-workshop-intermediate-1-of-2
import pandas as pd
url = 'https://raw.githubusercontent.com/davidrkearney/Kearney_Data_Science/master/_notebooks/df_panel_fix.csv'
df = pd.read_csv(url, error_bad_lines=False)
df

import pandas as pd
import sklearn
from sklearn.datasets import fetch_openml
from sklearn.model_selection import train_test_split



df.columns

sklearn.set_config(display='diagram')

df=df.dropna()

df.isnull().sum()



X, y = df.drop(['specific', 'Unnamed: 0'], axis = 1), df['specific']

X


_ = X.hist(figsize=(30, 15), layout=(5, 8))
import numpy as np
import sklearn
sklearn.set_config(display='diagram')<jupyter_output><empty_output><jupyter_text>Categorical Data  OridinalEncoder<jupyter_code>from sklearn.preprocessing import OrdinalEncoder
ord_encoder = OrdinalEncoder()
ord_encoder.fit_transform(df)
ord_encoder.categories_
ord_encoder.transform(df)<jupyter_output><empty_output><jupyter_text>Categories that are unknown during `fit`  How to handle unknown categories in OridinalEncoder?  Provide all the categories in the constructor  OneHotEncoder<jupyter_code>from sklearn.preprocessing import OneHotEncoder

ohe = OneHotEncoder()
X_trans = ohe.fit_transform(df)
X_trans
X_trans.toarray()<jupyter_output><empty_output><jupyter_text>Switch to dense<jupyter_code>ohe = OneHotEncoder(sparse=False)
ohe.fit_transform(df)<jupyter_output><empty_output><jupyter_text>Unknown categories during transform?  OHE can handle unknowns<jupyter_code>ohe = OneHotEncoder(sparse=False, handle_unknown='ignore')
ohe.fit(df)
ohe.transform(df)
ohe.categories_<jupyter_output><empty_output><jupyter_text>Two categorical features<jupyter_code>df_train = pd.DataFrame({
    "province": ["Zhejiang", "Beijing", "Shanghai"],
    "region": ["East China", "North China", "Southwest China"]
})
ohe.fit(df_train)
ohe.categories_
ohe.transform(df_train)<jupyter_output><empty_output><jupyter_text>Column Transformer!  With OridinalEncoder<jupyter_code>from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler
df
ct = ColumnTransformer([
    ('numerical', StandardScaler(), ['fdi', 'gdp']),
    ('categorical', OrdinalEncoder(), ['reg'])
])

ct.fit_transform(df)<jupyter_output><empty_output><jupyter_text>With OneHotEncoder<jupyter_code>ct = ColumnTransformer([
    ('numerical', StandardScaler(), ['fdi', 'gdp']),
    ('categorical', OneHotEncoder(), ['reg'])
])
ct.fit_transform(df)
# df
y
X.head()<jupyter_output><empty_output><jupyter_text>Are three categories already encoded in the dataset?<jupyter_code>X.dtypes<jupyter_output><empty_output><jupyter_text>Are there missing values in the dataset?<jupyter_code>missing_values = pd.concat({"na_cnt": X.isna().sum(), "dtypes": X.dtypes}, axis='columns')
missing_values<jupyter_output><empty_output><jupyter_text>Split data into training and test set<jupyter_code>from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    X, y, random_state=42)<jupyter_output><empty_output><jupyter_text>ColumnTransformer<jupyter_code>missing_values
X<jupyter_output><empty_output><jupyter_text>Numerical preprocessing<jupyter_code>numerical_features = ['general', 'gdp', 'fdi', 'i', 'rr']
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer

num_prep = Pipeline([
    ('imputer', SimpleImputer()),
    ('scaler', StandardScaler())
])
num_prep<jupyter_output><empty_output><jupyter_text>Running only on numerical features<jupyter_code>num_trans = num_prep.fit_transform(X_train[numerical_features])
num_trans
num_trans.shape
X<jupyter_output><empty_output><jupyter_text>Categorical preprocessing<jupyter_code>categorical_features = ['province', 'reg']
cat_prep = Pipeline([
    ('imputer', SimpleImputer(strategy='constant', fill_value='sk_missing')),
    ('ohe', OneHotEncoder(handle_unknown='ignore', sparse=False))
])
cat_prep<jupyter_output><empty_output><jupyter_text>Running only on the categorical features<jupyter_code>cat_trans = cat_prep.fit_transform(X_train[categorical_features])
cat_trans
cat_trans.shape<jupyter_output><empty_output><jupyter_text>ColumnTransformer!<jupyter_code>ct = ColumnTransformer([
   ('numerical', num_prep, numerical_features),
   ('categorical', cat_prep, categorical_features)
])
ct
X_trans = ct.fit_transform(X_train)
X_trans[:, :5]
X_trans[:, 5:]
X_trans.shape<jupyter_output><empty_output><jupyter_text>Linear model<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
lin_reg = Pipeline([
    ('preprocess', ct),
    ('lin_reg', LinearRegression())
])
lin_reg
lin_reg.fit(X_train, y_train)
lin_reg.score(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>from sklearn.ensemble import RandomForestRegressor
rf = Pipeline([
    ('preprocess', ct),
    ('log_reg', RandomForestRegressor(random_state=42))
])
rf
rf.fit(X_train, y_train)
rf.score(X_train, y_train)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12307692307692308, 'fraction_numerical': 0.00311587147030185, 'mean_word_length': 9.375757575757575, 'pattern_counts': {'":': 4, '<': 77, '<?xml version=': 0, '>': 77, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '528682', 'license': 'Apache-2.0', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1817, 'n_words': 298, 'path': '_notebooks/2021-05-31-pandas-interoperability.ipynb', 'repo_name': 'davidrkearney/Kearney_Data_Science', 'size': '5135'}