<jupyter_start><jupyter_text><jupyter_code>#!pip install eli5 pdpbox category_encoders
 # from google.colab import files
 # files.upload()
 !unzip caterpillar-tube-pricing.zip
 !unzip data.zip
from glob import glob
import pandas as pd

def search_column(name):
    for path in glob('competition_data/*.csv'):
        df = pd.read_csv(path)
        if name in df.columns:
            print(path, df.shape)
            print(df.columns.tolist(), '\n')
search_column('thickness')

import pandas as pd
mats = pd.read_csv('competition_data/bill_of_materials.csv')

ac = mats.melt(id_vars='tube_assembly_id', 
                                     value_vars=[f'component_id_{n}' for n in range(1,9)])

ac = (ac
                       .sort_values(by='tube_assembly_id')
                       .dropna()
                       .rename(columns={'value': 'component_id'}))

ac.head(10)
comp = pd.read_csv('competition_data/components.csv')
act = ac.merge(comp, how='left')
act.head(3)
table = pd.crosstab(act['tube_assembly_id'], 
                    act['component_type_id'])

table = table.reset_index()
table.columns.name = ''
print(table.shape)
table.head()
comp_float = pd.read_csv('competition_data/comp_float.csv')
comp_float['thickness'].value_counts()
comp_float.head()
comp_float['weight']
comp_float.head()
comp = pd.concat((pd.read_csv(path) for path in glob('competition_data/comp_*.csv')), sort=False)
columns = ['component_id', 'component_type_id', 'orientation', 'unique_feature', 'weight']
comp = comp[columns]
comp['orientation'] = (comp['orientation']=='Yes').astype(int)
comp['unique_feature'] = (comp['unique_feature']=='Yes').astype(int)
comp['weight'] = comp['weight'].fillna(comp['weight'].median())
comp.head()
mats.head()
comp_float.head()
comp_float['bolt_pattern_area'] = (comp_float[f'bolt_pattern_long']/ comp_float[f'bolt_pattern_wide'].fillna(0))
comp_float['bolt_pattern_long'] = (comp_float[f'bolt_pattern_long'].fillna(0)) 
comp_float['bolt_pattern_wide'] = (comp_float[f'bolt_pattern_wide'].fillna(0))
comp_float['thickness'] = (comp_float[f'thickness'].fillna(0))



comp_float = comp_float.merge(comp, left_on=f'component_id', right_on='component_id',how='left')

for col in comp_float:
    if 'orientation' in col or 'weight' in col:
        comp_float[col] = comp_float[col].fillna(0)
        
comp_float['orientation'] = (comp_float['orientation'])
comp_float['weight'] = (comp_float['weight'])
comp_float['bolt_pattern_area'] = (comp_float[f'bolt_pattern_long']/ comp_float[f'bolt_pattern_wide'])

features = ['component_id',	'bolt_pattern_long',	'bolt_pattern_wide',
            'thickness',	'orientation',	'weight']
comp_float = comp_float[features]
print(comp_float.shape)
comp_float.head()
mats['components_total'] = sum(mats[f'quantity_{n}'].fillna(0)  for n in range(1,9))
mats['components_distinct'] = sum(mats[f'component_id_{n}'].notnull().astype(int) for n in range(1,9))
mats['weight'] = 0

for n in range(1,9):
    mats = mats.merge(comp, left_on=f'component_id_{n}', right_on='component_id', 
                                how='left', suffixes=('', f'_{n}'))

for col in mats:
    if 'orientation' in col or 'unique_feature' in col or 'weight' in col:
        mats[col] = mats[col].fillna(0)
        
mats['orientation'] = sum(mats[f'orientation_{n}'] for n in range(1,9))
mats['unique_feature'] = sum(mats[f'unique_feature_{n}'] for n in range(1,9))
materials['weight'] = sum(mats[f'weight_{n}'] for n in range(1,9))

features = ['tube_assembly_id', 'orientation', 'unique_feature', 'weight', 
            'components_total', 'components_distinct', 'component_id_1']
mats = mats[features]
print(mats.shape)
mats.head()
tube = pd.read_csv('competition_data/tube.csv')
tube.head()
from sklearn.model_selection import train_test_split

trainval = pd.read_csv('competition_data/train_set.csv')
test = pd.read_csv('competition_data/test_set.csv')

trainval_tube_assemblies = trainval['tube_assembly_id'].unique()
train_tube_assemblies, val_tube_assemblies = train_test_split(
    trainval_tube_assemblies, random_state=42)
train = trainval[trainval.tube_assembly_id.isin(train_tube_assemblies)]
val = trainval[trainval.tube_assembly_id.isin(val_tube_assemblies)]

def clean(X):
    X = X.copy()
    
    # Engineer date features
    X['quote_date'] = pd.to_datetime(X['quote_date'], infer_datetime_format=True)
    X['quote_date_year'] = X['quote_date'].dt.year
    X['quote_date_month'] = X['quote_date'].dt.month
    X = X.drop(columns='quote_date')
    
    X = (X.merge(table, how='left')
         .merge(mats, how='left')
         .merge(tube, how='left')
         .fillna(0))
    
    # Drop tube_assembly_id because our goal is to predict unknown assemblies
    X = X.drop(columns='tube_assembly_id')
    return X

train = clean(train)
val = clean(val)
test = clean(test)
import numpy as np

target = 'cost'
X_train = train.drop(columns=target)
X_val = val.drop(columns=target)
X_test = test.drop(columns='id')
y_train = train[target]
y_val = val[target]
y_train_log = np.log1p(y_train)
y_val_log = np.log1p(y_val)
import category_encoders as ce
from xgboost import XGBRegressor
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

encoder = ce.OrdinalEncoder()
X_train_encoded = encoder.fit_transform(X_train)
X_val_encoded = encoder.transform(X_val)
eval_set = [(X_train_encoded, y_train_log), 
            (X_val_encoded, y_val_log)]
model = XGBRegressor(n_estimators=2000, n_jobs=-1)
model.fit(X_train_encoded, y_train_log, 
          eval_set=eval_set, eval_metric='rmse', early_stopping_rounds=50)
def generate_submission(estimator, X_test, filename):
    y_pred_log = estimator.predict(X_test)
    y_pred = np.expm1(y_pred_log) 
    submission = pd.read_csv('sample_submission.csv')
    submission['cost'] = y_pred
    submission.to_csv(filename, index=False)
    
X_test_encoded = encoder.transform(X_test)
generate_submission(model, X_test_encoded, 'submission.csv')
import matplotlib.pyplot as plt

importances = pd.Series(model.feature_importances_, X_train_encoded.columns)

n = len(X_train_encoded.columns)
plt.figure(figsize=(10,n/2))
plt.title(f'Top {n} features')
importances.sort_values()[-n:].plot.barh(color='purple');
%%time
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score

def rmse(y_true, y_pred):
    return np.sqrt(mean_squared_error(y_true, y_pred))

column  = 'annual_usage'
model = XGBRegressor(n_estimators=1000, n_jobs=-1)
model.fit(X_train_encoded.drop(columns=column), y_train_log)
y_pred_log = model.predict(X_val_encoded.drop(columns=column))
score_without = rmse(y_val_log, y_pred_log)
print(f'Validation RMSLE without {column}:', score_without)

model = XGBRegressor(n_estimators=1000, n_jobs=-1)
model.fit(X_train_encoded, y_train_log)
y_pred_log = model.predict(X_val_encoded)
score_with = rmse(y_val_log, y_pred_log)
print(f'Validation RMSLE with {column}:', score_with)

print(f'Drop-Column Importance for {column}:', score_without - score_with)
feature = 'quantity'
x_val_permuted = X_val_encoded.copy()
x_val_permuted[feature] = np.random.permutation(X_val_encoded[feature])
y_pred_log = model.predict(x_val_permuted)
score_permuted = rmse(y_val_log, y_pred_log)
print(f'Validation RMSLE with {feature}:', score_with)
print(f'Validation RMSLE with {feature} permuted:', score_permuted)
print(f'Permutation Importance:', score_permuted - score_with)
feature = 'annual_usage'
X_val_permuted = X_val_encoded.copy()
X_val_permuted[feature] = np.random.permutation(X_val_encoded[feature])
y_pred_log = model.predict(x_val_permuted)
score_permuted = rmse(y_val_log, y_pred_log)
print(f'Validation RMSLE with {feature}:', score_with)
print(f'Validation RMSLE with {feature} permuted:', score_permuted)
print(f'Permutation Importance:', score_permuted - score_with)
import eli5
from eli5.sklearn import PermutationImportance

permuter = PermutationImportance(model, scoring='neg_mean_squared_error', 
                                 cv='prefit', n_iter=2, random_state=42)

permuter.fit(X_val_encoded, y_val_log)
feature_names = X_val_encoded.columns.tolist()
eli5.show_weights(permuter, top=None, feature_names=feature_names)
print('Shape before removing features:', X_train.shape)
mask = permuter.feature_importances_ > 0
features = X_train.columns[mask]
X_train = X_train[features]
print('Shape after removing features:', X_train.shape)
X_val = X_val[features]
encoder = ce.OrdinalEncoder()
X_train_encoded = encoder.fit_transform(X_train)
X_val_encoded = encoder.transform(X_val)
model = XGBRegressor(n_estimators=1000, n_jobs=-1)
model.fit(X_train_encoded, y_train_log)
y_pred_log = model.predict(X_val_encoded)
print(f'Validation RMSLE', rmse(y_val_log, y_pred_log))
from pdpbox.pdp import pdp_interact, pdp_interact_plot

features = ['min_order_quantity', 'quantity']

interaction = pdp_interact(
    model=model, 
    dataset=X_val_encoded, 
    model_features=X_val_encoded.columns, 
    features=features
)

pdp_interact_plot(interaction, plot_type='grid', feature_names=features);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13672391017173052, 'fraction_numerical': 0.0071554381329810655, 'mean_word_length': 5.810344827586207, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '391405', 'license': 'MIT', 'n_tokens_mistral': 3377, 'n_tokens_neox': 3225, 'n_words': 548, 'path': 'Daniel_Calimayor_permutation_importances_partial_dependence_plots_Assignment.ipynb', 'repo_name': 'DanielCalimayor/DS-Unit-2-Regression-2', 'size': '9084'}