<jupyter_start><jupyter_text>Predicting Cost of CaterpillarTM Tube Assemblies.<jupyter_code># Imports
import re
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from pprint import pprint
from glob import glob
from category_encoders import OrdinalEncoder
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_log_error, mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
# from xgboost import XGBRegressor
%matplotlib inline
pd.set_option('display.max_columns', 250)<jupyter_output><empty_output><jupyter_text>Begin examining, cleaning and organizing data.<jupyter_code># Get data into dataframes.
trainval = pd.read_csv('data/competition_data/train_set.csv')
test = pd.read_csv('data/competition_data/test_set.csv')
# See what we're working with.
trainval.head()
test.head()
trainval.shape, test.shape
# Ensure dates are datetimes.
trainval['quote_date'] = pd.to_datetime(trainval['quote_date'], infer_datetime_format=True)
test['quote_date'] = pd.to_datetime(test['quote_date'], infer_datetime_format=True)

# Is our data chronological?
trainval['quote_date'].describe()
test['quote_date'].describe()
# Dates are concurrent. The sets must be separated in another way.
set(trainval['tube_assembly_id'].unique()) & set(test['tube_assembly_id'].unique())
# Assembly id is not shared, split training data into train and validate by assembly id.
trainval_assemblies = trainval['tube_assembly_id'].unique()
test_assemblies = test['tube_assembly_id'].unique()

train_tubes, val_tubes = train_test_split(trainval_assemblies, random_state=42)

train = trainval[trainval['tube_assembly_id'].isin(train_tubes)]
val = trainval[trainval['tube_assembly_id'].isin(val_tubes)]
train.shape, val.shape, test.shape<jupyter_output><empty_output><jupyter_text>Get a dummy baseline prediction for later comparison.<jupyter_code># Our error metric will be Root Mean Squared Log Error
def rmsle(y_true, y_pred):
    return np.sqrt(mean_squared_log_error(y_true, y_pred))
# Use the cost mean to make our cost prediction.
target = 'cost'
y_train = train[target]
y_val = val[target]

y_pred = np.full_like(y_val, fill_value=y_train.mean())
print(f'Validation RMSLE, Mean Baseline: {rmsle(y_val, y_pred)}')<jupyter_output>Validation RMSLE, Mean Baseline: 0.9418101276064408<jupyter_text>Off by 94%. Improvements can be made. Start with a very simple model.<jupyter_code>features = ['quantity']
X_train = train[features]
X_val = val[features]

model = RandomForestRegressor(n_estimators=100,
                              n_jobs=3,
                              random_state=42)
model.fit(X_train, y_train)
r2 = model.score(X_val, y_val)
print(f'Validation R^2 score: {r2}')

y_pred = model.predict(X_val)
print(f'Validation RMSLE, with one feature: {features[0]}\n: {rmsle(y_val, y_pred)}')<jupyter_output>Validation R^2 score: 0.08879518736792147
Validation RMSLE, with one feature: quantity
: 0.6908805822519186<jupyter_text>Quite a bit better than before, but much more room to improve.<jupyter_code># Check out the distribution of the target, cost.
sns.distplot(y_train);
# We can get these into a smaller space by taking the log of each target value.
y_train_log = np.log1p(y_train)
sns.distplot(y_train_log);
# Now predict with a model fit on scaled targets.
model.fit(X_train, y_train_log)
y_pred_log = model.predict(X_val)
y_pred = np.expm1(y_pred_log)
print(f'RMSLE with scaled targets: {rmsle(y_val, y_pred)}')<jupyter_output>RMSLE with scaled targets: 0.622925465880199<jupyter_text>A 7% improvement over the non-scaled targets.<jupyter_code>y_val_log = np.log1p(y_val)
sns.distplot(y_val_log, label='True')
sns.distplot(y_pred_log, label='Predicted')
plt.title('Starting to fit the data somewhat.')
plt.legend();<jupyter_output><empty_output><jupyter_text>Engineer some features from the data we have so far.<jupyter_code>def clean(data):
    df = data.copy()
    
    df['quote_date'] = pd.to_datetime(df['quote_date'], infer_datetime_format=True)
    df['quote_year'] = df['quote_date'].dt.year
    df['quote_month'] = df['quote_date'].dt.month
    df = df.drop(columns='quote_date')
    
    # df = df.drop(columns='tube_assembly_id')
    return df
train_clean = clean(train)
val_clean = clean(val)
test_clean = clean(test)
train_clean.shape, val_clean.shape, test_clean.shape
def rmse(y_true, y_pred):
    return np.sqrt(mean_squared_error(y_true, y_pred))
# Use new features and assess any improvement.
features = train_clean.columns.drop('cost')
X_train = train_clean[features]
X_val = val_clean[features]
pipeline = make_pipeline(OrdinalEncoder(),
                         RandomForestRegressor(n_estimators=100,
                                               n_jobs=3,
                                               random_state=42))
pipeline.fit(X_train, y_train_log)
y_pred_log = pipeline.predict(X_val)
y_pred = np.expm1(y_pred_log)
print(f'RMSE for model with new features: {rmse(y_val_log, y_pred_log)}')
print(f'RMSLE: {rmsle(y_val, y_pred)}')<jupyter_output>RMSE for model with new features: 0.43919385455635707
RMSLE: 0.43919385455635707<jupyter_text>Nice!<jupyter_code># What features are most important for making predictions?
rf = pipeline.named_steps['randomforestregressor']
importances = pd.Series(rf.feature_importances_, X_train.columns)
importances.sort_values().plot.barh()
# Have we improved fit any more?
sns.distplot(y_val_log, label='True')
sns.distplot(y_pred_log, label='Predicted')
plt.title('Fit to data with all features.')
plt.legend();<jupyter_output><empty_output><jupyter_text>Add more features by combining relational data.<jupyter_code>data = {}
for path in glob('data/competition_data/*.csv'):
    df = pd.read_csv(path)
    file = re.search(r'(?<=\\)(.+)(?=[.])', path)
    data[file.group(1)] = df
data['tube'].head()
# We can left join this with our current data for more features.
train = train_clean.merge(data['tube'], how='left')
val = val_clean.merge(data['tube'], how='left')
test = test_clean.merge(data['tube'], how='left')
train.shape, val.shape, test.shape
# train = clean(train)
# val = clean(val)
# test = clean(test)

features = train.columns.drop('cost')

X_train = train[features]
X_val = val[features]
pipeline = make_pipeline(OrdinalEncoder(),
                         RandomForestRegressor(n_estimators=10,
                                               n_jobs=3,
                                               random_state=42))
pipeline.fit(X_train, y_train_log)
y_pred_log = pipeline.predict(X_val)
y_pred = np.expm1(y_pred_log)
print(f'RMSE with additional features: {rmse(y_val_log, y_pred_log)}')
train = train.merge(data['bill_of_materials'], on='tube_assembly_id', how='left')
val = val.merge(data['bill_of_materials'], on='tube_assembly_id', how='left')
test = test.merge(data['bill_of_materials'], on='tube_assembly_id', how='left')
train.shape, val.shape, test.shape
# features = train.drop(columns=['tube_assembly_id', 'cost']).columns
X_train = train[features]
X_val = val[features]
pipeline = make_pipeline(OrdinalEncoder(),
                         RandomForestRegressor(n_estimators=100,
                                               n_jobs=3,
                                               random_state=42))
pipeline.fit(X_train, y_train_log)
y_pred_log = pipeline.predict(X_val)
print(f'RMSE with even more features: {rmse(y_val_log, y_pred_log)}')
train.head()
data.keys()
data['comp_adaptor'].head()
data['comp_hfl'].head()
def merge_components(data, train, title, columns):
#     for x in range(1, 9):
    x = 1
    component = 'component_id_' + str(x)
    data[component] = data['component_id']
    columns = [component] + columns
    new = data[columns].copy()
    new_columns = [title + '_' + name + '_' + str(x) for name in new.columns[1:]]
    new_columns = [component] + new_columns
    new.columns = new_columns
    new = new.set_index(component)
    columns = columns[1:]
#     print(new.columns)
    for i, comp in enumerate(train[component]):
        for column in new.columns:
            train[column] = np.nan
            if comp in new.index:
                value = new[column].loc[comp]
                train.loc[i:column] = value
#                 print('bing', value)
    return train
cols = ['component_type_id', 'material', 'plating', 'weight']
hfl = data['comp_hfl']
train = merge_components(hfl, train, 'hfl', cols)
val = merge_components(hfl, val, 'hfl', cols)
test = merge_components(hfl, test, 'hfl', cols)
train.shape, val.shape, test.shape
train['hfl_plating_1'].value_counts()
train.isnull().sum()
cols = ['component_type_id', 'unique_feature', 'weight']
adapt = data['comp_adaptor']
train = merge_components(adapt, train, 'adapt', cols)
val = merge_components(adapt, val, 'adapt', cols)
test = merge_components(adapt, test, 'adapt', cols)
train.shape, val.shape, test.shape
cols = ['component_type_id', 'groove', 'unique_feature', 'weight']
boss = data['comp_boss']
train = merge_components(boss, train, 'boss', cols)
val = merge_components(boss, val, 'boss', cols)
test = merge_components(boss, test, 'boss', cols)
train.shape, val.shape, test.shape
train.isnull().sum()
cols = ['component_type_id', 'groove', 'unique_feature', 'weight']
straight = data['comp_straight']
train = merge_components(straight, train, 'strt', cols)
val = merge_components(straight, val, 'strt', cols)
test = merge_components(straight, test, 'strt', cols)
train.shape, val.shape, test.shape
cols = ['component_type_id', 'unique_feature', 'weight']
thread = data['comp_threaded']
train = merge_components(thread, train, 'tread', cols)
val = merge_components(thread, val, 'tread', cols)
test = merge_components(thread, test, 'tread', cols)
train.shape, val.shape, test.shape
cols = ['component_type_id', 'groove', 'unique_feature', 'weight']
elbow = data['comp_elbow']
train = merge_components(elbow, train, 'elbow', cols)
val = merge_components(elbow, val, 'elbow', cols)
test = merge_components(elbow, test, 'elbow', cols)
train.shape, val.shape, test.shape
train = train.fillna('none')
val = val.fillna('none')
test = test.fillna('none')
train.shape, val.shape, test.shape
target = 'cost'
X_train = train.drop(columns=target)
X_val = val.drop(columns=target)
y_train = train[target]
y_train_log = np.log1p(y_train)
y_val = val[target]
test = test.drop(columns='id')
X_train.info()
encoder = OrdinalEncoder()
X_train = encoder.fit_transform(X_train.values)
X_val = encoder.transform(X_val.values)
test = encoder.transform(test.values)
pipeline = make_pipeline(OrdinalEncoder(),
                        RandomForestRegressor(n_estimators=100,
                                             n_jobs=3,
                                             random_state=42))
pipeline.fit(X_train.values, y_train_log)
y_pred_log = pipeline.predict(X_val.values)
print(f'Validation RMSE with hella features: {rmse(y_val_log, y_pred_log)}')
test.head()
log_pred = pipeline.predict(test.values)
pred = np.expm1(log_pred)
submission = pd.read_csv('sample_submission.csv')
submission['cost'] = pred
submission.to_csv('sub_0.csv', index=False)
train.head()
eval_set = [(X_train, y_train_log), 
            (X_val, y_val_log)]


model = XGBRegressor(n_estimators=1000, n_jobs=-1, verbose=1)
model.fit(X_train, y_train_log, eval_set=eval_set, eval_metric='rmse', 
          early_stopping_rounds=10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.129342784632887, 'fraction_numerical': 0.01478953356086462, 'mean_word_length': 4.7980720446473875, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '50086', 'license': 'MIT', 'n_tokens_mistral': 4059, 'n_tokens_neox': 3871, 'n_words': 841, 'path': 'module1-log-linear-regression/Paul_K_Regression_2.ipynb', 'repo_name': 'llpk79/DS-Unit-2-Regression-2', 'size': '11427'}