<jupyter_start><jupyter_text><jupyter_code>!wget https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Applied-Modeling/master/data/caterpillar/caterpillar-tube-pricing.zip
!unzip caterpillar-tube-pricing.zip
!unzip data.zip
SOURCE = '../data/caterpillar/caterpillar-tube-pricing/competition_data/'
SOURCE = 'competition_data/'
import pandas as pd

materials = pd.read_csv(SOURCE + 'bill_of_materials.csv')

components = pd.read_csv(SOURCE + 'components.csv')
from glob import glob

for path in glob('competition_data/*.csv'):
    df = pd.read_csv(path)
    print(path, df.shape)<jupyter_output>competition_data/type_end_form.csv (8, 2)
competition_data/comp_elbow.csv (178, 16)
competition_data/tube_end_form.csv (27, 2)
competition_data/type_connection.csv (14, 2)
competition_data/comp_adaptor.csv (25, 20)
competition_data/bill_of_materials.csv (21198, 17)
competition_data/comp_sleeve.csv (50, 10)
competition_data/comp_straight.csv (361, 12)
competition_data/comp_hfl.csv (6, 9)
competition_data/components.csv (2048, 3)
competition_data/comp_float.csv (16, 7)
competition_data/comp_threaded.csv (194, 32)
competition_data/test_set.csv (30235, 8)
competition_data/comp_nut.csv (65, 11)
competition_data/tube.csv (21198, 16)
competition_data/train_set.csv (30213, 8)
competition_data/specs.csv (21198, 11)
competition_data/comp_other.csv (1001, 3)
competition_data/type_component.csv (29, 2)
competition_data/comp_boss.csv (147, 15)
competition_data/comp_tee.csv (4, 14)<jupyter_text>We are evaluating based on RMSLE (Root Mean Squared Logarithmic Error).<jupyter_code>import numpy as np
from sklearn.metrics import mean_squared_log_error

#Create a function for finding RMSLE.

def rmsle(y_true, y_pred):
  return np.sqrt(mean_squared_log_error(y_true, y_pred))
#What kind of date ranges does our test set deal with?

#Are they overlapping date ranges and overlapping tube assemblies from train to test? If train is earlier dates and then test is later dates, then we
#probably want to do a time based split. That's telling us that the goal is to forecast out into the future using earlier information to forecast or predict later information. But if the
#test set has the same date ranges, then that is evidently not how this model is being used. We would then have to look at some different strategies for splitting.
train_and_val = pd.read_csv('competition_data/train_set.csv')
test = pd.read_csv('competition_data/test_set.csv')
#Let's take a look at our data.
#We specifically want to take a close look at the date ranges here.
train_and_val.head()
test.head()
#We are going to convert quote_date to a datetime in each data frame.

train_and_val['quote_date'] = pd.to_datetime(train_and_val['quote_date'], infer_datetime_format=True)

test['quote_date'] = pd.to_datetime(test['quote_date'], infer_datetime_format=True)
train_and_val['quote_date'].describe()
test['quote_date'].describe()
#These are highly overlapping date ranges. With this, their goal isn't to forecast from earlier information to later information. 
#Instead their goal might be to use information about tube assemblies that you have seen before to make predictions for tube assemblies that are unknown.
#Does the test set have different tube assemblies?

train_and_val['tube_assembly_id'].nunique()
test['tube_assembly_id'].nunique()
train_and_val_tube_assemblies = train_and_val['tube_assembly_id'].unique()

test_tube_assemblies = test['tube_assembly_id'].unique()

len(train_and_val_tube_assemblies), len(test_tube_assemblies)
#Show me the intersection.

set(train_and_val_tube_assemblies) & set(test_tube_assemblies)
#We notice that each set has almost the exact same number. But, do they have overlap?
#From previous information, we know that there is no overlap.
#Make the validation set like the test set.
train_and_val_tube_assemblies
from sklearn.model_selection import train_test_split

train_tube_assemblies, val_tube_assemblies = train_test_split(train_and_val_tube_assemblies, random_state=42)
len(train_tube_assemblies), len(val_tube_assemblies)
#Verifying that there is no intersection.

set(train_tube_assemblies) & set(val_tube_assemblies)

#An empty set means that we've done it correctly.
train = train_and_val[train_and_val.tube_assembly_id.isin(train_tube_assemblies)]

val = train_and_val[train_and_val.tube_assembly_id.isin(val_tube_assemblies)]

train.shape, val.shape, train_and_val.shape
len(train) + len(val) == len(train_and_val)<jupyter_output><empty_output><jupyter_text>**Establishing a baseline**<jupyter_code>#Numeric features of the train set.

train.describe()
#Non-numeric features of the train set.

train.describe(exclude='number')
import numpy as np
target = 'cost'

y_train = train[target]

y_val = val[target]

#We'll use the mean to establish a target baseline.

y_pred = np.full_like(y_val, fill_value=y_train.mean())
#Verify the length of y_pred.

len(y_pred) == len(y_val)
print('Validation RMSLE for Mean Baseline:', rmsle(y_val, y_pred))

#This is the error of our baseline. Now, we are looking to make a model as quickly as we can that produces an error lower than that.

#Overall, a lower RMSLE is better.
from sklearn.metrics import r2_score

print('Validation R^2 for Mean Baseline:', r2_score(y_val, y_pred))<jupyter_output>Validation R^2 for Mean Baseline: -4.701447715138585e-06<jupyter_text>**Let's develop a first model that performs better than our basic baseline.**<jupyter_code>train.cost.mean()
train.groupby('quantity').cost.mean()
#We'll start with using just the quantity feature.
#We've already defined target above.

features = ['quantity']
X_train = train[features]
X_val = val[features]
#Because we are projecting how many dollars(continuous variable), this is a regression problem.

from sklearn.ensemble import RandomForestRegressor

model = RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)

model.fit(X_train, y_train)

r2 = model.score(X_val, y_val)

print('Validation R^2:', r2)
y_pred = model.predict(X_val)

print(f'Validation RMSLE for Random Forest with {features}:')
print(rmsle(y_val, y_pred))
#Before we were off by .94 log dollars on average, and now we are off by only .69.
#One way we can get a better error here is by log transforming the regression target with our right skewed distribution.

#What does our distribution look like?

%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns

sns.distplot(y_train);
#Use numpy for exponents and log functions.

#Example

2 ** 3
#We can see that this undoes the above operation.

np.log2(8)<jupyter_output><empty_output><jupyter_text>**Let's go ahead and apply this to our training labels.**<jupyter_code>y_train_log = np.log1p(y_train)

#Let's go ahead and plot this.

sns.distplot(y_train_log);

#It's not a perfect bell curve by any means, but it will certainly make life easier for us.
#We've taken the log and now we are going to exponentiate it. So, we are transforming it again. This takes us back to where we started.

y_train_log_exp = np.expm1(y_train_log)

#And the plot for visual verification...

sns.distplot(y_train_log_exp);
#Let's refit our model with our log transformed target.

model.fit(X_train, y_train_log)   #Training the model on log dollars.

y_pred_log = model.predict(X_val) #Generating predictions in log dollars.

#Converting back to the original units...

y_pred = np.expm1(y_pred_log)

rmsle(y_val, y_pred)
#We observe a lower error.<jupyter_output><empty_output><jupyter_text>**RMSLE with original targets = RMSE with log-transformed targets**<jupyter_code>from sklearn.metrics import mean_squared_error

def rmse(y_true, y_pred):
  return np.sqrt(mean_squared_error(y_true, y_pred))
#Log transform of the validation targets

y_val_log = np.log1p(y_val)

rmse(y_val_log, y_pred_log ) #Keeping in the log dollar unit<jupyter_output><empty_output><jupyter_text>**Let's select some more features to use from our data.**<jupyter_code>!pip install category_encoders
#We'll remind ourselves of what features we have available to us...

train.head()
#We are going to engineer some features off of the quote date.

#Let's go ahead and define a wrangle function to do that...

def wrangle(X):
  X = X.copy()
  
  #Engineer date features
  
  X['quote_date'] = pd.to_datetime(X['quote_date'], infer_datetime_format=True)
  X['quote_date_year'] = X['quote_date'].dt.year
  X['quote_date_month'] = X['quote_date'].dt.month
  X = X.drop(columns='quote_date')
  
  #Drop tube_assembly_id because our goal is to predict unknown assemblies.
  
  X = X.drop(columns='tube_assembly_id')
  
  return X

train_wrangled = wrangle(train)
val_wrangled = wrangle(val)
features = train_wrangled.columns.drop(target)

print('Features:', features.tolist())

#These are the features we'll be using that are already conveniently available to us.
X_train = train_wrangled[features]
X_val = val_wrangled[features]
#Let's create a pipeline.

import category_encoders as ce
from sklearn.pipeline import make_pipeline

pipeline = make_pipeline(ce.OrdinalEncoder(), RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1))

pipeline.fit(X_train, y_train_log)
y_pred_log = pipeline.predict(X_val) #Again, these are in the log dollars units.

rmse(y_val_log, y_pred_log)

#We observe an even lower error.
#Equivalent when converting back from log dollars to regular dollars.

y_pred = np.expm1(y_pred_log)
rmsle(y_val, y_pred)<jupyter_output><empty_output><jupyter_text>**Looking at our feature importances.**<jupyter_code>rf = pipeline.named_steps['randomforestregressor']
importances = pd.Series(rf.feature_importances_, X_train.columns)

importances.sort_values()

#quantity in this simple model is associated with more feature importance than the other features included in our model.
#Let's plot this...

importances.sort_values().plot.barh(colors='grey');
#To better understand the relationship between quantity and the target becuase quantity is such an important feature...

quantity_quartiles = pd.qcut(train_wrangled['quantity'], q=4)

sns.pointplot(x=quantity_quartiles, y=train_wrangled['cost']);

#As quantity increases, the price drops. Notice that it is not linear. It's monotonically decreasing as we'd expect. At first it drops faster, and then the rate of change slows down.
sns.distplot(y_val_log, label='True')
sns.distplot(y_pred_log, label='Predicted')
plt.legend();<jupyter_output><empty_output><jupyter_text>**Feature engineering with relational data**<jupyter_code>for path in glob('competition_data/*.csv'):
  df = pd.read_csv(path)
  shared_columns = set(df.columns) & set(train.columns)
  if shared_columns:
    print(path, df.shape)
    print(df.columns.tolist(), '\n')
    
    #Try to see what columns each of these files might have in common with the training set.
train.columns
#We can merge on the columns our data frames have in common.
tube = pd.read_csv('competition_data/tube.csv')
#Numeric features

tube.describe()
#Non-numeric features

tube.describe(exclude='number')
#We are going to use the wrangle function we created above and redefine it.

def wrangle(X):
  X = X.copy()
  
  #Engineer date features
  
  X['quote_date'] = pd.to_datetime(X['quote_date'], infer_datetime_format=True)
  X['quote_date_year'] = X['quote_date'].dt.year
  X['quote_date_month'] = X['quote_date'].dt.month
  X = X.drop(columns='quote_date')
  
  #Merge tube data
  
  tube = pd.read_csv('competition_data/tube.csv')
  X = X.merge(tube, how='left')
  
  #Drop tube_assembly_id because our goal is to predict unknown assemblies.
  
  X = X.drop(columns='tube_assembly_id')
  
  return X

train_wrangled = wrangle(train)
val_wrangled = wrangle(val)
train_wrangled.shape, val_wrangled.shape
train_wrangled.head()
#Let's train our model.

X_train = train_wrangled.drop(columns=target)
X_val = val_wrangled.drop(columns=target)

pipeline = make_pipeline(ce.OrdinalEncoder(), RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1))

pipeline.fit(X_train, y_train_log)
y_pred_log = pipeline.predict(X_val)

rmse(y_val_log, y_pred_log)

#We observe that our error has dropped even further still.
#Before generating predictions to submit to Kaggle, we need to wrangle the test set.

test_wrangled = wrangle(test)
X_test = test_wrangled.drop(columns='id')
all(X_test.columns == X_train.columns)

#We verify that all of the columns in X_test are the same as the columns we trained on.
#Make predictions for the test set.

y_pred_log = pipeline.predict(X_test) #Remember that these predictions are in log dollars, and Kaggle expects that the predictions will be in the original units(dollars).
y_pred = np.expm1(y_pred_log) #This transforms log dollars to dollars.<jupyter_output><empty_output><jupyter_text>**Now use at least one more file and then to submit those results to Kaggle!**<jupyter_code>import category_encoders as ce
import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, mean_squared_log_error
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline

def rmse(y_true, y_pred):
    return np.sqrt(mean_squared_error(y_true, y_pred))

def rmsle(y_true, y_pred):
    return np.sqrt(mean_squared_log_error(y_true, y_pred))

def wrangle(X):
    X = X.copy()
    
    # Engineer date features
    X['quote_date'] = pd.to_datetime(X['quote_date'], infer_datetime_format=True)
    X['quote_date_year'] = X['quote_date'].dt.year
    X['quote_date_month'] = X['quote_date'].dt.month
    X = X.drop(columns='quote_date')
    
    # Merge tube data
    tube = pd.read_csv(SOURCE + 'tube.csv')
    X = X.merge(tube, how='left')
    
    # Engineer features from bill_of_materials
    materials = pd.read_csv(SOURCE + 'bill_of_materials.csv')
    
    materials['components_total'] = (materials['quantity_1'].fillna(0) + 
                                     materials['quantity_2'].fillna(0) + 
                                     materials['quantity_3'].fillna(0) + 
                                     materials['quantity_4'].fillna(0) + 
                                     materials['quantity_5'].fillna(0) + 
                                     materials['quantity_6'].fillna(0) + 
                                     materials['quantity_7'].fillna(0) + 
                                     materials['quantity_8'].fillna(0))

    materials['components_distinct'] = (materials['component_id_1'].notnull().astype(int) + 
                                        materials['component_id_2'].notnull().astype(int) + 
                                        materials['component_id_3'].notnull().astype(int) + 
                                        materials['component_id_4'].notnull().astype(int) + 
                                        materials['component_id_5'].notnull().astype(int) + 
                                        materials['component_id_6'].notnull().astype(int) + 
                                        materials['component_id_7'].notnull().astype(int) + 
                                        materials['component_id_8'].notnull().astype(int))
    
    # Merge selected features from bill_of_materials
    # Just use the first component_id, ignore the others for now!
    features = ['tube_assembly_id', 'component_id_1', 'components_total', 'components_distinct']
    X = X.merge(materials[features], how='left')
    
    # Get component_type_id (has lower cardinality than component_id)
    components = pd.read_csv(SOURCE + 'components.csv')
    components = components.rename(columns={'component_id': 'component_id_1'})
    features = ['component_id_1', 'component_type_id']
    X = X.merge(components[features], how='left')
    
    # Count the number of specs for the tube assembly
    specs = pd.read_csv(SOURCE + 'specs.csv')
    specs['specs_total'] = specs.drop(columns=['tube_assembly_id']).count(axis=1)
    features = ['tube_assembly_id', 'specs_total', 'spec1']
    X = X.merge(specs[features], how='left')
    
    # Drop tube_assembly_id because our goal is to predict unknown assemblies
    X = X.drop(columns='tube_assembly_id')
    
    return X


# Read data
trainval = pd.read_csv(SOURCE + 'train_set.csv')
test = pd.read_csv(SOURCE + 'test_set.csv')

# Split into train & validation sets
# All rows for a given tube_assembly_id should go in either train or validation
trainval_tube_assemblies = trainval['tube_assembly_id'].unique()
train_tube_assemblies, val_tube_assemblies = train_test_split(
    trainval_tube_assemblies, random_state=42)
train = trainval[trainval.tube_assembly_id.isin(train_tube_assemblies)]
val = trainval[trainval.tube_assembly_id.isin(val_tube_assemblies)]

# Wrangle train, validation, and test sets
train = wrangle(train)
val = wrangle(val)
test = wrangle(test)

# Arrange X matrix and y vector (log-transformed)
target = 'cost'
X_train = train.drop(columns=target)
X_val = val.drop(columns=target)
X_test = test.drop(columns='id')
y_train = train[target]
y_val = val[target]
y_train_log = np.log1p(y_train)
y_val_log = np.log1p(y_val)
# Make pipeline
pipeline = make_pipeline(
    ce.OrdinalEncoder(), 
    RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)
)

# Fit
pipeline.fit(X_train, y_train_log)

# Validate
y_pred_log = pipeline.predict(X_val)
print('Validation Error', rmse(y_val_log, y_pred_log))

# Predict
def generate_submission(estimator, X_test, filename):
    y_pred_log = estimator.predict(X_test)
    y_pred = np.expm1(y_pred_log)  # Convert from log-dollars to dollars
    submission = pd.read_csv(SOURCE + '../sample_submission.csv')
    submission['cost'] = y_pred
    submission.to_csv(filename, index=False)
    
generate_submission(pipeline, X_test, 'submission-02.csv')
from google.colab import files
files.download('submission-02.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10145497443963822, 'fraction_numerical': 0.011179147238919163, 'mean_word_length': 4.768632534024627, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '399933', 'license': 'MIT', 'n_tokens_mistral': 6066, 'n_tokens_neox': 5743, 'n_words': 1557, 'path': 'Unit_2_Sprint_3_Module_1_Assignment_1(Caterpillar Kaggle Submission 1).ipynb', 'repo_name': 'tcbic/DS-Unit-2-Applied-Modeling', 'size': '17801'}