<jupyter_start><jupyter_text>**Download data from kaggle**<jupyter_code>!pip install -U -q kaggle
!mkdir -p ~/.kaggle
!echo '{"username":"name","key":"key"}' > ~/.kaggle/kaggle.json
!chmod 600 ~/.kaggle/kaggle.json
!kaggle competitions download -c caterpillar-tube-pricing --force
!unzip data.zip<jupyter_output>Archive:  data.zip
   creating: competition_data/
  inflating: competition_data/bill_of_materials.csv  
  inflating: competition_data/comp_adaptor.csv  
  inflating: competition_data/comp_boss.csv  
  inflating: competition_data/comp_elbow.csv  
  inflating: competition_data/comp_float.csv  
  inflating: competition_data/comp_hfl.csv  
  inflating: competition_data/comp_nut.csv  
  inflating: competition_data/comp_other.csv  
  inflating: competition_data/comp_sleeve.csv  
  inflating: competition_data/comp_straight.csv  
  inflating: competition_data/comp_tee.csv  
  inflating: competition_data/comp_threaded.csv  
  inflating: competition_data/components.csv  
  inflating: competition_data/specs.csv  
  inflating: competition_data/test_set.csv  
  inflating: competition_data/train_set.csv  
  inflating: competition_data/tube.csv  
  inflating: competition_data/tube_end_form.csv  
  inflating: competition_data/type_component.csv  
  inflating: competition_data/type_connection.csv  
  i[...]<jupyter_text>**Unzip and examine the CSVs present**<jupyter_code>from glob import glob
import pandas as pd

for path in glob('competition_data/*.csv'):
    df = pd.read_csv(path)
    print(path, df.shape)<jupyter_output>competition_data/comp_threaded.csv (194, 32)
competition_data/comp_adaptor.csv (25, 20)
competition_data/tube_end_form.csv (27, 2)
competition_data/comp_straight.csv (361, 12)
competition_data/comp_tee.csv (4, 14)
competition_data/comp_boss.csv (147, 15)
competition_data/components.csv (2048, 3)
competition_data/comp_float.csv (16, 7)
competition_data/bill_of_materials.csv (21198, 17)
competition_data/comp_elbow.csv (178, 16)
competition_data/type_connection.csv (14, 2)
competition_data/train_set.csv (30213, 8)
competition_data/comp_sleeve.csv (50, 10)
competition_data/test_set.csv (30235, 8)
competition_data/tube.csv (21198, 16)
competition_data/comp_hfl.csv (6, 9)
competition_data/type_end_form.csv (8, 2)
competition_data/comp_other.csv (1001, 3)
competition_data/type_component.csv (29, 2)
competition_data/specs.csv (21198, 11)
competition_data/comp_nut.csv (65, 11)<jupyter_text>**Choose tube and bill of materials csv along with training data**<jupyter_code>trainval = pd.read_csv('competition_data/train_set.csv')
test = pd.read_csv('competition_data/test_set.csv')
trainval.head(5)
test.head()
tube = pd.read_csv('competition_data/tube.csv')
bill_mat = pd.read_csv('competition_data/bill_of_materials.csv')
tube.head()
bill_mat.head()
bill_mat.isnull().sum()
        
tube.isnull().sum()<jupyter_output><empty_output><jupyter_text>**Change quote date to dt object**<jupyter_code>trainval['quote_date'] = pd.to_datetime(trainval['quote_date'], infer_datetime_format=True)
test['quote_date'] = pd.to_datetime(test['quote_date'], infer_datetime_format=True)<jupyter_output><empty_output><jupyter_text>**Merge tube and bill of materials with common feature tube_assembly_id. Then do the same with training data to obtain final train_validation dataset**<jupyter_code>tube = tube.merge(bill_mat, on='tube_assembly_id')
trainval = trainval.merge(tube, on='tube_assembly_id')
test = test.merge(tube, on='tube_assembly_id')

trainval.shape, test.shape, tube.shape<jupyter_output><empty_output><jupyter_text>**Check to see if train and test sets have same tube assemby IDs**<jupyter_code>trainval_tube_assemblies = trainval['tube_assembly_id'].unique()
test_tube_assemblies = test['tube_assembly_id'].unique()
len(trainval_tube_assemblies), len(test_tube_assemblies)<jupyter_output><empty_output><jupyter_text>**Train test split based on tube assemblies**<jupyter_code>from sklearn.model_selection import train_test_split
train_tube_assemblies, val_tube_assemblies = train_test_split(
    trainval_tube_assemblies, random_state=40)
train = trainval[trainval.tube_assembly_id.isin(train_tube_assemblies)]
val = trainval[trainval.tube_assembly_id.isin(val_tube_assemblies)]
train.shape, val.shape, trainval.shape<jupyter_output><empty_output><jupyter_text>**Baseline calculations**<jupyter_code>import numpy as np
from sklearn.metrics import mean_squared_log_error

def rmsle(y_true, y_pred):
    return np.sqrt(mean_squared_log_error(y_true, y_pred))
target = 'cost'
y_train = train[target]
y_val = val[target]
y_pred = np.full_like(y_val, fill_value=y_train.mean())
print('Baseline RMSLE:', rmsle(y_val, y_pred))<jupyter_output>Baseline RMSLE: 0.9486220140675697<jupyter_text>**feature engineering and fit model**<jupyter_code>numeric_features = train.select_dtypes(include='number').columns.tolist()
cardinality = train.select_dtypes(exclude='number').nunique()
categorical_features = cardinality[cardinality <= 52].index.tolist()
features = numeric_features + categorical_features

X_train = train[features]
X_val = val[features]
def wrangle(X):
    X = X.copy()
    
    # Engineer date features
    X['quote_date'] = pd.to_datetime(X['quote_date'], infer_datetime_format=True)
    X['quote_date_year'] = X['quote_date'].dt.year
    X['quote_date_month'] = X['quote_date'].dt.month
    X = X.drop(columns='quote_date')
    
    # Drop tube_assembly_id because our goal is to predict unknown assemblies
    X = X.drop(columns='tube_assembly_id')
    
    X = X.fillna(0)
    return X

train_wrangled = wrangle(train)
val_wrangled = wrangle(val)
features = train_wrangled.columns.drop(target)
X_train = train_wrangled[features]
X_val = val_wrangled[features]
from sklearn.metrics import mean_squared_error

def rmse(y_true, y_pred):
    return np.sqrt(mean_squared_error(y_true, y_pred))
y_train_log = np.log1p(y_train)
y_val_log = np.log1p(y_val)
import category_encoders as ce
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestRegressor

pipeline = make_pipeline(
    ce.OrdinalEncoder(), 
    RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)
)

pipeline.fit(X_train, y_train_log)
y_pred_log = pipeline.predict(X_val)
print('RMSLE after log transform and feature engineering:',rmse(y_val_log, y_pred_log) )<jupyter_output>RMSLE after log transform and feature engineering: 0.30269814408958967<jupyter_text>**Apply model on test data**<jupyter_code>test_wrangled = wrangle(test)
X_test = test_wrangled.drop(columns='id')
all(X_test.columns == X_train.columns)<jupyter_output><empty_output><jupyter_text>**reverse log transform**<jupyter_code>y_pred_log = pipeline.predict(X_test)
y_pred = np.expm1(y_pred_log)<jupyter_output><empty_output><jupyter_text>**Submission**<jupyter_code>sample_submission = pd.read_csv('sample_submission.csv')
submission = sample_submission.copy()
submission['cost'] = y_pred
submission.to_csv('submission-01.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11378273937312151, 'fraction_numerical': 0.021325318448547303, 'mean_word_length': 7.211515863689777, 'pattern_counts': {'":': 2, '<': 46, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '153539', 'license': 'MIT', 'n_tokens_mistral': 2677, 'n_tokens_neox': 2497, 'n_words': 415, 'path': 'module1-log-linear-regression/Kaggle_sub.ipynb', 'repo_name': 'kmk028/DS-Unit-2-Applied-Modeling', 'size': '6987'}