<jupyter_start><jupyter_text>Get the Caterpillar data from Kaggle<jupyter_code># competition data from Kaggle

for path in glob('competition_data/*.csv'):
    df = pd.read_csv(path)
    print(path, df.shape)
# function for root mean squared log error

def rmsle(y_true, y_pred):
    return np.sqrt(mean_squared_log_error(y_true, y_pred))
# read csvs into dataframes

trainval = pd.read_csv('competition_data/train_set.csv')
test = pd.read_csv('competition_data/test_set.csv')
tube = pd.read_csv('competition_data/tube.csv')
bill_of_materials = pd.read_csv('competition_data/bill_of_materials.csv')
specs = pd.read_csv('competition_data/specs.csv')
components = pd.read_csv('competition_data/components.csv')
comp_sleeve = pd.read_csv('competition_data/comp_sleeve.csv')

trainval.shape, test.shape, tube.shape, bill_of_materials.shape, specs.shape, components.shape, comp_sleeve.shape
trainval.head()
tube.head(1)
# for each tube_assembly_id:
# list of components (by id) & quantity used
# name and type_id of components listed in components.csv

bill_of_materials.head(2)
bill_of_materials['component_id_1'].value_counts().head()
# list of 'specs' for each tube_assembly id
# specs appear to be 6 digits, two letters dash four numbers

specs.head(2)
specs.describe()
# name and type_id for each component id
# information for each component is organized into different csv files based on type
# for example, sleeve-type components are in comp_sleeve.csv

components.head()<jupyter_output><empty_output><jupyter_text>Train/val/test split<jupyter_code># create numpy array with unique tube_assembly_id

trainval_tube_assemblies = trainval['tube_assembly_id'].unique()
test_tube_assemblies = test['tube_assembly_id'].unique()
len(trainval_tube_assemblies), len(test_tube_assemblies)
# use train/test split to split the array

train_tube_assemblies, val_tube_assemblies = train_test_split(
    trainval_tube_assemblies, random_state=42)

len(train_tube_assemblies), len(val_tube_assemblies)
# getting dataframes such that their tube assembly ids are in the respective array created above

train = trainval[trainval.tube_assembly_id.isin(train_tube_assemblies)]
val = trainval[trainval.tube_assembly_id.isin(val_tube_assemblies)]
train.shape, val.shape, trainval.shape<jupyter_output><empty_output><jupyter_text>Wrangle function<jupyter_code>def wrangle(X):
    X = X.copy()
    
    # Engineer date features
    X['quote_date'] = pd.to_datetime(X['quote_date'], infer_datetime_format=True)
    X['quote_date_year'] = X['quote_date'].dt.year
    X['quote_date_month'] = X['quote_date'].dt.month
    X = X.drop(columns='quote_date')
        
    # Merge tube and bill_of_materials data
    X = X.merge(tube, how='left')
    X = X.merge(bill_of_materials, how='left')
    
    # Drop tube assembly_id because our goal is to predict unknown assemblies
    X = X.drop(columns='tube_assembly_id')
    
    return X

train_wrangled = wrangle(train)
val_wrangled = wrangle(val)<jupyter_output><empty_output><jupyter_text>Select features<jupyter_code>target = 'cost'
features = train_wrangled.columns.values
y_train = train_wrangled[target]
y_val = val_wrangled[target]
X_train = train_wrangled[features].drop(columns='cost')
X_val = val_wrangled[features].drop(columns='cost')
X_val.shape, y_val.shape<jupyter_output><empty_output><jupyter_text>Use NumPy to get the log of the target vector<jupyter_code>y_train_log = np.log1p(y_train)
X_train_filled = X_train.fillna(0)
X_val_filled = X_val.fillna(0)
X_val_filled.shape<jupyter_output><empty_output><jupyter_text>Fit a model<jupyter_code>pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)
)
pipeline.fit(X_train_filled, y_train_log)<jupyter_output><empty_output><jupyter_text>Get validation RMSLE<jupyter_code>y_val_log = np.log1p(y_val)
y_pred_log = pipeline.predict(X_val_filled)
def rmse(y_true, y_pred):
    return np.sqrt(mean_squared_error(y_true, y_pred))
print(f'Validation Root Mean Squared Log Error: {rmse(y_val_log, y_pred_log)}')<jupyter_output>Validation Root Mean Squared Log Error: 0.31170850932200994<jupyter_text>Create Kaggle submission<jupyter_code>test_wrangled = wrangle(test)
X_test = test_wrangled.drop(columns='id')
X_test = X_test.fillna(0)
all(X_test.columns==X_train.columns)
y_pred_log = pipeline.predict(X_test)
y_pred = np.expm1(y_pred_log)
submission = pd.read_csv('sample_submission.csv')
submission['cost'] = y_pred
submission.to_csv('submission-01.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1082984811798371, 'fraction_numerical': 0.008584635703279771, 'mean_word_length': 7.014109347442681, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '410110', 'license': 'MIT', 'n_tokens_mistral': 1698, 'n_tokens_neox': 1623, 'n_words': 302, 'path': 'Sinclair_log_linear_regression_feature_engineering_assignment.ipynb', 'repo_name': 'SMSinclair/DS-Unit-2-Regression-2', 'size': '4543'}