<jupyter_start><jupyter_text>Add 1 more file<jupyter_code>def wrangle(X):
    X = X.copy()
    
    X['quote_date'] = pd.to_datetime(X['quote_date'], infer_datetime_format=True)
    X['quote_date_year'] = X['quote_date'].dt.year
    X['quote_date_month'] = X['quote_date'].dt.month
    X = X.drop(columns='quote_date')
    
    tube = pd.read_csv('../data/competition_data/tube.csv')
    bill_of_materials = pd.read_csv('../data/competition_data/bill_of_materials.csv')
    
    X = X.merge(tube, how='left')
    X = X.merge(bill_of_materials, how='left')
    
    columns_all = X.columns
    for col in columns_all:
        X[col] = X[col].fillna(0)
    
    X = X.drop(columns='tube_assembly_id')
    
    return X
train_wrangled = wrangle(train)
val_wrangled = wrangle(val)
from sklearn.impute import SimpleImputer

X_train = train_wrangled.drop(columns=target)
X_val = val_wrangled.drop(columns=target)

pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(strategy='median'),
    RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)
)

pipeline.fit(X_train, y_train_log)
y_pred_log = pipeline.predict(X_val)
rmse(y_val_log, y_pred_log)
from sklearn.impute import SimpleImputer
from xgboost import XGBRegressor

X_train = train_wrangled.drop(columns=target)
X_val = val_wrangled.drop(columns=target)

pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(strategy='mean'),
    XGBRegressor(objective= "reg:linear", eta=0.017, min_child_weight=6, subsample=0.75, colsample_bytree=0.6, 
                 scale_pos_weight=0.8, silent=1, max_depth=9, max_delta_step=2)
)

pipeline.fit(X_train, y_train_log)
y_pred_log = pipeline.predict(X_val)
rmse(y_val_log, y_pred_log)
test = pd.read_csv('../data/competition_data/test_set.csv')
test_wrangled = wrangle(test)
X_test = test_wrangled.drop(columns='id')
all(X_test.columns == X_train.columns)
y_pred_log = pipeline.predict(X_test)
y_pred = np.expm1(y_pred_log)
sample_submission = pd.read_csv('../data/sample_submission.csv')
submission = sample_submission.copy()
submission['cost'] = y_pred
submission.to_csv('submission-03.csv', index=False)<jupyter_output><empty_output><jupyter_text>More Data Wrangling and Feature Engineering<jupyter_code>SOURCE = '../data/competition_data/'
def wrangle(X):
    X = X.copy()
    
    # Engineer date features
    X['quote_date'] = pd.to_datetime(X['quote_date'], infer_datetime_format=True)
    X['quote_date_year'] = X['quote_date'].dt.year
    X['quote_date_month'] = X['quote_date'].dt.month
    X = X.drop(columns='quote_date')
    
    # Merge tube data
    tube = pd.read_csv(SOURCE + 'tube.csv')
    X = X.merge(tube, how='left')
    
    # Engineer features from bill_of_materials
    materials = pd.read_csv(SOURCE + 'bill_of_materials.csv')
    
    materials['components_total'] = (materials['quantity_1'].fillna(0) + 
                                     materials['quantity_2'].fillna(0) + 
                                     materials['quantity_3'].fillna(0) + 
                                     materials['quantity_4'].fillna(0) + 
                                     materials['quantity_5'].fillna(0) + 
                                     materials['quantity_6'].fillna(0) + 
                                     materials['quantity_7'].fillna(0) + 
                                     materials['quantity_8'].fillna(0))

    materials['components_distinct'] = (materials['component_id_1'].notnull().astype(int) + 
                                        materials['component_id_2'].notnull().astype(int) + 
                                        materials['component_id_3'].notnull().astype(int) + 
                                        materials['component_id_4'].notnull().astype(int) + 
                                        materials['component_id_5'].notnull().astype(int) + 
                                        materials['component_id_6'].notnull().astype(int) + 
                                        materials['component_id_7'].notnull().astype(int) + 
                                        materials['component_id_8'].notnull().astype(int))
    
    # Merge selected features from bill_of_materials
    # Just use the first component_id, ignore the others for now!
    features = ['tube_assembly_id', 'component_id_1', 'components_total', 'components_distinct']
    X = X.merge(materials[features], how='left')
    
    # Get component_type_id (has lower cardinality than component_id)
    components = pd.read_csv(SOURCE + 'components.csv')
    components = components.rename(columns={'component_id': 'component_id_1'})
    features = ['component_id_1', 'component_type_id']
    X = X.merge(components[features], how='left')
    
    # Count the number of specs for the tube assembly
    specs = pd.read_csv(SOURCE + 'specs.csv')
    specs['specs_total'] = specs.drop(columns=['tube_assembly_id']).count(axis=1)
    features = ['tube_assembly_id', 'specs_total', 'spec1']
    X = X.merge(specs[features], how='left')
    
    # Drop tube_assembly_id because our goal is to predict unknown assemblies
    X = X.drop(columns='tube_assembly_id')
    
    return X
train_wrangled = wrangle(train)
val_wrangled = wrangle(val)
X_train = train_wrangled.drop(columns=target)
X_val = val_wrangled.drop(columns=target)

pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(strategy='mean'),
    XGBRegressor(objective= "reg:linear", eta=0.017, min_child_weight=6, subsample=0.75, colsample_bytree=0.6, 
                 scale_pos_weight=0.8, silent=1, max_depth=9, max_delta_step=2)
)

pipeline.fit(X_train, y_train_log)
y_pred_log = pipeline.predict(X_val)
rmse(y_val_log, y_pred_log)<jupyter_output>C:\Users\nchib\Anaconda3\lib\site-packages\xgboost\core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version
  if getattr(data, 'base', None) is not None and \
{'fraction_non_alphanumeric': 0.1289500509683996, 'fraction_numerical': 0.012572205232755691, 'mean_word_length': 3.1663128096249116, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '495587', 'license': 'MIT', 'n_tokens_mistral': 1994, 'n_tokens_neox': 1921, 'n_words': 296, 'path': 'module2-gradient-boosting/Unit 2 Sprint 8 Gradient Boosting Assignment.ipynb', 'repo_name': 'nchibana/DS-Unit-2-Applied-Modeling', 'size': '5886'}