<jupyter_start><jupyter_text>Load data<jupyter_code>DATA_DIR = './data'
train, test = load_features(DATA_DIR, with_ext=True, with_geo=False)<jupyter_output><empty_output><jupyter_text>Build train and test data sets<jupyter_code>X_train = train.drop(['id', 'natoms', 'spacegroup',
                      'alpha', 'beta', 'gamma',
                      'ga', 'o_cnt', 'cellvol', 'o_fraction', 'avg_mass',
                      'bandgap', 'E0'], axis=1)
X_test = test.drop(['id', 'natoms', 'spacegroup',
                    'alpha', 'beta', 'gamma',
                    'ga', 'o_cnt', 'cellvol', 'o_fraction', 'avg_mass'], axis=1)
# Use log1p of energies to correct for skew
y_bg_train = train['bandgap']
y_e0_train = np.log1p(train['E0'])
# One-hot encode spacegroup_natoms
X_train = pd.concat([X_train.drop('spacegroup_natoms', axis=1),
                    pd.get_dummies(X_train['spacegroup_natoms'])], axis=1)
X_test = pd.concat([X_test.drop('spacegroup_natoms', axis=1),
                    pd.get_dummies(X_test['spacegroup_natoms'])], axis=1)<jupyter_output><empty_output><jupyter_text>Build Sklearn Model with XGBRegressor<jupyter_code>param = {'learning_rate': 0.10,      # Step size shrinkage used in update (Learning rate)
         'reg_alpha': 0.01,          # L1 regularization term on weights
         'n_estimators': 60,
         'max_depth': 5,
         'subsample': 1,
         'colsample_bytree': 0.90,
         'colsample_bylevel': 0.90,
         'silent': True,
         'random_state': 42,
         'objective': 'reg:linear'}
# Estimator pipeline
est = Pipeline([
    ('scaler', StandardScaler()),
    ('xgbreg', XGBRegressor(**param)),
])
est
# Perform grid search for bandgap model
t_md = [3, 4, 5, 6, 7]
t_lr = [0.01, 0.02, 0.05, 0.10, 0.20, 0.3]
t_ne = range(50, 500, 50)
gridsearch_bg_0 = GridSearchCV(est,
                              {'xgbreg__max_depth': t_md,
                               'xgbreg__learning_rate': t_lr,
                               'xgbreg__n_estimators': t_ne,
                              },
                              cv=5,
                              n_jobs=4,
                              scoring='neg_mean_squared_error',
                              return_train_score=False,
                              verbose=True)
gridsearch_bg_0.fit(X_train, y_bg_train)
gridsearch_bg_0.best_params_
-gridsearch_bg_0.best_score_
# Check best regularization
t_ra = [0.1, 0.05, 0.01, 0.005, 0.001]
gridsearch_bg_1 = GridSearchCV(est,
                              {'xgbreg__reg_alpha': t_ra,
                              },
                              cv=5,
                              n_jobs=4,
                              scoring='neg_mean_squared_error',
                              return_train_score=False,
                              verbose=True)
gridsearch_bg_1.fit(X_train, y_bg_train)
gridsearch_bg_1.best_params_
-gridsearch_bg_1.best_score_
# Perform grid search for formation energy model
t_md = [3, 4, 5, 6, 7]
t_lr = [0.01, 0.02, 0.05, 0.10, 0.20, 0.3]
t_ne = range(50, 500, 50)
gridsearch_e0_0 = GridSearchCV(est,
                              {'xgbreg__max_depth': t_md,
                               'xgbreg__learning_rate': t_lr,
                               'xgbreg__n_estimators': t_ne,
                              },
                              cv=5,
                              n_jobs=4,
                              scoring='neg_mean_squared_error',
                              return_train_score=False,
                              verbose=True)
gridsearch_e0_0.fit(X_train, y_e0_train)
gridsearch_e0_0.best_params_
-gridsearch_e0_0.best_score_
# Check best regularization
t_ra = [0.2, 0.1, 0.05, 0.01, 0.005, 0.001]
gridsearch_e0_1 = GridSearchCV(est,
                              {'xgbreg__reg_alpha': t_ra,
                              },
                              cv=5,
                              n_jobs=4,
                              scoring='neg_mean_squared_error',
                              return_train_score=False,
                              verbose=True)
gridsearch_e0_1.fit(X_train, y_e0_train)
gridsearch_e0_1.best_params_
-gridsearch_e0_1.best_score_<jupyter_output><empty_output><jupyter_text>Fit models using optimized hyperparameters<jupyter_code>pa_bg = {'learning_rate': 0.05,      # Step size shrinkage used in update (Learning rate)
         'reg_alpha': 0.01,          # L1 regularization term on weights
         'n_estimators': 250,
         'max_depth': 3,
         'subsample': 1,
         'colsample_bytree': 0.90,
         'colsample_bylevel': 0.90,
         'silent': True,
         'random_state': 42,
         'objective': 'reg:linear'}
pa_e0 = {'learning_rate': 0.05,      # Step size shrinkage used in update (Learning rate)
         'reg_alpha': 0.10,          # L1 regularization term on weights
         'n_estimators': 150,
         'max_depth': 4,
         'subsample': 1,
         'colsample_bytree': 0.90,
         'colsample_bylevel': 0.90,
         'silent': True,
         'random_state': 42,
         'objective': 'reg:linear'}
# Estimator pipeline
est_bg = Pipeline([
    ('scaler', StandardScaler()),
    ('xgbreg', XGBRegressor(**pa_bg)),
])
est_e0 = Pipeline([
    ('scaler', StandardScaler()),
    ('xgbreg', XGBRegressor(**pa_e0)),
])
est_bg.fit(X_train, y_bg_train)
est_e0.fit(X_train, y_e0_train)<jupyter_output><empty_output><jupyter_text>Output predictions<jupyter_code>predicted_bg = est_bg.predict(X_test)
predicted_e0 = np.expm1(est_e0.predict(X_test))
predicted = pd.DataFrame({'formation_energy_ev_natom': predicted_e0,
                          'bandgap_energy_ev': predicted_bg}) \
              .reset_index().rename(columns={'index': 'id'})
predicted['id'] += 1
predicted.head()
err_e0 = rmsle(np.expm1(est_e0.predict(X_train)), np.expm1(y_e0_train))
err_bg = rmsle(est_bg.predict(X_train), y_bg_train)
# Training RMSLE values
print "RMSLE BG: {}, RMSLE E0: {}, RMSLE AVG: {}".format(err_bg, err_e0,
                                                         0.5 * (err_bg + err_e0))<jupyter_output>RMSLE BG: 0.0728351173101, RMSLE E0: 0.027578048241, RMSLE AVG: 0.0502065827755<jupyter_text>Write to file:<jupyter_code>now = dt.datetime.now().strftime("%Y%m%d-%H%M%S")
predicted.to_csv(os.path.join('output', 'xgb-ext-skl-{}.csv'.format(now)),
                columns=['id', 'formation_energy_ev_natom', 'bandgap_energy_ev'],
                index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12592821782178218, 'fraction_numerical': 0.039913366336633664, 'mean_word_length': 2.108173076923077, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '417793', 'license': 'MIT', 'n_tokens_mistral': 2382, 'n_tokens_neox': 2180, 'n_words': 378, 'path': 'xgboost_model_sklearn-with-ext.ipynb', 'repo_name': 'jzmnd/tco-nomad2018', 'size': '6464'}