<jupyter_start><jupyter_text>Load and preprocessing data  Load data<jupyter_code>script_dir = osp.abspath(os.path.dirname(__file__))
DATA_DIR = script_dir + '../data/'
train = pd.read_csv(f'{DATA_DIR}/train.csv')
test = pd.read_csv(f'{DATA_DIR}/test.csv')<jupyter_output><empty_output><jupyter_text>Split train valid<jupyter_code>molecules = train.molecule_name.drop_duplicates().sort_values()
train_ind, valid_ind = train_test_split(np.arange(len(molecules)),
                                        test_size=5000,
                                        random_state=1234)<jupyter_output><empty_output><jupyter_text>Create train valid subset<jupyter_code>train_data = train.loc[train.molecule_name.isin(molecules.iloc[train_ind])]
val_data = train.loc[train.molecule_name.isin(molecules.iloc[valid_ind])]
val_data.head()<jupyter_output><empty_output><jupyter_text>Load pred<jupyter_code>glob(script_dir + '/guillaume/train*.csv')
def create_list_prediction(folder_path, text_find, text_replace):
    sub_val = []
    sub_test = []
    for sub_val_path in glob(folder_path):
        sub_val.append(sub_val_path)
        sub_test.append(sub_val_path.replace(text_find, text_replace))
    
    return sub_val, sub_test
def load_pred_guillaume(path):
    df = pd.read_csv(path)
    return df.loc[df.dataset=='valid', ['id', 'prediction']].set_index('id')
def load_pred_lam(path):
    return pd.read_csv(path, index_col=0)
def load_pred_thanhtu(path):
    return pd.read_csv(path, index_col=0)
preds_val = pd.concat([
    *[load_pred_guillaume(path) for path in create_list_prediction(script_dir + '/guillaume/train*.csv', 'train', 'submission')[0]],
    *[load_pred_lam(path) for path in create_list_prediction(script_dir + '/lam_01_v1/pred*.csv', 'pred', 'sub')[0]],
    *[load_pred_thanhtu(path) for path in create_list_prediction(script_dir + '/thanhtu/valid*.csv', 'valid', 'sub')[0]]
], axis=1)
preds_test = pd.concat([
    *[load_pred_lam(path) for path in create_list_prediction(script_dir + '/guillaume/train*.csv', 'train', 'submission')[1]],
    *[load_pred_lam(path) for path in create_list_prediction(script_dir + '/lam_01_v1/pred*.csv', 'pred_val', 'sub')[1]],
    *[load_pred_thanhtu(path) for path in create_list_prediction(script_dir + '/thanhtu/valid*.csv', 'valid', 'sub')[1]]
], axis=1)
preds_val.shape, preds_test.shape<jupyter_output><empty_output><jupyter_text>Stacking XGB<jupyter_code>preds = preds_val.loc[val_data['id']]
val_xgb = val_data.copy()
val_xgb['scalar_coupling_constant'] = 0
val_xgb = val_xgb.reset_index()
X = preds.values
y = val_data.scalar_coupling_constant.values
test_pred_xgb = test.copy()
test_pred_xgb['scalar_coupling_constant'] = 0
test_pred_xgb = test_pred_xgb.reset_index()
Xtest = preds_test.loc[test['id']].values
groups = val_xgb.molecule_name.astype('category').cat.codes.values
# types = val.type.astype('category').cat.codes.values

cv = GroupKFold(20)
for train_index, val_index in cv.split(X, y, groups):
    clf = LGBMRegressor(
        num_leaves=60,
        max_depth=-1,
        learning_rate=0.01,
        n_estimators=20000,
        subsample_for_bin=200000,
        objective='regression_l1',
        reg_alpha=0.0,
        random_state=None,
        n_jobs=12,
        silent=False,
        )
    X_train, X_val = X[train_index], X[val_index]
    y_train, y_val = y[train_index], y[val_index]
#     type_train, type_val = types[train_index], types[val_index]

    clf.fit(
            np.concatenate([
                X_train - X_train.mean(axis=1).reshape(-1, 1),
                X_train, 
#                 type_train.reshape(-1, 1)
            ], axis = 1
            ),
            y_train - X_train.mean(axis = 1),
            eval_set = [
                (
                    np.concatenate(
                    [
                        X_val - X_val.mean(axis = 1).reshape(-1, 1),
                        X_val,
#                         type_val.reshape(-1, 1)
                    ], axis = 1
                    ),
                    y_val - X_val.mean(axis = 1)
                )
            ],
            eval_metric = ['mae'],
            early_stopping_rounds = 10,
            verbose = 100
    )
    res_lgbm = clf.predict(np.concatenate([X_val - X_val.mean(axis = 1).reshape(-1, 1),
                                           X_val,
#                                            type_val.reshape(-1, 1)
                                          ], axis = 1)
                                           ) + X_val.mean(axis = 1)

    res_test = clf.predict(np.concatenate([
                                           Xtest - Xtest.mean(axis = 1).reshape(-1, 1),
                                           Xtest,
#                                            type_test.reshape(-1, 1)
                                        ], axis = 1)) + Xtest.mean(axis = 1)
    
    print(np.log(np.abs(res_lgbm - y_val).mean()))
    val_xgb.loc[val_index, 'scalar_coupling_constant'] += res_lgbm
    test_pred_xgb.loc[:, 'scalar_coupling_constant'] += res_test

test_pred_xgb['scalar_coupling_constant'] /= 20
val_xgb['mae'] = (val_xgb['scalar_coupling_constant'] - y).abs()
print(np.log(val_xgb.groupby('type')['mae'].mean()).mean())
np.log(val_xgb.groupby('type')['mae'].mean())<jupyter_output><empty_output><jupyter_text>Stacking Huberloss<jupyter_code>preds = preds_val.loc[val_data['id']]
val_huber = val_data.copy()
val_huber['scalar_coupling_constant'] = 0
val_huber = val_huber.reset_index()
X = preds.values
y = val_data.scalar_coupling_constant.values
test_pred_huber = test.copy()
test_pred_huber['scalar_coupling_constant'] = 0
test_pred_huber = test_pred_huber.reset_index()
Xtest = preds_test.loc[test['id']].values
groups = val_huber.molecule_name.astype('category').cat.codes.values
types = val_huber.type.astype('category').cat.codes.values

for bond_type in val_huber.type.unique():
    sub = val_huber.loc[val_huber.type == bond_type]
    sub_groups = groups[sub.index]
    sub_X_type = X[sub.index]
    sub_y_type = y[sub.index]
#     sub_types = types[sub.index]
    
    sub_test = test_pred_huber.loc[test_pred_huber.type == bond_type]
    sub_Xtest = Xtest[sub_test.index]
    
    cv = GroupKFold(20)
    for train_index, val_index in cv.split(sub_X_type, sub_y_type, sub_groups):        
        clf = HuberRegressor(epsilon = 1.01, max_iter = 50000, alpha = 1e-6, tol = 1e-5)        
        X_train, X_val = sub_X_type[train_index], sub_X_type[val_index]
        y_train, y_val = sub_y_type[train_index], sub_y_type[val_index]        

#         type_train, type_test = sub_types[train_index], sub_types[test_index]

        clf.fit(
                np.concatenate([
                    X_train - X_train.mean(axis=1).reshape(-1, 1),
                    X_train, 
#                     type_train.reshape(-1, 1)
                ], axis = 1
                ),
                y_train - X_train.mean(axis = 1)
        )
        res_lgbm = clf.predict(np.concatenate([X_val - X_val.mean(axis = 1).reshape(-1, 1),
                                               X_val,
#                                                type_test.reshape(-1, 1)
                                              ], axis = 1)
                                               ) + X_val.mean(axis = 1)
        
        res_test = clf.predict(np.concatenate([
                                           sub_Xtest - sub_Xtest.mean(axis = 1).reshape(-1, 1),
                                           sub_Xtest,
        #                                            type_test.reshape(-1, 1)
                                        ], axis = 1)) + sub_Xtest.mean(axis = 1)
        
        print(bond_type, np.log(np.abs(res_lgbm - y_val).mean()))
        val_huber.loc[sub.index[val_index], 'scalar_coupling_constant'] += res_lgbm
        test_pred_huber.loc[sub_test.index, 'scalar_coupling_constant'] += res_test
        
test_pred_huber['scalar_coupling_constant'] /= 20
val_huber['mae'] = (val_huber['scalar_coupling_constant'] - y).abs()
print(np.log(val_huber.groupby('type')['mae'].mean()).mean())<jupyter_output><empty_output><jupyter_text>Stacking final<jupyter_code>val_huber.head()
val_xgb.head()
val_final = val_huber.copy()
val_final['scalar_coupling_constant'] = val_xgb['scalar_coupling_constant'] * 0.5 + val_huber['scalar_coupling_constant'] *0.5
val_final['mae'] = (val_final['scalar_coupling_constant'] - y).abs()
print(np.log(val_final.groupby('type')['mae'].mean()).mean())<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>sub_final = test_pred_xgb.copy()
sub_final['scalar_coupling_constant'] = test_pred_xgb['scalar_coupling_constant'] * 0.5 + test_pred_huber['scalar_coupling_constant'] * 0.5
sub_final = sub_final.drop(['index', 'molecule_name', 'atom_index_0', 'atom_index_1', 'type'], axis = 1)
sub_final.to_csv('sub_stacking.csv', index=False)
pd.read_csv('sub_stacking.csv').shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13402871242709735, 'fraction_numerical': 0.015141318977119785, 'mean_word_length': 2.5825632784250705, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '434658', 'license': 'MIT', 'n_tokens_mistral': 3195, 'n_tokens_neox': 3052, 'n_words': 408, 'path': 'solutions/5/stacking_final.ipynb', 'repo_name': 'larsbratholm/champs_kaggle', 'size': '8916'}