<jupyter_start><jupyter_text>Volve Well Log Imputation Firstly we try preparing the data and importing the liabraries important for the problem statement<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.preprocessing import LabelEncoder, StandardScaler

import missingno as msno

from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.neighbors import KNeighborsRegressor
from sklearn.linear_model import BayesianRidge
from sklearn import metrics
from lightgbm import LGBMRegressor<jupyter_output><empty_output><jupyter_text>The data for this project has been sourced from the Volve Field opendataset provided by Equinor in 2018. There are 19 wells with 187,396 samples below the Hordaland Group Formation.<jupyter_code># The zone map can convert the ZONE log of ints to named Zones.
zone_map = {
0: 'Seabed',
1: 'NORDLAND',
2: 'Utsira',
3: 'HORDALAND',
4: 'Ty',
5: 'SHETLAND',
6: 'Ekofisk',
7: 'Hod',
8: 'Draupne',
9: '<NAME>',
10: '<NAME>',
11: '<NAME>',
12: '<NAME>',
13: '<NAME>',
14: '<NAME>',
15: '<NAME>',
16: 'Sleipner',
17: 'Skagerrak',
18: '<NAME>'
}

# Lets rename logs in the HDF5 input to more common terms.
col_rename_map = {
    'ZONE_NO':'ZONE',
    'DTE':'DT',
    'DTSE':'DTS',
    'DRHOE':'DRHO',
    'GRE':'GR',
    'NPHIE':'NPHI',
    'PEFE':'PEF',
    'RHOBE':'RHOB',
    'RME':'RM',
    'RSE':'RS',
    'RDE':'RD',
    "WELL": "WELL_ID"
}
# Load data and perform encoding of labels.

data = pd.read_hdf('data/volve_ml_logs.hdf5').rename(col_rename_map, axis=1)
# look at deeper zones -> shallow zones poorly sampled/not of interest
data = data.query("ZONE>=4")
data['ZONE'] = data["ZONE"].astype(int)
data["ZONE_NAME"] = data["ZONE"].map(zone_map)


well_name_encoder=LabelEncoder()
data['WELL']=well_name_encoder.fit_transform(data['WELL_ID'])
display(data.head(5))
display("Samples, Features", data.shape)
# How many wells are there?
display("Wells N:", data.WELL.max() + 1)<jupyter_output><empty_output><jupyter_text>Sampling StatisticsThe number of missing values and missing values per well and per zone.Wells where key logs DTE, DTSE and RHOBE are missing? Are there any trends here, how might the distribution of missing data affect our imputation performance? **In your accuracy score Debajoy it would be useful to do some deeper analysis to see if any zones perform better than others, an important part of this process is to understand the limitations of the algorithm you are using and to know when and where it will and won't work -> and for a paper, most importantly, why.**<jupyter_code>missing_pc = pd.DataFrame({'ALL':data.count()/data.shape[0]})
for well, sub in data.groupby('WELL_ID'):
    missing_pc[well] = 1- sub.count()/sub.shape[0]
for zn, sub in data.groupby('ZONE'):
    missing_pc[zone_map[zn]] = 1- sub.count()/sub.shape[0]

import matplotlib as mpl

sns.set_style('whitegrid')
sns.set_context('paper')
plasma_r5 = mpl.cm.get_cmap('plasma_r', 11)
plasma_r5.colors[0] = (1.0, 1.0, 1.0, 1.0)

fig = plt.figure(figsize=(10, 6))
sns.heatmap(missing_pc.T.iloc[:20, :10], cbar_kws={'label':'Fraction Missing'}, cmap=plasma_r5, vmin=-0.1, vmax=1)
plt.tight_layout()
fig.savefig('figures/well_fraction_missing.png', dpi=150)

fig = plt.figure(figsize=(10, 4))
sns.heatmap(missing_pc.T.iloc[20:, :10], cbar_kws={'label':'Fraction Missing'}, cmap=plasma_r5, vmin=-0.1, vmax=1)
plt.tight_layout()
fig.savefig('figures/zone_fraction_missing.png', dpi=150)<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code># dropping rows that are all NA (i.e. 10 of the logs are all nan)
data_fe = data.dropna(thresh=10).copy()
print(data_fe.shape)

data_fe['RD10'] = np.log10(data_fe['RD'])
data_fe['RM10'] = np.log10(data_fe['RM'])

# remove label columns
drop = ["WELL_ID", "ZONE_NAME"]
data_fe_labels = data_fe[drop].copy()
data_fe = data_fe.drop(drop, axis=1)<jupyter_output><empty_output><jupyter_text>Blind Wells Train Test SplitHere we remove wells F-4, F-12 and F-1 from the training dataset.Try to select wells that preserve the split of missing values across the train/test sets.<jupyter_code>test_wells = well_name_encoder.transform(["F-4", "F-12", "F-1", "F-15D"])
train = data_fe[~data_fe.WELL.isin(test_wells)].copy()
test = data_fe[data_fe.WELL.isin(test_wells)].copy()
test_noedits = test.copy()
# introduce some missing elastic values to test
test["RHOB"].iloc[25000:31000] = np.nan
test["DT"].iloc[25000:31000] = np.nan
test["DTS"].iloc[25000:31000] = np.nan



print(train.size, test.size, test.shape[0]/train.shape[0])
print(pd.DataFrame({"train": 1 - train.count()/train.shape[0], "test": 1 - test.count()/test.shape[0]}).T)

# calculate a scalar on train and apply to both
sscaler = StandardScaler()
sscaler.fit(data_fe)

train.loc[:, :] = sscaler.transform(train)
test.loc[:, :] = sscaler.transform(test)
test_noedits.loc[:, :] = sscaler.transform(test_noedits)
test
s = msno.matrix(data.iloc[:, :10], figsize=(15, 7))
plt.savefig('figures/volve_missingno.png', dpi=150)
s = msno.matrix(train.iloc[:, :10], figsize=(15, 7))
plt.savefig('figures/volve_missingno_train.png', dpi=150)
s = msno.matrix(test.iloc[:, :10], figsize=(15, 7))
plt.savefig('figures/volve_missingno_Test.png', dpi=150)<jupyter_output><empty_output><jupyter_text>Training Models<jupyter_code>def data_prep(data, j, set_to_nan=0.3):
    """This method sets set_to_nan fraction of the values to nan so we can measure the model accuracy.
    """
    data = data.copy()
    sub = data.dropna(subset=[j])
    rand_set_mask = np.random.random(len(sub)) < set_to_nan
    replace = sub.index[rand_set_mask]
    data.loc[replace, j] = np.nan
    data['set_nan'] = False
    data.loc[replace, 'set_nan'] = True
    data['was_nan'] = data[j].isna()
    print('Col, InputSize, Number of Nan, % NaN, Original Nan', 'Training Size')
    print(
        f'{j:>3}',
        f'{data.shape[0]:>10}',
        f'{replace.size:>14}',
        f'{100*np.sum(data.set_nan)/sub.shape[0]:>6.2f}',
        f'{np.sum(data.was_nan):>13}',
        f'{sub.shape[0]-replace.size:>13}'
    )

    return data, replace
missing = pd.DataFrame({
    "all_data":data_fe.isna().sum(axis=0)/data_fe.shape[0]*100,
    "train":train.isna().sum(axis=0)/data_fe.shape[0]*100,
    "test":test.isna().sum(axis=0)/data_fe.shape[0]*100
})
missing.T
imputation_train_dfs = dict()
imputation_train_keys = ['DT', 'DTS', 'RHOB']

for key in imputation_train_keys:
    imputation_train_dfs[key] = data_prep(train.copy(), key)<jupyter_output><empty_output><jupyter_text>**because random is used to create the training gaps -> we might need to rethink this so the numbre of nan for training in each case stays relatively constant** Also, this approach troubles me a little because when logs are missing the missing sections in logs are usually associated with each other, *i.e.* missing dtse -> missing dte as well  Imputaiton ModelsVarious imputation models are tried: - LGBM with MICE (Random imputation order). - LGBM with MICE (Ascending number of missing imputation order).<jupyter_code># training models

imputation_args = dict(
    # Random Order MICE - LGBM
    lgbrand = dict(
        training_set = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RHOB', 'RM10', 'RD10', 'ZONE'],
        estimator = LGBMRegressor(n_jobs=4),
        kwargs = dict(random_state=456, max_iter=20, tol=0.01, imputation_order='random',)
    ),
    # Ascending Order MICE - LGBM
    lgbasc = dict(
        training_set = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RHOB', 'RM10', 'RD10', 'ZONE'],
        estimator = LGBMRegressor(random_state=456, n_jobs=4),
        kwargs = dict(random_state=456, max_iter=20, tol=0.01, imputation_order='ascending',)
    ),
    # Bayesian Ridge Regression
    brr = dict(
        estimator = BayesianRidge(),
        training_set = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RHOB', 'RM10', 'RD10'],
        kwargs=dict(random_state=456), 
    ),
    brr1 = dict(
        estimator = BayesianRidge(),
        training_set = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RHOB', 'RM10', 'RD10'],
        kwargs=dict(random_state=456, max_iter=1, imputation_order="ascending"), 
    ),
    knn = dict(
        training_set = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RHOB', 'RM10', 'RD10'],
        estimator = KNeighborsRegressor(n_jobs=4),
        kwargs = dict(random_state=456, max_iter=20, tol=0.01)
    ),
    knn1 = dict(
        training_set = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RHOB', 'RM10', 'RD10'],
        estimator = KNeighborsRegressor(n_jobs=4),
        kwargs = dict(random_state=456, max_iter=1, imputation_order="ascending", tol=0.01)
    ),
)


imputation_args2 = dict(
    # Random Order MICE - LGBM
    lgbrand = dict(
        training_set = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RHOB', 'RM10', 'RD10', 'ZONE'],
        estimator = LGBMRegressor(n_jobs=4, num_leaves=100, max_depth=5),
        kwargs = dict(random_state=456, max_iter=20, tol=0.01, imputation_order='random')
    ),
    # Ascending Order MICE - LGBM
    lgbasc = dict(
        training_set = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RHOB', 'RM10', 'RD10', 'ZONE'],
        estimator = LGBMRegressor(random_state=456, n_jobs=4, num_leaves=100, max_depth=5),
        kwargs = dict(random_state=456, max_iter=20, tol=0.01, imputation_order='ascending',)
    ),
)

def train_models(data, training_set=None, estimator=None, **kwargs):
    """Train a model using a MICE iterative imputer.
    """
    # print(kwargs)
    mice = IterativeImputer(estimator, **kwargs['kwargs'])
    mice.fit(data[training_set])
    data.loc[:, training_set] = mice.transform(data[training_set])
    return data, mice
models = dict()
imputed = dict()
for imp_mod, args in imputation_args.items():
# for imp_mod, args in [(key, imputation_args[key]) for key in ["brr1", "knn1"]]:

    for key, val in imputation_train_dfs.items():
        imputed[imp_mod+'_'+key], models[imp_mod+'_'+key] = train_models(val[0].copy(), **args)
# add here some models where we have broader trees, this seemed to work better with direct imputers.
models2 = dict()
imputed2 = dict()
for imp_mod, args in imputation_args2.items():
    for key, val in imputation_train_dfs.items():
        imputed2[imp_mod+'_'+key], models2[imp_mod+'_'+key] = train_models(val[0].copy(), **args)
from collections import defaultdict

def evaluate(data1, data2, j):
    """Evaluate the models against the NANed data from the training set.
    """
    mask = data1.set_nan.values
    truth = data2.loc[mask, j].values
    test = data1.loc[mask, j].values
    se = np.power((truth - test)/truth, 2)
    score = np.nanmean(np.power(se, 0.5))*100.0
    er = dict(
        perc_error=score,
        explained_var=metrics.explained_variance_score(truth, test),
        max_error=metrics.max_error(truth, test),
        mae=metrics.mean_absolute_error(truth, test),
        mse=metrics.mean_squared_error(truth, test),
        r2=metrics.r2_score(truth, test),
    )
    return er

scores = defaultdict(dict)
for key, d in imputed.items():
    mod, key = key.split('_')
    scores[f'{key}_{mod}'] = evaluate(d, train, key)

mices_score = pd.DataFrame(scores)
mices_score.T
scores2 = defaultdict(dict)
for key, d in imputed2.items():
    mod, key = key.split('_')
    scores2[f'{key}_{mod}'] = evaluate(d, train, key)

mices_score2 = pd.DataFrame(scores2)
mices_score2
fig, axs = plt.subplots(nrows=4, figsize=(15, 7), sharex=True)
mod = "lgbasc"

axs[0].plot(train.DT[imputed[f'{mod}_DT'].set_nan].values)
axs[0].plot(imputed[f"{mod}_DT"].DT[imputed[f'{mod}_DT'].set_nan].values)
axs[0].set_ylabel("DT")
axs[1].plot(train.DTS[imputed[f'{mod}_DTS'].set_nan].values)
axs[1].plot(imputed[f"{mod}_DTS"].DTS[imputed[f'{mod}_DTS'].set_nan].values)
axs[1].set_ylabel("DTS")
axs[2].plot(train.RHOB[imputed[f'{mod}_RHOB'].set_nan].values)
axs[2].plot(imputed[f"{mod}_RHOB"].RHOB[imputed[f'{mod}_RHOB'].set_nan].values)
axs[2].set_ylabel("RHOB")
axs[3].plot(train.isna().sum(axis=1)[imputed[f'{mod}_DT'].set_nan].values, label="DT")
axs[3].plot(train.isna().sum(axis=1)[imputed[f'{mod}_DTS'].set_nan].values, label="DTS")
axs[3].plot(train.isna().sum(axis=1)[imputed[f'{mod}_RHOB'].set_nan].values, label="RHOB")
axs[3].legend()
axs[3].set_ylabel("Missing Features")
axs[3].set_xlabel("Sample")
axs[0].set_title("Imputation Validation Samples", fontsize=12)
fig, axs = plt.subplots(nrows=4, ncols=9, figsize=(15, 20), sharey="row", sharex="col")
mod='lgbasc'
logs = [ 'ZONE', 'GR', 'NPHI', 'PEF', 'RM10', 'RD10', 'DT', 'DTS', "RHOB"]
for row, (well, val) in zip(range(4), train.groupby("WELL")):
    axs[row, -3].plot(imputed[f"{mod}_DT"].loc[val.index, "DT"], -val.TVDSS, 'g')
    axs[row, -2].plot(imputed[f"{mod}_DTS"].loc[val.index, "DTS"], -val.TVDSS, 'g')
    axs[row, -1].plot(imputed[f"{mod}_RHOB"].loc[val.index, "RHOB"], -val.TVDSS, 'g')
    for col, log in enumerate(logs):
        axs[row, col].plot(val[log], -val.TVDSS, 'k', alpha=0.5)
for col, log in enumerate(logs):
    axs[0, col].set_title(log)
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>One Step LGBMHere we test the MICE imputation against a single imputation pass using LGBM. The inputs and the predictors are the same but multiple imputation chaining is not applied.<jupyter_code>dt_lgbm = LGBMRegressor(random_state=456, max_depth=5, num_leaves=100)
dts_lgbm = LGBMRegressor(random_state=456, max_depth=5, num_leaves=100)
rho_lgbm = LGBMRegressor(random_state=456, max_depth=5, num_leaves=100)
dt_col = ['DTS', 'GR', 'NPHI', 'PEF', 'RD10', 'RHOB', 'RM10', 'ZONE',]
dts_col = ['DT', 'GR', 'NPHI', 'PEF', 'RD10', 'RHOB', 'RM10', 'ZONE',]
rhob_col = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RD10', 'RM10', 'ZONE',]

dt_lgbm.fit(imputation_train_dfs['DT'][0].dropna(subset=['DT']).loc[:, dt_col], imputation_train_dfs['DT'][0].dropna(subset=['DT'])['DT'])
dts_lgbm.fit(imputation_train_dfs['DTS'][0].dropna(subset=['DTS']).loc[:, dts_col], imputation_train_dfs['DTS'][0].dropna(subset=['DTS'])['DTS'])
rho_lgbm.fit(imputation_train_dfs['RHOB'][0].dropna(subset=['RHOB']).loc[:, rhob_col], imputation_train_dfs['RHOB'][0].dropna(subset=['RHOB'])['RHOB'])

dt_one_imputed = imputation_train_dfs['DT'][0].copy()
dts_one_imputed = imputation_train_dfs['DTS'][0].copy()
rhob_one_imputed = imputation_train_dfs['RHOB'][0].copy()


dt_one_imputed.loc[dt_one_imputed["DT"].isna(), "DT"] = dt_lgbm.predict(imputation_train_dfs['DT'][0][dt_col])[dt_one_imputed["DT"].isna().values]
dts_one_imputed.loc[dts_one_imputed["DTS"].isna(), "DTS"] = dts_lgbm.predict(imputation_train_dfs['DTS'][0][dts_col])[dts_one_imputed["DTS"].isna().values]
rhob_one_imputed.loc[rhob_one_imputed["RHOB"].isna(), "RHOB"] = rho_lgbm.predict(imputation_train_dfs['RHOB'][0][rhob_col])[rhob_one_imputed["RHOB"].isna().values]
mices_score["DT_onepass"] = evaluate(dt_one_imputed, train, "DT").values()
mices_score["DTS_onepass"] = evaluate(dts_one_imputed, train, "DTS").values()
mices_score["RHOB_onepass"] = evaluate(rhob_one_imputed, train, "RHOB").values()

mices_score.sort_index(axis=1).T.round(2)<jupyter_output><empty_output><jupyter_text>Observations from the training results: - KNN and LGB tend to be pretty equivalent in terms of results. - The R2 fit for BRR in particular tends to be much lower than the other models. - One pass models are similar or better than MICE models suggesting in this case there is no benefit to running MICE style imputation. - All data types have R2 scores greater than 90% when using KNN or GB-Tree models.  - Random or Ascending order had very little impact upon the prediction capabilities of GB-Trees. Calculate scores when certain logs are absent.<jupyter_code>imputed["lgbonce_DT"] = dt_one_imputed
imputed["lgbonce_DTS"] = dts_one_imputed
imputed["lgbonce_RHOB"] = rhob_one_imputed
# DTS when DT and RHOB are absent.
imputed["lgbonce_DTS"][np.logical_and(imputed["lgbonce_DTS"]["DT"].isna(), imputed["lgbonce_DTS"]["RHOB"].isna())]
# find metrics for predicting Y when X is missing
missing_logs = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RHOB', 'RM10', 'RD10']

missing_scores = []
for missing_log in missing_logs:
    scores_when_missing = defaultdict(dict)
    for key, d in imputed.items():
        mod, key = key.split('_')
        if missing_log == key:
            scores_when_missing[f'{key}_{mod}'] = evaluate(d, train, key)
            scores_when_missing[f'{key}_{mod}']
        else:

            mask = train[missing_log].isna()
            d = d.loc[~mask, :]
            scores_when_missing[f'{key}_{mod}'] = evaluate(d, train.loc[~mask, :], key)
            scores_when_missing[f'{key}_{mod}']
    temp_df = pd.DataFrame(scores_when_missing)
    temp_df["always_present_log"] = missing_log
    missing_scores.append(temp_df)
missing_scores = pd.concat(missing_scores)<jupyter_output><empty_output><jupyter_text>Validation Plots for all Models<jupyter_code>fig = plt.figure(figsize=(20, 10))
melted = missing_scores.reset_index().melt(id_vars=["index", "always_present_log"], var_name="model").sort_values("model")
isnonemodel = melted.model.apply(lambda x: x.split("_")[0]) == melted.always_present_log
melted.loc[isnonemodel, "always_present_log"] = "none"
g = sns.FacetGrid(melted, height=5, aspect=2, col="index", col_wrap=2, sharey=False)
g.map_dataframe(sns.barplot, x="model", y="value", hue="always_present_log", palette="pastel",  hue_order=["none", 'DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RHOB', 'RM10', 'RD10'])
g.add_legend(title="Log has no\nNan Values")
for ax in g.axes.flat:
    for label in ax.get_xticklabels():
        label.set_rotation(45)
from matplotlib import cm
from matplotlib import colors
from matplotlib import gridspec

for mod, name in zip(['lgbrand', 'knn', 'knn1', 'brr', 'brr1', 'lgbonce'], ['GBT Random', 'KNN Regresssion', 'KNN Regresssion Once', 'Bayesian Ridge Regression', 'Bayesian Ridge Regression Once', "GBT Direct"]):
    fig, axs = plt.subplots(nrows=4, ncols=2, gridspec_kw={"width_ratios":[9, 1]}, figsize=(15, 7), sharex="col")
    
    gs = axs[2, 1].get_gridspec()
    # remove the underlying axes
    for ax in axs[:, 1]:
        ax.remove()
    cbar_ax = fig.add_subplot(gs[2:, -1])
    
    
#     cb = axs[:, 1]
    axs = axs[:, 0]
    axs[0].plot(train.DT[imputed[f'{mod}_DT'].set_nan].values)
    axs[0].plot(imputed[f"{mod}_DT"].DT[imputed[f'{mod}_DT'].set_nan].values)
    axs[0].set_ylabel("DT")
    axs[1].plot(train.DTS[imputed[f'{mod}_DTS'].set_nan].values)
    axs[1].plot(imputed[f"{mod}_DTS"].DTS[imputed[f'{mod}_DTS'].set_nan].values)
    axs[1].set_ylabel("DTS")
    axs[2].plot(train.RHOB[imputed[f'{mod}_RHOB'].set_nan].values)
    axs[2].plot(imputed[f"{mod}_RHOB"].RHOB[imputed[f'{mod}_RHOB'].set_nan].values)
    axs[2].set_ylabel("RHOB")

    train_n = train/train
    for i, col in enumerate(train_n.columns):
        train_n[col] = (i + 1)
        train_n.loc[~train[col].isna().values, col] = np.nan
        
    missing = axs[3].imshow(train_n.T.values, aspect='auto', cmap=cm.get_cmap('tab20', train_n.shape[1]), interpolation='none', origin='lower')
    axs[3].set_ylabel("Missing Features")
    axs[3].set_xlabel("Sample")
    axs[0].set_title(f"Imputation Validation Samples for {name}", fontsize=12)
    axs[3].set_yticklabels([])
    axs[3].grid(False)
    plt.colorbar(missing, cax=cbar_ax,)
    cbar_ax.set_yticks([], minor=False, major=False)
    cbar_ax.set_yticklabels([])
    
    
    for lab, v in zip(train_n.columns.to_list(), np.linspace(1, 14.2, train_n.shape[1])):
        cbar_ax.text(-7, v, lab, fontsize=9)
    cbar_ax.set_title("Missing Features")
    fig.savefig('figures/qual_{mod}.png', dpi=150)<jupyter_output><empty_output><jupyter_text>Test EvaluationHere we use the trained models on the blind well test sets to see if the predictive accuracy is the same.<jupyter_code>imputation_test_dfs = dict()
imputation_test_keys = ['DT', 'DTS', 'RHOB']

for key in imputation_test_keys:
    imputation_test_dfs[key] = data_prep(test.copy(), key, 1.0)
# impute test sets
imputed_test = dict()
for imp_mod, args in imputation_args.items():
    for key, val in imputation_test_dfs.items():
        temp_df = val[0].copy()
        temp_df.loc[:, args["training_set"]] = models[imp_mod+'_'+key].transform(val[0][args["training_set"]])
        imputed_test[imp_mod+'_'+key] = temp_df
imputed_test2 = dict()
for imp_mod, args in imputation_args2.items():
    for key, val in imputation_test_dfs.items():
        temp_df = val[0].copy()
        temp_df.loc[:, args["training_set"]] = models2[imp_mod+'_'+key].transform(val[0][args["training_set"]])
        imputed_test2[imp_mod+'_'+key] = temp_df
# impute tests sets using lgbonce models
dt_one_imputed_test = imputation_test_dfs['DT'][0].copy()
dts_one_imputed_test = imputation_test_dfs['DTS'][0].copy()
rhob_one_imputed_test = imputation_test_dfs['RHOB'][0].copy()
dt_one_imputed_test.loc[dt_one_imputed_test["DT"].isna(), "DT"] = dt_lgbm.predict(imputation_test_dfs['DT'][0][dt_col])[dt_one_imputed_test["DT"].isna().values]
dts_one_imputed_test.loc[dts_one_imputed_test["DTS"].isna(), "DTS"] = dts_lgbm.predict(imputation_test_dfs['DTS'][0][dts_col])[dts_one_imputed_test["DTS"].isna().values]
rhob_one_imputed_test.loc[rhob_one_imputed_test["RHOB"].isna(), "RHOB"] = rho_lgbm.predict(imputation_test_dfs['RHOB'][0][rhob_col])[rhob_one_imputed_test["RHOB"].isna().values]

imputed_test["lgbonce_DT"] = dt_one_imputed_test
imputed_test["lgbonce_DTS"] = dts_one_imputed_test
imputed_test["lgbonce_RHOB"] = rhob_one_imputed_test
scores_test = dict()
for key, d in imputed_test.items():
    mod, key = key.split('_')
    scores_test[f'{key}_{mod}'] = evaluate(d, test, key)
pd.DataFrame(scores_test).sort_index(axis=1).T.round(2)
def evaluate(data1, data2, j, no_masking=False):
    """Evaluate the models against the NANed data from the training set.
    """
    if not no_masking:
        mask = data1.set_nan.values
    else:
        mask = ~np.logical_or(data2[j].isna(), data1[j].isna()).values
    truth = data2.loc[mask, j].values
    test = data1.loc[mask, j].values

    se = np.power((truth - test)/truth, 2)
    score = np.nanmean(np.power(se, 0.5))*100.0
    er = dict(
        perc_error=score,
        explained_var=metrics.explained_variance_score(truth, test),
        max_error=metrics.max_error(truth, test),
        mae=metrics.mean_absolute_error(truth, test),
        mse=metrics.mean_squared_error(truth, test),
        r2=metrics.r2_score(truth, test),
    )
    return er

scores_test_sub = dict()
for key, d in imputed_test.items():
    mod, key = key.split('_')
#     print(key, d.iloc[25000:31000, :])
    scores_test_sub[f'{key}_{mod}'] = evaluate(d.iloc[25000:31000, :], test_noedits.iloc[25000:31000, :], key, no_masking=True)
pd.DataFrame(scores_test_sub).sort_index(axis=1).T.round(2)
d.iloc[25000:31000, :]
scores_test2 = dict()
for key, d in imputed_test2.items():
    mod, key = key.split('_')
    scores_test2[f'{key}_{mod}'] = evaluate(d, test, key)
pd.DataFrame(scores_test2).sort_index(axis=1).T.round(2)
from matplotlib import cm
from matplotlib import colors
from matplotlib import gridspec

c_map = colors.ListedColormap(['white', 'red', 'blue', 'green'])
bounds = [-15, 35, 45, 55, 65]
norm = colors.BoundaryNorm(bounds, c_map.N)

for mod, name in zip(['lgbrand', 'knn', 'knn1', 'brr', 'brr1', 'lgbonce'], ['GBT Random', 'KNN Regresssion', 'KNN Regresssion Once', 'Bayesian Ridge Regression', 'Bayesian Ridge Regression Once', "GBT Direct"]):
    fig, axs = plt.subplots(nrows=4, ncols=2, gridspec_kw={"width_ratios":[9, 1]}, figsize=(15, 7), sharex="col")
    
    gs = axs[2, 1].get_gridspec()
    # remove the underlying axes
    for ax in axs[:, 1]:
        ax.remove()
    cbar_ax = fig.add_subplot(gs[2:, -1])
    
    
#     cb = axs[:, 1]
    axs = axs[:, 0]
    axs[0].plot(imputed_test[f"{mod}_DT"].DT.values)
    axs[0].plot(test_noedits.DT.values)
    axs[0].set_ylabel("DT")
    axs[1].plot(imputed_test[f"{mod}_DTS"].DTS.values)
    axs[1].plot(test_noedits.DTS.values)
    axs[1].set_ylabel("DTS")
    axs[2].plot(imputed_test[f"{mod}_RHOB"].RHOB.values)
    axs[2].plot(test_noedits.RHOB.values)
    axs[2].set_ylabel("RHOB")

    test_n = test/test
    for i, col in enumerate(test_n.columns):
        test_n[col] = (i + 1)
        test_n.loc[~test[col].isna().values, col] = np.nan
        
    missing = axs[3].imshow(test_n.T.values, aspect='auto', cmap=cm.get_cmap('tab20', test_n.shape[1]), interpolation='none', origin='lower')
    axs[3].set_ylabel("Missing Features")
    axs[3].set_xlabel("Sample")
    axs[0].set_title(f"Imputation Validation Samples for {name}", fontsize=12)
    axs[3].set_yticklabels([])
    axs[3].grid(False)
    plt.colorbar(missing, cax=cbar_ax,)
    cbar_ax.set_yticks([], minor=False, major=False)
    cbar_ax.set_yticklabels([])
    
    
    for lab, v in zip(test_n.columns.to_list(), np.linspace(1, 14.2, test_n.shape[1])):
        cbar_ax.text(-7, v, lab, fontsize=9)
    cbar_ax.set_title("Missing Features")
    fig.savefig(f'figures/qual_test_{mod}.png', dpi=150)
fig, axs = plt.subplots(nrows=4, ncols=9, figsize=(15, 20), sharey="row", sharex="col")
mod='lgbasc'
logs = [ 'ZONE', 'GR', 'NPHI', 'PEF', 'RM10', 'RD10', 'DT', 'DTS', "RHOB"]
for row, (well, val) in zip(range(4), train.groupby("WELL")):
    axs[row, -3].plot(imputed[f"{mod}_DT"].loc[val.index, "DT"], -val.TVDSS, 'g')
    axs[row, -2].plot(imputed[f"{mod}_DTS"].loc[val.index, "DTS"], -val.TVDSS, 'g')
    axs[row, -1].plot(imputed[f"{mod}_RHOB"].loc[val.index, "RHOB"], -val.TVDSS, 'g')
    for col, log in enumerate(logs):
        axs[row, col].plot(val[log], -val.TVDSS, 'k', alpha=0.5)
for col, log in enumerate(logs):
    axs[0, col].set_title(log)
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>Actual Well Imputation<jupyter_code>df_pred = pd.DataFrame(index=df.index)
df_pred['dte']= dt_lgbm.predict(df[dt_col])
df_pred['dtse'] = dts_lgbm.predict(df[dts_col])
df_pred['rhob'] = rho_lgbm.predict(df[rhob_col])
for w in data.Well.unique():
    fig, axs = plt.subplots(ncols=9, figsize=(8, 8), sharey=True)

    well = data.query("Well == @w")
    sub = df.loc[well.index, :]
    sub_pred = df_pred.loc[well.index, :]

    frame = 0
    axs[frame].plot(sub_pred.dte, -well.TVDSS, color='green')
    axs[frame].plot(sub.DTE,  -well.TVDSS, color='#4d4f4e')
    axs[frame].set_title('DT')

    frame+=1
    axs[frame].plot(sub_pred.dtse, -well.TVDSS, color='green')
    axs[frame].plot(sub.DTSE,  -well.TVDSS, color='#4d4f4e')

    axs[frame].set_title('DTS')

    frame+=1
    axs[frame].plot(sub_pred.rhob, -well.TVDSS, color='green')
    axs[frame].plot(sub.RHOBE,  -well.TVDSS, color='#4d4f4e')
    axs[frame].set_title('RHOB')

    for name, idx in (
        ('GR', 'GRE'), ('NPHI', 'NPHIE'), ('PEF', 'PEFE'), ('log(Rd)', 'RDElog'), ('log(Rm)', 'RMElog'), ('Zone', 'ZONE_NO')
    ):
        frame+=1
        axs[frame].plot(sub[idx], -well.TVDSS, color='#4d4f4e')
        axs[frame].set_title(name)
        
    w = encoder.inverse_transform([w])[0]
    fig = plt.gcf()
    rect = fig.patch
    rect.set_facecolor('white')
    fig.tight_layout(pad=0.5, w_pad=0, h_pad=0.0)
    fig.savefig(f'figures/imputation/lgbm_{w}_logs_pred.png', dpi=150)
fig, axs = plt.subplots(nrows=3, figsize=(20, 12))

mask = imputed['lgbrand_DTE'].set_nan
axs[0].plot(df[mask].DTE.values)
axs[0].plot(dt_pred1[mask])
axs[0].plot(imputed['lgbrand_DTE'][mask].DTE.values)

# err1 = np.power(np.power((df.loc[mask, 'DTE'].values - dt_pred1[mask])/df.loc[mask, 'DTE'], 2), 0.5)[mask].values
# err2 = np.power(np.power((df.loc[mask, 'DTE'].values - imputed['lgbrand_DTE'][mask].DTE.values)/df.loc[mask, 'DTE'], 2), 0.5)[mask].values
# axs[1].plot(err1)
# axs[1].plot(err2)

mask = imputed['lgbrand_DTSE'].set_nan
axs[1].plot(df[mask].DTSE.values)
axs[1].plot(dts_pred1[mask])
axs[1].plot(imputed['lgbrand_DTSE'][mask].DTSE.values)

mask = imputed['lgbrand_RHOBE'].set_nan
axs[2].plot(df[mask].RHOBE.values)
axs[2].plot(rho_pred1[mask])
axs[2].plot(imputed['lgbrand_RHOBE'][mask].RHOBE.values)
imputed['lgbrand_DTE']
from collections import defaultdict

def evaluate(data1, data2, j):
    se = np.power((data1.loc[data1.set_nan.values, j] - data2.loc[data1.set_nan.values, j])/data2.loc[data1.set_nan.values, j], 2)
    score = np.nanmean(np.power(se, 0.5))*100.0
    #score = mean_squared_error(data2.loc[data1.set_nan, j].values, data1.loc[data1.set_nan, j].values)
    return score

scores = defaultdict(dict)
for key, d in imputed_for.items():
    mod, key = key.split('_')
    scores[mod][key] = evaluate(d, df, key)

pd.DataFrame(scores)<jupyter_output><empty_output><jupyter_text>Imputation Cross Plots<jupyter_code>fig, axs = plt.subplots(ncols=3, nrows=3, figsize=(10,7.5))

for mod, axrow, name in zip(['lgbrand', 'knn', 'brr'], axs, ['LightGBM', 'KNN Regresssion', 'Bayesian Ridge Regression']):
    for key, ax in zip(impute_for.keys(), axrow):
        model = imputed[mod+'_'+key]
        set_nan = model.set_nan
        smin = np.nanmin(df.loc[set_nan, key].values)
        smax = np.nanmax(df.loc[set_nan, key].values)
        ax.scatter(df.loc[set_nan, key].values, model.loc[set_nan, key].values, c=model.loc[set_nan, 'ZONE_NO'].values, alpha=0.1, cmap='Set3', marker='.')
        ax.set_xlim(smin, smax)
        ax.set_ylim(smin, smax)
        ax.plot((smin, smax), (smin, smax), '--', color='r')
        ax.grid()
        ax.set_xlabel(key[:-1])

    axrow[0].set_ylabel(f'Imputed {name}')

plt.tight_layout()
fig.savefig('figures/imputation/versus.png', dpi=150)<jupyter_output><empty_output><jupyter_text>SHAP Analysis<jupyter_code>import shap
explainers = [shap.TreeExplainer(model.estimator, feature_perturbation="tree_path_dependent") for model in models_gbt['DTE'].imputation_sequence_]
for m in models_gbt['DTE'].imputation_sequence_[9:18] + models_gbt['DTE'].imputation_sequence_[-9:]:
    print(m.neighbor_feat_idx, m.feat_idx)
shap_vals_end = []
for exp, mod in zip(explainers[-9:], models_gbt['DTE'].imputation_sequence_[-9:]):
    shap_vals_end.append(exp.shap_values(impute_for['DTE'][0].iloc[:, mod.neighbor_feat_idx]) 
    )
shap_vals_beg = []
for exp, mod in zip(explainers[9:18], models_gbt['DTE'].imputation_sequence_[9:18]):
    shap_vals_beg.append(exp.shap_values(impute_for['DTE'][0].iloc[:, mod.neighbor_feat_idx]) 
    )
shap.initjs()
sel = np.random.randint(0, df.index.size, size=df.index.size) <= 500
sub = df.iloc[sel, :]
sub = sub.loc[:, ['DTSE', 'GRE', 'NPHIE', 'PEFE', 'RHOBE', 'RMElog', 'RDElog', 'ZONE_NO']]
shap.force_plot(exp.expected_value, shap_vals[0][sel,:], sub)
col_rename_map = {
    'ZONE_NO':'ZONE',
    'DTE':'DT',
    'DTSE':'DTS',
    'GRE':'GR',
    'NPHIE':'NPHI',
    'PEFE':'PEF',
    'RHOBE':'RHOB',
    'RMElog':'LogRM',
    'RDElog':'LogRD'
}
names = ['DTE', 'DTSE', 'GRE', 'NPHIE', 'PEFE', 'RHOBE', 'RMElog', 'RDElog', 'ZONE_NO']
sel = np.random.randint(0, df.index.size, size=df.index.size) <= 100
sub = df.iloc[sel, :]
for i, m in enumerate(range(-9, 0, 1)):
    mod = models_gbt['DTE'].imputation_sequence_[m]
    m_names = [names[j] for j in mod.neighbor_feat_idx]
    shap.summary_plot(
        shap_vals_end[i],
        df[m_names].rename(columns=col_rename_map),
        title=f"{names[mod.feat_idx]}",
        color_bar_label=f"Feature Value {names[mod.feat_idx]}",
        show=False,
        plot_size=(5, 3),
    )
    fig = plt.gcf()
    rect = fig.patch
    rect.set_facecolor('white')
    fig.tight_layout(pad=0.5, w_pad=0, h_pad=0.0)
    fig.savefig(f'figures/imputation/{names[mod.feat_idx]}_end.png', dpi=150)
    fig.clear()
df.rename(columns={'ZONE_NO':'ZONE'})
names = ['DTE', 'DTSE', 'GRE', 'NPHIE', 'PEFE', 'RHOBE', 'RMElog', 'RDElog', 'ZONE_NO']
sel = np.random.randint(0, df.index.size, size=df.index.size) <= 100
sub = df.iloc[sel, :]
for i, m in enumerate(range(9, 18, 1)):
    mod = models_gbt['DTE'].imputation_sequence_[m]
    m_names = [names[j] for j in mod.neighbor_feat_idx]
    shap.summary_plot(
        shap_vals_beg[i],
        df[m_names].rename(columns=col_rename_map),
        title=f"{names[mod.feat_idx]}",
        color_bar_label=f"Feature Value {names[mod.feat_idx]}",
        show=False,
        plot_size=(5, 3)
    )
    fig = plt.gcf()
    rect = fig.patch
    rect.set_facecolor('white')
    fig.tight_layout(pad=0.5, w_pad=0, h_pad=0.0)
    fig.savefig(f'figures/imputation/{names[mod.feat_idx]}_beg.png', dpi=150)
    fig.clear()
['DTE', 'DTSE', 'GRE', 'NPHIE', 'PEFE', 'RHOBE', 'RMElog', 'RDElog', 'ZONE_NO']<jupyter_output><empty_output><jupyter_text>Imputation ErrorIn this section we explore the impact of data sparsity on the predictions. The sparsity is introduced through random removal of data from model in each column. The SHAP analysis should answer other questions about feature importance e.g. missing features and how that might impact prediction quality.<jupyter_code>def data_prep_all(data, j, set_to_nan=0.3):
    data = data.copy()
    
    for col in data.columns:
        if col == 'j':
            continue
        rand_set_mask = np.random.random(data.shape[0]) < set_to_nan
        data.loc[data.index[rand_set_mask], col] = np.nan
    
    sub = data.dropna(subset=[j])
    rand_set_mask = np.random.random(len(sub)) < set_to_nan
    replace = sub.index[rand_set_mask]
    data.loc[replace, j] = np.nan
    data['set_nan'] = False
    data.loc[replace, 'set_nan'] = True
    data['was_nan'] = data[j].isna()
    return data

def evaluate(data1, data2, j):
    mask = data1.set_nan.values
    truth = data2.loc[mask, j].values
    test = data1.loc[mask, j].values
    se = np.power((truth - test)/truth, 2)
    score = np.nanmean(np.power(se, 0.5))*100.0
    er = dict(
        perc_error=score,
        explained_var=metrics.explained_variance_score(truth, test),
        max_error=metrics.max_error(truth, test),
        mae=metrics.mean_absolute_error(truth, test),
        mse=metrics.mean_squared_error(truth, test),
        r2=metrics.r2_score(truth, test),
    )
    return er

blank_scores = dict(
                    perc_error=np.nan,
                    explained_var=np.nan,
                    max_error=np.nan,
                    mae=np.nan,
                    mse=np.nan,
                    r2=np.nan,
                )


# models_err = dict()
# imputed_err = defaultdict(dict)
# impute_for_err = dict()
# for key in ['DT', 'DTS', 'RHOB']:
#     for missing in np.linspace(0.1, 0.9, 9):
#         mkey = f'{missing:0.1f}'
#         temp_df = train.copy()
#         temp_df.fillna(0.0)
#         impute_for_err[key+'_'+mkey] = data_prep_all(temp_df.copy(), key, missing)
    
for key in ['DT', 'DTS', 'RHOB']:
    for missing in np.linspace(0.1, 0.9, 9):
        mkey = f'{missing:0.1f}'
#         for imp_mod, args in imputation_args.items():
        for imp_mod, args in [(key, imputation_args[key]) for key in ["brr1", "knn1"]]:

            val = impute_for_err[f'{key}_{mkey}']
            try:
                models_err[f'{imp_mod}_{key}_{mkey}'] = train_models(val.copy(), **args)

                scores = evaluate(models_err[f'{imp_mod}_{key}_{mkey}'][0], train, key)
            except:
                scores = blank_scores
            scores['fmissing'] = missing
            scores['key'] = key
            scores['model'] = imp_mod
            imputed_err[f'{imp_mod}_{key}_{mkey}'] = scores
for missing in np.linspace(0.1, 0.9, 9):
    mkey = f'{missing:0.1f}'
    dt_col = ['DTS', 'GR', 'NPHI', 'PEF', 'RD10', 'RHOB', 'RM10', 'ZONE',]
    dts_col = ['DT', 'GR', 'NPHI', 'PEF', 'RD10', 'RHOB', 'RM10', 'ZONE',]
    rhob_col = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RD10', 'RM10', 'ZONE',]

    # DT
    try:
        temp_pred = imputation_test_dfs["DT"][0].copy()
        temp_pred['DT'] = dt_lgbm.predict(val.copy().loc[:, dt_col])
        scores = evaluate(temp_pred, train, 'DT')
    except:
        scores = blank_scores
    scores['fmissing'] = missing
    scores['key'] = 'DT'
    scores['model'] = 'lgbm1'
    imputed_err_test[f'lgb1_DT_{mkey}'] = scores
    
    # DTS
    try:
        temp_pred = imputation_test_dfs["DTS"][0].copy()
        temp_pred['DTS'] = dts_lgbm.predict(val.copy().loc[:, dts_col])
        scores = evaluate(temp_pred, train, 'DTS')
    except:
        scores = blank_scores
    scores['fmissing'] = missing
    scores['key'] = 'DTS'
    scores['model'] = 'lgbm1'
    imputed_err_test[f'lgb1_DTS_{mkey}'] = scores

    # RHOB
    try:
        temp_pred = imputation_test_dfs["RHOB"][0].copy()
        temp_pred['RHOB'] = rho_lgbm.predict(val.copy().loc[:, rhob_col])
        scores = evaluate(temp_pred, train, 'RHOB')
    except:
        scores=blank_scores
    scores['fmissing'] = missing
    scores['key'] = 'RHOB'
    scores['model'] = 'lgbm1'
    imputed_err_test[f'lgb1_RHOB_{mkey}'] = scores
imputed_err_test = dict()
for key, (_, imputer) in models_err.items():
    modk, log, fmissing = key.split("_")
    training_set = imputation_args[modk]["training_set"]
    temp_df = imputation_test_dfs[log][0].copy()
    temp_df.loc[:, training_set] = imputer.transform(temp_df[training_set])
    scores = evaluate(temp_df, test, log)
    
    scores['fmissing'] = fmissing
    scores['key'] = log
    scores['model'] = modk
    imputed_err_test[key] = scores
# run error testing for single pass

for missing in np.linspace(0.1, 0.9, 9):
    mkey = f'{missing:0.1f}'
    dt_lgbm = LGBMRegressor(random_state=456, max_depth=5, num_leaves=100, n_jobs=4)
    dts_lgbm = LGBMRegressor(random_state=456, max_depth=5, num_leaves=100, n_jobs=4)
    rho_lgbm = LGBMRegressor(random_state=456, max_depth=5, num_leaves=100, n_jobs=4)
    dt_col = ['DTS', 'GR', 'NPHI', 'PEF', 'RD10', 'RHOB', 'RM10', 'ZONE',]
    dts_col = ['DT', 'GR', 'NPHI', 'PEF', 'RD10', 'RHOB', 'RM10', 'ZONE',]
    rhob_col = ['DT', 'DTS', 'GR', 'NPHI', 'PEF', 'RD10', 'RM10', 'ZONE',]

    # DT
    val = impute_for_err[f'DT_{mkey}']

    try:
        dt_lgbm.fit(
            val.copy().dropna(subset=['DT']).loc[:, dt_col], 
            val.copy().dropna(subset=['DT'])['DT']
        )

        temp_pred = val.copy()
        temp_pred['DT'] = dt_lgbm.predict(val.copy().loc[:, dt_col])
        temp_pred_test = imputation_test_dfs["DT"][0].copy()
        temp_pred_test['DT'] = dt_lgbm.predict(temp_pred_test.loc[:, dt_col])
        scores = evaluate(temp_pred, train, 'DT')
        scores_test = evaluate(temp_pred_test, test, "DT")
    except:
        scores = blank_scores
        scores_test = blank_scores
        
    scores.update(dict(fmissing=mkey, key="DT", model="lgbm1"))
    imputed_err[f'lgb1_DT_{mkey}'] = scores
    scores_test.update(dict(fmissing=mkey, key="DT", model="lgbm1"))
    imputed_err_test[f'lgb1_DT_{mkey}'] = scores_test
    
    # DTS
    val = impute_for_err[f'DTS_{mkey}']

    try:
        dts_lgbm.fit(
            val.copy().dropna(subset=['DTS']).loc[:, dts_col], 
            val.copy().dropna(subset=['DTS'])['DTS']
        )

        temp_pred = val.copy()
        temp_pred['DTS'] = dts_lgbm.predict(val.copy().loc[:, dts_col])
        scores = evaluate(temp_pred, train, 'DTS')
        temp_pred_test = imputation_test_dfs["DTS"][0].copy()
        temp_pred_test['DTS'] = dts_lgbm.predict(temp_pred_test.loc[:, dts_col])
        scores = evaluate(temp_pred, train, 'DTS')
        scores_test = evaluate(temp_pred_test, test, "DTS")
    except:
        scores = blank_scores
        scores_test = blank_scores
        
    scores.update(dict(fmissing=mkey, key="DTS", model="lgbm1"))
    imputed_err[f'lgb1_DTS_{mkey}'] = scores
    scores_test.update(dict(fmissing=mkey, key="DTS", model="lgbm1"))
    imputed_err_test[f'lgb1_DTS_{mkey}'] = scores_test

    # RHOB
    val = impute_for_err[f'RHOB_{mkey}']

    try:
        rho_lgbm.fit(
            val.copy().dropna(subset=['RHOB']).loc[:, rhob_col], 
            val.copy().dropna(subset=['RHOB'])['RHOB']
        )

        temp_pred = val.copy()
        temp_pred['RHOB'] = rho_lgbm.predict(val.copy().loc[:, rhob_col])
        scores = evaluate(temp_pred, train, 'RHOB')
        temp_pred_test = imputation_test_dfs["RHOB"][0].copy()
        temp_pred_test['RHOB'] = rho_lgbm.predict(temp_pred_test.loc[:, rhob_col])
        scores = evaluate(temp_pred, train, 'RHOB')
        scores_test = evaluate(temp_pred_test, test, "RHOB")
    except:
        scores = blank_scores
        scores_test = blank_scores

    scores.update(dict(fmissing=mkey, key="RHOB", model="lgbm1"))
    imputed_err[f'lgb1_RHOB_{mkey}'] = scores
    scores_test.update(dict(fmissing=mkey, key="RHOB", model="lgbm1"))
    print(scores_test)
    imputed_err_test[f'lgb1_RHOB_{mkey}'] = scores_test
display(pd.DataFrame(imputed_err).T)
err_results_df = pd.DataFrame(imputed_err).T
err_results_df.to_csv('missing_err.csv', index=True)
imputed_err_test_df = pd.DataFrame(imputed_err_test).T

for col in ["perc_error", "explained_var", "max_error", "mae", "mse", "r2", "fmissing"]:
    err_results_df[col] = pd.to_numeric(err_results_df[col])
    imputed_err_test_df[col] = pd.to_numeric(imputed_err_test_df[col])    
import pickle

pickle.dump(imputed_err, open( "imputed_err.p", "wb" ))
pickle.dump(models_err, open("models_err.p", "wb"))

# imputed_err = pickle.load(open("imputed_err.p", "rb"))
# models_err = pickle.load(open("models_err.p", "rb"))
del imputed_err
del models_err
for measure, mt, ys in zip(
    ['perc_error', 'explained_var', 'max_error', 'mae', 'mse', 'r2'],
    ['Percentage Imputation Error', 'Explained Variance', 'Maximum Error', 'Mean Absolute Error', 'Mean Squared Error', r'$R^2$'],
    [True, True, False, False, False, True]
):
    fig, axs = plt.subplots(ncols=3, figsize=(15, 4), sharex=True, sharey=ys)

    sns.set_context('paper')

    for log, ax in zip(['DT', 'DTS', 'RHOB'], axs):
        sub = err_results_df.query("key == @log")
        for mod, name in zip(['lgbrand', 'brr', 'brr1', 'knn', 'knn1', 'lgbm1'], ['LightGBM', 'BRR', 'SingleBRR', 'KNN', 'SingleKNN', 'SingleLGBM']):
            sub_plot = sub.query("model == @mod").sort_values("fmissing")
            ax.plot(sub_plot.fmissing, sub_plot[measure], label=name)
        ax.set_title(log)
        ax.set_xlabel('Fraction of Data Missing')
        ax.set_ylabel(mt)
        ax.legend()
    fig.savefig(f'figures/error_{measure}.png', dpi=300)
for measure, mt, ys in zip(
    ['perc_error', 'explained_var', 'max_error', 'mae', 'mse', 'r2'],
    ['Percentage Imputation Error', 'Explained Variance', 'Maximum Error', 'Mean Absolute Error', 'Mean Squared Error', r'$R^2$'],
    [True, True, False, False, False, True]
):
    fig, axs = plt.subplots(ncols=3, figsize=(15, 4), sharex=True, sharey=ys)

    sns.set_context('paper')

    for log, ax in zip(['DT', 'DTS', 'RHOB'], axs):
        sub = imputed_err_test_df.query("key == @log")
        for mod, name in zip(['lgbrand', 'brr', 'brr1', 'knn', 'knn1', 'lgbm1'], ['LightGBM', 'BRR', 'SingleBRR', 'KNN', 'SingleKNN', 'SingleLGBM']):
            sub_plot = sub.query("model == @mod").sort_values("fmissing")
            ax.plot(sub_plot.fmissing.values, sub_plot[measure].values, label=name)
        ax.set_title(log)
        ax.set_xlabel('Fraction of Data Missing')
        ax.set_ylabel(mt)
        ax.set_ylim(0)
        ax.legend()
    fig.savefig(f'figures/error_test_{measure}.png', dpi=300)<jupyter_output><empty_output><jupyter_text>Impute Actual Logs<jupyter_code>well_index = data.query("WELL == 'F-12'")
# well_index = data.query("WELL == 'F-4'")


#apply model
def apply_model(df, models):
    apply_set = ['DTE', 'DTSE', 'GRE', 'NPHIE', 'PEFE', 'RHOBE', 'RMElog', 'RDElog', 'ZONE_NO']
    output = df.copy()
    for key, mod in models.items():
        index = apply_set.index(key)
        filled = mod.transform(df[apply_set])
        output.loc[:, key] = filled[:, index]
    return output

filled = apply_model(df.loc[well_index.index, :], models_gbt)

fig, axs = plt.subplots(nrows=3, figsize=(20,10))
for key, ax in zip(impute_for.keys(), axs):
    ax.plot(filled[key].values)
    ax.plot(df.loc[well_index.index, key].values)
#     ax.plot(imputed_brr[key].loc[set_nan, key].values, color='r')
    ax.set_title(key)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17726086463179394, 'fraction_numerical': 0.021876772949052537, 'mean_word_length': 4.392315222711699, 'pattern_counts': {'":': 8, '<': 79, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '584681', 'license': 'MIT', 'n_tokens_mistral': 18129, 'n_tokens_neox': 17249, 'n_words': 3140, 'path': '01_Imputation_Volve.ipynb', 'repo_name': 'trhallam/mice_well_log_imputation', 'size': '44090'}