<jupyter_start><jupyter_text>Feature Impute and Train Summary:Below are the predictions for 3yr cumulative oil recovery.![image.png](attachment:image.png) We model oil recovery using an artificial neural network that considers the petrophysical properties of the wellbore, measured at each depth. Using our ANN, we predict the oil recovery at each depth interval, and aggregate these predictions to produce the mean oil recovery per well. The following outline the advantages of this approach:  Predictions scale with well depth and new information. That is, there is no need to retrain the model if additional well log data comes is added, because the model outputs incremental recovery predictions for every new data point.The model is tolerant to incomplete data. (so are others?)  Predictions have less uncertainty, because there is no upscaling. Users can predict depth-dependent recovery estimates, and identify depth intervals of high production.We choose the following depth-dependent numerical attributes as predictor features: Porosity, fraction	Permeability, mD	Acoustic Impedance, kg*s/m^2	Density, g/cm3	Compressible velocity, m/s	Youngs modulus, GPa	Shear velocity, m/s	Shear modulus, GPa.  Label: Petrophysics-dependent cumulative oil recovery, MTSB, predicted for each depth and for each well.   Training StepsFor predictor features  Drop non predictor features from the input,  Standardize all predictor features,    Perform KNearestNeighbors with 8 neighbors,  Normalize to 0-1 (minmax).    For labels,  Perform log transformationANN hyperparameters:  2 layers, 10 nodes each, 100 epochs.  Rock facies has 8.9\% missing values, and is categorical which takes an extra step to impute.<jupyter_code># Import all the good stuff
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# To use this experimental feature, we need to explicitly ask for it:
from sklearn.experimental import enable_iterative_imputer  # noqa
from sklearn.datasets import fetch_california_housing
from sklearn.impute import SimpleImputer
from sklearn.impute import IterativeImputer
from sklearn.linear_model import BayesianRidge
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.pipeline import make_pipeline
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import FunctionTransformer
from sklearn.decomposition import PCA
from sklearn.base import BaseEstimator
from sklearn.base import TransformerMixin


import tensorflow as tf

import os
import pandas as pd
import tensorflow as tf
import subprocess
import pickle
import datetime
from tensorflow import keras
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.compose import make_column_transformer

# Read producer
prod = pd.read_csv("https://raw.githubusercontent.com/PGEHackathon/darcy-analytics/data_exp/wellbore_data_producer_wells.csv?token=<KEY>")
# Read cumulative oil
cumoil = pd.read_csv('https://raw.githubusercontent.com/PGEHackathon/darcy-analytics/data_exp/production_history.csv?token=<KEY>')
# Join on Well_ID (Left outer join)
df = pd.merge(prod, cumoil, on='Well_ID',how='left')
df.head()

# Split test-train, set 10% aside as test set
df_train, df_test=train_test_split(df, test_size=0.1, random_state=42)
# Select label column.
y_train = df_train['Cumulative oil production (3 yr), MSTB'].copy()
y_test = df_test['Cumulative oil production (3 yr), MSTB'].copy()
y_full = df['Cumulative oil production (3 yr), MSTB'].copy()
# Reshape label column, because pipelines require 2D data.
y_train = np.array(y_train).reshape((-1,1))
y_test = np.array(y_test).reshape((-1,1))
y_full = np.array(y_full).reshape((-1,1))
# Create ANN model for oil recov
def model_construct(n_layers, n_nodes, n_inputs):
    '''
    n_layers
    n_nodes
    n_inputs: Dims of inputs, X_train_prepared.shape[1]
    '''
    model = tf.keras.Sequential()
    model.add(
        tf.keras.layers.Dense(n_nodes, activation=tf.keras.layers.LeakyReLU(alpha=0.1), input_shape=[n_inputs]))
    for _ in range(n_layers-1):
        model.add(tf.keras.layers.Dense(n_nodes, activation=tf.keras.layers.LeakyReLU(alpha=0.1)))
    model.add(tf.keras.layers.Dense(1))

    # Remove lr if scheduler in use?
    model.compile(loss='mse', optimizer=keras.optimizers.Adam(),
                  metrics=['mse', 'mae', tf.keras.metrics.MeanAbsolutePercentageError()])
    return model
# Store version info as file in directory
def get_git_revision_hash():
    return subprocess.check_output(['git', 'rev-parse', 'HEAD'])
# Function to train ANN model
def train_model(model,
                X,
                y,
                X_test,
                y_test,
                batch_size,
                n_layers,
                n_nodes,
                epochs,
                initial_epoch,
                log_save_dir,
                name_prefix):
    # Create Directory for Logs
    model_name = name_prefix+'_'+str(batch_size)+'_'+str(n_layers)+'_'+str(n_nodes)+'_'+str(epochs)+'_'
    logdir = log_save_dir + ".\\logs\\scalars\\" + model_name + str(
        datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
    logdir = logdir
    if not os.path.exists(logdir):
        os.makedirs(logdir)
        print('New logdir created.')
        
    # Logs
    tensorboard_callback = keras.callbacks.TensorBoard(
        log_dir=logdir,
        histogram_freq=0,  # How often to log histogram visualizations
        write_graph=True,
        update_freq='epoch',
        profile_batch=0,  # set to 0. Else bug Tensorboard not show train loss.
        embeddings_freq=0,  # How often to log embedding visualizations
    )
    
    # Early stop
    early_stop = tf.keras.callbacks.EarlyStopping(monitor='mse', min_delta=0.001, patience=3)

    # Save Checkpoint
    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
        filepath=logdir,  # +'.\\{epoch:.02d}-{mse:.2f}',
        verbose=1,
        save_weights_only=False,
        monitor='mse',  # Not sure
        mode='auto',
        save_best_only=True)

    with open(logdir + '.\\version_info.txt', 'a', newline='') as file:
        file.write('model_name'+' '+str(get_git_revision_hash()) + '\n')

    # Train. History stores loss metrics per epoch
    history = model.fit(X, y, initial_epoch=initial_epoch, epochs=epochs,
                        callbacks=[tensorboard_callback, model_checkpoint_callback],
                        validation_data=(X_test, y_test),
                        shuffle=True,
                        batch_size=batch_size, verbose=2)

    # Save entire model with training config
    model.save(logdir + '.\\' + model_name + '{}'.format(str(datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))))
    
    return model

    endTime = datetime.datetime.now()
    print('Ended at ' + str(endTime))
    print('end')
# Define Transformers
# Select columns and standardize them. Drop all remaining.
std_scaler_X = StandardScaler()
select_and_std=make_column_transformer((std_scaler_X,[4,5,6,8,9,10,11,12]))
# Transformer for features, min-max normalize
x_min_max_scaler = MinMaxScaler()
# Custom log transformer
log_transformer = FunctionTransformer(np.log,np.exp, validate=True)
# Define transformation pipeline for label

label_pipe = Pipeline([
    ('select column and log transform', log_transformer)
])
# Define transformation pipeline for features
feature_pipe = Pipeline([
    ('Select_cols_and_std',select_and_std),
    ('Iterative_impute_KNN',IterativeImputer(random_state=0, estimator=KNeighborsRegressor(n_neighbors=8))),
    ('PCA',PCA(n_components=4)),
    ('minmax',x_min_max_scaler)
])
# Fit transform predictor features
X_train_prepared=feature_pipe.fit_transform(df_train)
# Transform test predictor features
X_test_prepared=feature_pipe.transform(df_test)
# Fit transform labels
y_train_prepared = label_pipe.fit_transform(y_train)
# Transform test labels
y_test_prepared = label_pipe.transform(y_test)
# Test Inverse transform the labels
temp=pd.DataFrame(label_pipe.inverse_transform(y_train_prepared))
temp.describe()
# Verify inverse transform for label is good. That is, the summary statistics are identical.
pd.DataFrame(y_train).describe()<jupyter_output><empty_output><jupyter_text>TrainTrain model using the train data subset (90% of entire data) to find good hyperparameters. We found 2 layers, 10 nodes to be good. Model does not overfit, as the validation error does not exceed the train error.<jupyter_code># Compile ANN architecture (nodes, layers)
# Choose leaky ReLU arbitratily. Should have fast convergence.
# Search hyperparameters for best model. n_layers, n_nodes, training epochs.
n_layers=2
n_nodes=10
oilrecov3_model = model_construct(n_layers, n_nodes, X_train_prepared.shape[1])
log_save_dir=r'C:\Users\win7\PycharmProjects\PGE_hack_local\logs'
name_prefix='oilrecov_log_nlayers{}_nodes{}_epochs{}_split_v4'.format(n_layers,n_nodes,50)
model = train_model(oilrecov3_model,
                X_train_prepared,
                y_train_prepared,
                X_test_prepared,
                y_test_prepared,                    
                batch_size=32,
                n_layers=n_layers,
                n_nodes=n_nodes,
                epochs=100,
                initial_epoch=0,
                log_save_dir=log_save_dir,
                name_prefix=name_prefix)<jupyter_output>New logdir created.
Train on 1314 samples, validate on 146 samples
Epoch 1/100

Epoch 00001: mse improved from inf to 42.49601, saving model to C:\Users\win7\PycharmProjects\PGE_hack_local\logs.\logs\scalars\oilrecov_log_nlayers2_nodes10_epochs50_split_v4_32_2_10_100_20210418-081259
INFO:tensorflow:Assets written to: C:\Users\win7\PycharmProjects\PGE_hack_local\logs.\logs\scalars\oilrecov_log_nlayers2_nodes10_epochs50_split_v4_32_2_10_100_20210418-081259\assets
1314/1314 - 2s - loss: 42.4960 - mse: 42.4960 - mae: 6.5023 - mean_absolute_percentage_error: 96.2913 - val_loss: 39.6666 - val_mse: 39.6666 - val_mae: 6.2834 - val_mean_absolute_percentage_error: 92.8619
Epoch 2/100

Epoch 00002: mse improved from 42.49601 to 36.33027, saving model to C:\Users\win7\PycharmProjects\PGE_hack_local\logs.\logs\scalars\oilrecov_log_nlayers2_nodes10_epochs50_split_v4_32_2_10_100_20210418-081259
INFO:tensorflow:Assets written to: C:\Users\win7\PycharmProjects\PGE_hack_local\logs.\logs\scalars\oilrecov[...]<jupyter_text>Train on full dataRepeat training on full data, with hyperparameters found in previous step. Validation mean absolute percent error: 4.5%.<jupyter_code># Fit transform predictor features
X_full_prepared=feature_pipe.fit_transform(df)
# Transform test labels
y_full_prepared = label_pipe.transform(y_full)
# Compile ANN architecture (nodes, layers)
# Choose leaky ReLU arbitratily. Should have fast convergence.
# Search hyperparameters for best model. n_layers, n_nodes, training epochs.
n_layers=2
n_nodes=10
oilrecov3_model = model_construct(n_layers, n_nodes, X_train_prepared.shape[1])
log_save_dir=r'C:\Users\win7\PycharmProjects\PGE_hack_local\logs'
name_prefix='oilrecov_log_nlayers{}_nodes{}_epochs{}_full'.format(n_layers,n_nodes,100)
model = train_model(oilrecov3_model,
                X_train_prepared,
                y_train_prepared,
                X_test_prepared,
                y_test_prepared,                    
                batch_size=32,
                n_layers=n_layers,
                n_nodes=n_nodes,
                epochs=100,
                initial_epoch=0,
                log_save_dir=log_save_dir,
                name_prefix=name_prefix)
# Load test data, the preproduction wells
preprod = pd.read_csv(r'https://raw.githubusercontent.com/PGEHackathon/darcy-analytics/main/wellbore_data_preproduction_well.csv?token=<KEY>')

# Add dummy columns, because need input dims match
dummy=pd.DataFrame(np.zeros((preprod.shape[0],19-preprod.shape[1])))
preprod = pd.concat([preprod, dummy],axis=1)

preprod.shape
# Transform features
X_preprod_prepared=feature_pipe.transform(preprod)
# Predict
y_preprod= model.predict(X_preprod_prepared)
# Inverse transform lable pipeline
y_preprod = label_pipe.inverse_transform(y_preprod)
preprod['Predicted 3 yr Oil Recovery, MTSB'] = y_preprod
preprod.iloc[:,[0,-1]].to_csv('Predicted_3yr_oil_recov_per_depth_and_well.csv')
preprod.head()
# Final Predictions: Cumulative oil recovery, 3yrs.
# Group predictions by Well_ID, and then take their mean. This is the final prediction for cumulative oil recovery for each well
preprod.groupby('Well_ID').agg('mean').iloc[:,-1]#.to_csv('mean_predicted_3yr_oil_recov_per_well.csv')
preprod.groupby('Well_ID').describe().iloc[:,-8:]#.to_csv('Stats_on_pred.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09149672951135052, 'fraction_numerical': 0.0235475182762601, 'mean_word_length': 4.225572979493365, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '170318', 'license': 'Apache-2.0', 'n_tokens_mistral': 4384, 'n_tokens_neox': 4068, 'n_words': 1014, 'path': '.ipynb_checkpoints/Feature Impute and Train and train-test-checkpoint.ipynb', 'repo_name': 'mark-zhao1/PGE_ML_Hackathon', 'size': '13067'}