<jupyter_start><jupyter_text>Prepare conda environment<jupyter_code>!conda env create -n pytf36 tensorflow-gpu==1.14.0<jupyter_output><empty_output><jupyter_text>Load Python Libraries<jupyter_code>#
# data processing
import pandas as pd
import numpy as np
from scipy.signal import medfilt
from sklearn.preprocessing import MinMaxScaler
#model estimation
from sklearn.metrics import accuracy_score
#custom functions
from config import *
from DataGenerator import *
import tensorflow as tf
from tensorflow.compat.v1.keras.backend import set_session
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True  # dynamically grow the memory used on the GPU
config.log_device_placement = True  # to log device placement (on which device the operation ran)
sess = tf.compat.v1.Session(config=config)
set_session(sess)<jupyter_output>Using TensorFlow backend.
/home/geoanton/anaconda3/envs/keras/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint8 = np.dtype([("qint8", np.int8, 1)])
/home/geoanton/anaconda3/envs/keras/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
/home/geoanton/anaconda3/envs/keras/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint16 = np.dtype([("qint16", np.int16, 1)])
/home/geoanton/an[...]<jupyter_text>Load the dataTo run on a private test set, set `TEST_NAME` to a location of the private test csv.<jupyter_code>DATA_PATH = './data/raw/'
TEST_NAME = f'{DATA_PATH}test.csv'

test = pd.read_csv(TEST_NAME)
test.head()
GetData=DataGenerator()
predictions = np.zeros((GetData.X_test.shape[0],1104,5))

for i in range(5):
    model=DL_model(input_size=(GetData.X_test.shape[1],GetData.X_test.shape[2]) ,hyperparams=HYPERPARAM)
    model.load_weights(f'./data/weights/LSTM_model_{i}_.h5')
    predictions += model.predict(GetData.X_test)/5
    
predictions = predictions[:,:1100:,:]
predictions[0,:,0]
def prepare_test(pred_test, df_test):
    
    wells = df_test['well_id'].sort_values().unique().tolist()
    list_df_wells = [df_test.loc[df_test['well_id'].isin([w]), :].copy() for w in wells]
    
    for df in list_df_wells:
        df.index = np.arange(df.shape[0])

    for i, df_well in enumerate(list_df_wells):
        df_well['label'] = np.argmax(pred_test[i, :], axis=1)

    result = pd.concat(list_df_wells, axis=0)
    return result

submit = prepare_test(predictions, test)
submit[['row_id', 'well_id', 'label']].to_csv('data/result/0.985_submit.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.136986301369863, 'fraction_numerical': 0.025440313111545987, 'mean_word_length': 5.323711340206185, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '574499', 'license': 'MIT', 'n_tokens_mistral': 1144, 'n_tokens_neox': 1107, 'n_words': 253, 'path': 'submission_notebook.ipynb', 'repo_name': 'AntonBiryukovUofC/gamma_log_classification', 'size': '3066'}