<jupyter_start><jupyter_text>loading Configurartion<jupyter_code># load some configurations configurations of model and others

configs = json.load(open('config_test_lstm.json', 'r'))
output_path = configs['paths']['save_dir']
output_logs = configs['paths']['log_dir']

if os.path.isdir(output_path) == False:
    os.mkdir(output_path)

if os.path.isdir(output_logs) == False:
    os.mkdir(output_logs)        

save_fname = os.path.join(output_path, 'architecture-%s.png' % configs['model']['name'])
save_fnameh5 = os.path.join(output_path, 'model-%s.h5' % configs['model']['name'])
time_steps =  configs['model']['layers'][0]['input_timesteps']  # the number of points or hits
num_features = configs['model']['layers'][0]['input_features']  # the number of features of each hits

split = configs['data']['train_split']  # the number of features of each hits
cylindrical = configs['data']['cylindrical']  # set to polar or cartesian coordenates
normalise = configs['data']['normalise'] 
num_hits = configs['data']['num_hits']
print(save_fname) 
print(save_fnameh5)
print('OK reading of json file')<jupyter_output><empty_output><jupyter_text>Setup Data<jupyter_code># reading path of data
#path = './dataset/2020_100_sorted.csv'
#path = './dataset/event_2000_2020_train.csv'
#path = './dataset/events.csv'
#path = './dataset/phi025-025_eta025-025/phi025-025_eta025-025_filtered.csv'
#path = './dataset/phi025-025_eta025-025_train1_lasthit_20200219.csv'
#"filename": "./dataset/phi025-025_eta025-025_train1_lasthit_20200219.csv",

path = configs['data']['filename']
path
data = Dataset(path, KindNormalization.Zscore)

dataset = data.get_training_data(cylindrical=cylindrical, hits=10)
#dataset = dataset.iloc[0:2640,0:]
#dataset = dataset.iloc[0:31600,0:]

print(dataset.shape)

dataset.head(6)
# reshape data     
time_steps =  configs['model']['layers'][0]['input_timesteps']  # the number of points or hits
num_features = configs['model']['layers'][0]['input_features']  # the number of features of each hits
split = configs['data']['train_split']  # the number of features of each hits
cylindrical = configs['data']['cylindrical']  # set to polar or cartesian coordenates
normalise = configs['data']['normalise'] 

print("[Data] Converting to supervised ...")

X, y = data.convert_to_supervised(dataset.values, n_hit_in=time_steps,
                            n_hit_out=1, n_features=num_features, normalise=normalise)
print('[Data] New Shape: X%s y%s :' % (X.shape, y.shape))
    
X.head(12)

# there is empty data?
print(X.isnull().values.any())
print(y.isnull().values.any())

#X.isnull().sum()
#X.isna().sum()
#y.isnull().sum()
#X = X.dropna()
#print('[Data] New Shape: X%s y%s :' % (X.shape, y.shape))
X = data.reshape3d(X, time_steps, num_features)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1-split, shuffle=False)

print('[Data] shape X_train ', X_train.shape)
print('[Data] shape y_train ', y_train.shape)
print('[Data] shape X_test ', X_test.shape)
print('[Data] shape y_test ', y_test.shape)
def manage_models(config):
    
    type_model = config['model']['name']
    model = None

    if type_model == 'lstm': #simple LSTM
        model = ModelLSTM(config)
    elif type_model == 'lstm-paralel':
        model = ModelLSTMParalel(config)
    elif type_model == 'cnn':
        model = ModelCNN(config)
    elif type_model == 'mlp':
        model = ModelMLP(config)        

    return model<jupyter_output><empty_output><jupyter_text>Create Model<jupyter_code># geramos o modelo
#configs = json.load(open('config_test.json', 'r'))
model = manage_models(configs)

if model is None:
    print('Please instance model')

loadModel = configs['training']['load_model']
#loadModel = False
loadModel = True<jupyter_output><empty_output><jupyter_text>Training Model<jupyter_code>from core.utils.metrics import *
# in-memory training
if loadModel == False:

    model.build_model()

    # in-memory training
    history = model.train(
        x=X_train,
        y=y_train,
        epochs = 10, #configs['training']['epochs'],
        batch_size = configs['training']['batch_size']
    )

    evaluate_training(history, output_path)

elif loadModel == True:       
    if not model.load_model():
        print ('[Error] please change the config file : load_model')

from core.utils.metrics import *
evaluate_training(history, output_path)<jupyter_output><empty_output><jupyter_text>Evaluate The model<jupyter_code>#evaluate the model
from core.utils.metrics import *
evaluate_training(history,output_path)<jupyter_output><empty_output><jupyter_text>Prediction We concatenate features to predict one hit.<jupyter_code># Prediction
import numpy as np
from core.utils.metrics import *
from core.utils.utils import *

predicted = model.predict_one_hit(X_test)
print('[Data] shape predicted output ', predicted.shape)
print('[Data] shape y_test ', y_test.shape)

# Convert to original data
#y_predicted = data.reshape2d(predicted, 1)
   
y_predicted = np.reshape(predicted, (predicted.shape[0]*predicted.shape[1], 1))
y_true_ = data.reshape2d(y_test, 1)

result = calc_score(y_true_, y_predicted, report=True)
r2, rmse, rmses = evaluate_forecast(y_test.values, predicted)
summarize_scores(r2, rmse,rmses)

if normalise:
    print('[Data] Inverse Transforming ...')
    y_test_orig = data.inverse_transform_y(y_test)
    y_predicted_orig = data.inverse_transform_y(predicted)
else:
    y_test_orig = y_test
    y_predicted_orig = predicted

print('[Data] shape y_test_orig ', y_test_orig.shape)
print('[Data] shape y_predicted_orig ', y_predicted_orig.shape)<jupyter_output><empty_output><jupyter_text>convert values<jupyter_code>X_train, X_test_new = train_test_split(dataset, test_size=1-0.8, shuffle=False)

print('[Data] shape X_train ', X_train.shape)
print('[Data] shape X_test_new ', X_test_new.shape)

y_predicted_orig = pd.DataFrame(y_predicted_orig)
y_predicted_orig = data.convert_supervised_to_normal(y_predicted_orig.values, n_hit_in=4, n_hit_out=1, hits=10)
y_predicted_orig = pd.DataFrame(y_predicted_orig)
print('y_predicted_orig shape ', y_predicted_orig.shape)

y_true_orig = pd.DataFrame(y_test_orig)
y_true_orig = data.convert_supervised_to_normal(y_true_orig.values, n_hit_in=4, n_hit_out=1, hits=10)
y_true_orig = pd.DataFrame(y_true_orig)
print('y_true_orig shape ', y_true_orig.shape)
X_test_new.head(12)
x_true = X_test_new.iloc[:,0:time_steps*num_features]
y_true = X_test_new.iloc[:,time_steps*num_features:]
print('shape x_true ', x_true.shape)
print('shape y_true ', y_true.shape)
# join both dataset must be equal to X_test
data_test = np.concatenate([x_true , y_true_orig], axis = 1 )
data_test = pd.DataFrame(data_test)
print('Shape ', data_test.shape)
# must be equal
data_test.values == X_test_new.values

# join both dataset should be equal to X_test but not equal
data_pred = np.concatenate([x_true , y_predicted_orig], axis = 1 )
data_pred = pd.DataFrame(data_pred)
print('Shape ', data_pred.shape)
# should not be equal
data_pred.values == X_test_new.values
if cylindrical:

    y_true_orig.to_csv(os.path.join(output_path, 'y_true_%s_cylin.csv' % configs['model']['name']),
                header=False, index=False)

    y_predicted_orig.to_csv(os.path.join(output_path, 'y_pred_%s_cylin.csv' % configs['model']['name']),
                header=False, index=False)

    x_true.to_csv(os.path.join(output_path, 'x_true_%s_cylin.csv' % configs['model']['name']),
                header=False, index=False)
else:
    
    y_true_orig.to_csv(os.path.join(output_path, 'y_true_%s_xyz.csv' % configs['model']['name']),
                header=False, index=False)
    y_predicted_orig.to_csv(os.path.join(output_path, 'y_pred_%s_xyz.csv' % configs['model']['name']),
                header=False, index=False)

    x_true.to_csv(os.path.join(output_path, 'x_true_%s_xyz.csv' % configs['model']['name']),
                header=False, index=False)

print('[Output] Results saved at %', output_path)<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code># Dados originais
N_tracks = 30
name = configs['model']['name']
path_html = ''

if cylindrical:
    path_html=os.path.join(output_path, 'track-original-%s_cylin.html' % name)
else:
    path_html=os.path.join(output_path, 'track-original-%s_xyz.html' % name)
    
fig = track_plot_xyz([X_test_new], n_hits = 10, cylindrical = cylindrical, n_tracks = 50, 
               path=path_html, title='Track Original #10 Hit - Model %s' % name.upper())

fig.show()
# Dados test

fig = track_plot_xyz([data_test], n_hits = 10, cylindrical = False, n_tracks = N_tracks, 
                title='Track Prediction #10 Hit')

fig.show()
# predicted data
path_html = ''
N_tracks = 30

if cylindrical:
    path_html=os.path.join(output_path, 'track-predicted-%s_cylin.html' % name)
else:
    path_html=os.path.join(output_path, 'track-predicted-%s_xyz.html' % name)
    
fig = track_plot_xyz([data_test, data_pred], n_hits = 10, cylindrical = cylindrical, n_tracks = N_tracks, 
               path=path_html, title='Track Prediction #10 Hit - Model %s' % name.upper())

fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14097641250685683, 'fraction_numerical': 0.01689522764673615, 'mean_word_length': 5.2782369146005506, 'pattern_counts': {'":': 1, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '302089', 'license': 'MIT', 'n_tokens_mistral': 3434, 'n_tokens_neox': 3267, 'n_words': 604, 'path': 'notebooks/main_test.ipynb', 'repo_name': 'SPRACE/track-ml', 'size': '9115'}