<jupyter_start><jupyter_text>Define tensorflow model<jupyter_code>def create_placeholders(input_shape, output_shape):
    
    X = tf.placeholder(tf.float32, shape=(None,input_shape), name="X")
    y = tf.placeholder(tf.float32, shape=None, name="y")
    
    return X, y


def create_placeholders_additional(input_shape, output_shape):
    X = tf.placeholder(tf.float32, shape=(None, input_shape), name="X")
    y = tf.placeholder(tf.float32, shape=(None), name="y")
    c = tf.placeholder(tf.float32, shape=(), name="c")
    k = tf.placeholder(tf.float32, shape=(), name="k")
    D = tf.placeholder(tf.float32, shape=(), name="D")
    batch_size = tf.placeholder(tf.int32, shape=(), name="batch_size")
    deltas = tf.placeholder(tf.float32, shape=(input_shape), name="deltas")

    return X, y, c, k, D, batch_size, deltas

def tf_model(X):
    
    l2_lambda_regularization = 0.20
    l1_lambda_regularization = 0.10
    
    A1 = tf.layers.dense(X, 20, activation=tf.nn.relu, 
                         kernel_initializer=tf.contrib.layers.xavier_initializer(uniform=False), 
                         kernel_regularizer=tf.contrib.layers.l1_l2_regularizer(l1_lambda_regularization,l2_lambda_regularization), 
                         name="fc1")
    A2 = tf.layers.dense(A1, 20, activation=tf.nn.relu, 
                         kernel_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
                         kernel_regularizer=tf.contrib.layers.l1_l2_regularizer(l1_lambda_regularization,l2_lambda_regularization), name="fc2")
    y = tf.layers.dense(A2, 1, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
                        kernel_regularizer=tf.contrib.layers.l1_l2_regularizer(l1_lambda_regularization,l2_lambda_regularization), name="out")
    
    return y<jupyter_output><empty_output><jupyter_text>Create Model TF<jupyter_code>def tf_compiled_model(num_features, output_shape):
    tf.reset_default_graph()

    X, y = create_placeholders(num_features, output_shape)

    y_pred = tf_model(X)
    loss_function = loss_functions.tf_squared_residual_function(k, c, D, deltas, num_fevals)
    cost = loss_function(X, y_pred)
    reg_cost = tf.losses.get_regularization_loss()
    total_cost = cost + reg_cost

    optimizer = tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.5).minimize(total_cost)

    return {'X_placeholder': X, 'y_placeholder': y, 'y_pred': y_pred, 'cost': cost, 'total_cost': total_cost, 'optimizer': optimizer}


def tf_compiled_model_additional(num_features, output_shape):
    tf.reset_default_graph()
    
    X, y, c, k, D, batch_size, deltas = create_placeholders_additional(num_features, output_shape)

    y_pred = tf_model(X)
    cost = loss_functions.squared_residual_function(X, y_pred, deltas, k, c, D, batch_size)
    reg_cost = tf.losses.get_regularization_loss()
    total_cost = cost + reg_cost

    optimizer = tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.5).minimize(total_cost)

    return {'X_placeholder': X, 'y_placeholder': y, 'c_placeholder': c, 'k_placeholder': k, 'D_placeholder': D,
            'batch_size_placeholder': batch_size,
            'deltas_placeholder': deltas, 'y_pred': y_pred, 'cost': cost, 'total_cost': total_cost,
            'optimizer': optimizer}<jupyter_output><empty_output><jupyter_text>Train Functions<jupyter_code>def train_model_additional(tf_session, model, X_train, y_train, batch_size, epochs, get_minibatches_function_handle, deltas, k, c,
                D, verbose=1):

    # Retrieve model variables
    X = model['X_placeholder']
    y = model['y_placeholder']
    c_tf = model['c_placeholder']
    k_tf = model['k_placeholder']
    D_tf = model['D_placeholder']
    batch_size_tf = model['batch_size_placeholder']
    deltas_tf = model['deltas_placeholder']

    optimizer = model['optimizer']
    total_cost = model['total_cost']
    cost = model['cost']

    avg_cost_reg = 0.0
    avg_cost = 0.0

    with tf_session.as_default():

        # To reset all variables
        tf_session.run(tf.global_variables_initializer())

        for epoch in range(epochs):

            cost_tot = 0.0
            cost_reg_tot = 0.0
            
            X_batches, y_batches, total_batch = get_minibatches_function_handle(X_train, y_train, batch_size, deltas=deltas)

            # Train with the minibatches
            for i in range(total_batch):
                batch_x, batch_y = X_batches[i], y_batches[i]
                batch_size_real = int(batch_x.shape[0] / num_fevals)

                _, c_reg, f_cost = tf_session.run([optimizer, total_cost, cost],
                                     feed_dict={X: batch_x, y: batch_y, k_tf: k, c_tf: c, D_tf: D,
                                                batch_size_tf: batch_size_real, deltas_tf: deltas})
                cost_tot += f_cost
                cost_reg_tot += c_reg

                avg_cost = cost_tot / total_batch
                avg_cost_reg = cost_reg_tot / total_batch

            if verbose == 1:
                print("Epoch:", '%04d' % (epoch + 1), "cost_reg=", "{:.9f}".format(avg_cost_reg), "cost=", "{:.9f}".format(avg_cost))

    print("Epoch:Final", "cost_reg=", "{:.9f}".format(avg_cost_reg), "cost=", "{:.9f}".format(avg_cost))


def train_model(tf_session, model, X_train, y_train, batch_size, epochs, get_minibatches_function_handle, deltas, verbose=1):

    # Retrieve model variables
    X = model['X_placeholder']
    y = model['y_placeholder']

    optimizer = model['optimizer']
    total_cost = model['total_cost']
    cost = model['cost']

    avg_cost_reg = 0.0
    avg_cost = 0.0

    with tf_session.as_default():

        # To reset all variables
        tf_session.run(tf.global_variables_initializer())

        for epoch in range(epochs):

            cost_tot = 0.0
            cost_reg_tot = 0.0

            X_batches, y_batches, total_batch = get_minibatches_function_handle(X_train, y_train, batch_size, deltas=deltas)

            # Train with the minibatches
            for i in range(total_batch):
                batch_x, batch_y = X_batches[i], y_batches[i]

                _, c_reg, f_cost = tf_session.run([optimizer, total_cost, cost], feed_dict={X: batch_x, y: batch_y})
                cost_tot += f_cost
                cost_reg_tot += c_reg

                avg_cost = cost_tot / total_batch
                avg_cost_reg = cost_reg_tot / total_batch

            if verbose == 1:
                print("Epoch:", '%04d' % (epoch + 1), "cost_reg=", "{:.9f}".format(avg_cost_reg), "cost=", "{:.9f}".format(avg_cost))

    print("Epoch:Final", "cost_reg=", "{:.9f}".format(avg_cost_reg), "cost=", "{:.9f}".format(avg_cost))<jupyter_output><empty_output><jupyter_text>Prepare data (create data handler and load)<jupyter_code>dhandler_stochastic = OscillatorDataHandler()
dhandler_stochastic.load_data(verbose=1, cross_validation_ratio=0.2, x=[0, 0], boundaries=[10, 10], n=[50, 50])
dhandler_stochastic.print_data()

minibatches_function_handle = aux_functions_stochastic.get_minibatches

X_train = dhandler_stochastic.X_train
y_train = dhandler_stochastic.y_train
X_crossVal = dhandler_stochastic.X_crossVal
y_crossVal = dhandler_stochastic.y_crossVal
X_test = dhandler_stochastic.X_test
y_test = dhandler_stochastic.y_test<jupyter_output>Loading data. Cros-Validation ratio 0.2
Printing shapes

Training data (X, y)
(7200, 2)
(7200, 1)
Cross-Validation data (X, y)
(1800, 2)
(1800, 1)
Testing data (X, y)
(1000, 2)
(1000, 1)
Printing first 5 elements

Training data (X, y)
[[ -1.51515152   8.98989899]
 [  4.74747475   2.92929293]
 [  0.90909091   0.70707071]
 [ -0.50505051  -4.94949495]
 [  6.56565657 -10.        ]]
[[0.]
 [0.]
 [0.]
 [0.]
 [0.]]
Cross-Validation data (X, y)
[[-2.12121212 -8.38383838]
 [ 6.36363636 -9.7979798 ]
 [ 0.50505051 -7.17171717]
 [ 9.7979798   2.72727273]
 [10.         -1.71717172]]
[[0.]
 [0.]
 [0.]
 [0.]
 [0.]]
Testing data (X, y)
[[-8.78787879 -5.35353535]
 [-8.38383838 -8.38383838]
 [ 7.37373737  8.78787879]
 [ 2.92929293  5.75757576]
 [-0.70707071 -1.31313131]]
[[0.]
 [0.]
 [0.]
 [0.]
 [0.]]<jupyter_text>Train with function additional<jupyter_code>model_additional = tf_compiled_model_additional(2, 1)

X = model_additional['X_placeholder']
y = model_additional['y_placeholder']
c_tf = model_additional['c_placeholder']
k_tf = model_additional['k_placeholder']
D_tf = model_additional['D_placeholder']
batch_size_tf = model_additional['batch_size_placeholder']
deltas_tf = model_additional['deltas_placeholder']

sess = tf.Session()
#writer = tf.summary.FileWriter('./graphs', sess.graph)

train_model_additional(sess, model_additional, X_train, y_train, 512, 100, 
                       minibatches_function_handle, deltas, k, c, D, verbose=1)
phi_nn = model_additional['y_pred']
phi_pred = sess.run(phi_nn, feed_dict={X: X_crossVal})

#Evaluate real model
sigma_x = np.sqrt(D / (k * c))
sigma_y = np.sqrt(D / c)
p_real, phi_real = analytic_functions.real_p(X_crossVal[:,0], X_crossVal[:,1], sigma_x, sigma_y)

print("Predicted phi")
print(phi_pred)

print("Real phi")
print(phi_real)

d = 2 * np.pi * sigma_x * sigma_y
c_not = 1/d
p_pred = c_not * np.exp(-phi_pred)

p_pred.flatten()
p_real.flatten()

print("Predicted p")
print(p_pred)

print("Real p")
print(p_real)

i = range(len(phi_real))

for i, phi_r, phi_predicted in zip(i, phi_real, phi_pred):
    print('xy {}, Real Phi {}, Predicted Phi {}'.format(X_crossVal[i], phi_r, phi_predicted))

e_phi = mean_squared_error(phi_pred, phi_real)
print("Cross validation MSE (For phi)")
print(e_phi)

e_p = mean_squared_error(p_pred, p_real)
print("Cross validation MSE (For p)")
print(e_p)<jupyter_output>Epoch: 0001 cost_reg= 480.348300680 cost= 468.770448049
Epoch: 0002 cost_reg= 202.942771276 cost= 191.470499452
Epoch: 0003 cost_reg= 106.024066925 cost= 94.625225131
Epoch: 0004 cost_reg= 66.531316566 cost= 55.193386761
Epoch: 0005 cost_reg= 49.754636383 cost= 38.464515241
Epoch: 0006 cost_reg= 41.013288307 cost= 29.773623705
Epoch: 0007 cost_reg= 34.839072609 cost= 23.642713483
Epoch: 0008 cost_reg= 30.578040695 cost= 19.424820272
Epoch: 0009 cost_reg= 27.428244909 cost= 16.315952404
Epoch: 0010 cost_reg= 25.418596331 cost= 14.344959537
Epoch: 0011 cost_reg= 24.019105657 cost= 12.986459057
Epoch: 0012 cost_reg= 23.168436559 cost= 12.170939648
Epoch: 0013 cost_reg= 22.128615061 cost= 11.170932730
Epoch: 0014 cost_reg= 23.083789062 cost= 12.158154090
Epoch: 0015 cost_reg= 22.669623947 cost= 11.779777559
Epoch: 0016 cost_reg= 21.372068914 cost= 10.516628385
Epoch: 0017 cost_reg= 20.574759102 cost= 9.747726019
Epoch: 0018 cost_reg= 20.459273020 cost= 9.664748303
Epoch: 0019 cost_reg= 19.[...]<jupyter_text>Train with function minimal<jupyter_code>model = tf_compiled_model(2, 1)

X = model['X_placeholder']
y = model['y_placeholder']

sess = tf.Session()
#writer = tf.summary.FileWriter('./graphs', sess.graph)

train_model(sess, model, X_train, y_train, 512, 10, minibatches_function_handle, deltas, verbose=1)
phi_nn = model['y_pred']
phi_pred = sess.run(phi_nn, feed_dict={X: X_crossVal})

#Evaluate real model
sigma_x = np.sqrt(D / (k * c))
sigma_y = np.sqrt(D / c)
p_real, phi_real = analytic_functions.real_p(X_crossVal[:,0], X_crossVal[:,1], sigma_x, sigma_y)

print("Predicted phi")
print(phi_pred)

print("Real phi")
print(phi_real)

d = 2 * np.pi * sigma_x * sigma_y
c_not = 1/d
p_pred = c_not * np.exp(-phi_pred)

p_pred.flatten()
p_real.flatten()

print("Predicted p")
print(p_pred)

print("Real p")
print(p_real)

i = range(len(phi_real))

for i, phi_r, phi_predicted in zip(i, phi_real, phi_pred):
    print('xy {}, Real Phi {}, Predicted Phi {}'.format(X_crossVal[i], phi_r, phi_predicted))

e_phi = mean_squared_error(phi_pred, phi_real)
print("Cross validation MSE (For phi)")
print(e_phi)

e_p = mean_squared_error(p_pred, p_real)
print("Cross validation MSE (For p)")
print(e_p)<jupyter_output>Epoch: 0001 cost_reg= 176.777750397 cost= 165.784838867
Epoch: 0002 cost_reg= 108.088999367 cost= 97.399994342
Epoch: 0003 cost_reg= 68.811987813 cost= 58.394070848
Epoch: 0004 cost_reg= 45.568504079 cost= 35.382243252
Epoch: 0005 cost_reg= 32.458363597 cost= 22.490811372
Epoch: 0006 cost_reg= 25.954092026 cost= 16.180771041
Epoch: 0007 cost_reg= 23.092154757 cost= 13.479665196
Epoch: 0008 cost_reg= 20.324902471 cost= 10.862019563
Epoch: 0009 cost_reg= 19.179658190 cost= 9.858623912
Epoch: 0010 cost_reg= 17.390146573 cost= 8.192548227
Epoch:Final cost_reg= 17.390146573 cost= 8.192548227
Predicted phi
[[ 0.01619785]
 [-0.01041145]
 [-0.01640668]
 ...
 [-0.0170958 ]
 [ 0.00247907]
 [-0.00682392]]
Real phi
[[3.73941435]
 [6.82481379]
 [2.58443016]
 ...
 [1.24987246]
 [3.55575962]
 [4.89847975]]
Predicted p
[[0.01565977]
 [0.01608206]
 [0.01617877]
 ...
 [0.01618992]
 [0.01587609]
 [0.01602447]]
Real p
[[3.78279770e-04]
 [1.72918381e-05]
 [1.20064986e-03]
 ...
 [4.56044706e-03]
 [4.5454130[...]<jupyter_text>Create Tunable Model Tensorflow and assign data<jupyter_code>tModel = SequenceTunableModelRegression('ModelStochastic_SN_1', model, lib_type='tensorflow', 
                                        data_handler=dhandler_stochastic)

tModel.X_test = dhandler_stochastic.X_test
tModel.X_train = dhandler_stochastic.X_train
tModel.X_crossVal = dhandler_stochastic.X_crossVal
tModel.y_crossVal = dhandler_stochastic.y_crossVal
tModel.y_test = dhandler_stochastic.y_test
tModel.y_train = dhandler_stochastic.y_train

tModel.print_data()<jupyter_output>Printing shapes

Training data (X, y)
(7200, 2)
(7200, 1)
Cross-Validation data (X, y)
(1800, 2)
(1800, 1)
Testing data (X, y)
(1000, 2)
(1000, 1)
Printing first 5 elements

Training data (X, y)
[[ -1.51515152   8.98989899]
 [  4.74747475   2.92929293]
 [  0.90909091   0.70707071]
 [ -0.50505051  -4.94949495]
 [  6.56565657 -10.        ]]
[[0.]
 [0.]
 [0.]
 [0.]
 [0.]]
Cross-Validation data (X, y)
[[-2.12121212 -8.38383838]
 [ 6.36363636 -9.7979798 ]
 [ 0.50505051 -7.17171717]
 [ 9.7979798   2.72727273]
 [10.         -1.71717172]]
[[0.]
 [0.]
 [0.]
 [0.]
 [0.]]
Testing data (X, y)
[[-8.78787879 -5.35353535]
 [-8.38383838 -8.38383838]
 [ 7.37373737  8.78787879]
 [ 2.92929293  5.75757576]
 [-0.70707071 -1.31313131]]
[[0.]
 [0.]
 [0.]
 [0.]
 [0.]]<jupyter_text>Train model Tensorflow<jupyter_code>tModel.epochs = 100
lrate = LearningRateScheduler(aux_functions.step_decay)
minibatches_function_handle = aux_functions_stochastic.get_minibatches

#tf.get_variable("deltas", initializer=deltas)

sess = tf.Session()

writer = tf.summary.FileWriter('./graphs', sess.graph)

tModel.train_model(tf_session=sess, get_minibatches_function_handle=minibatches_function_handle, 
                   verbose=1, deltas=deltas)
#Evaluate real model
X_test = tModel.X_crossVal

sigma_x = np.sqrt(D / (k * c))
sigma_y = np.sqrt(D / c)
p_real, phi_real = analytic_functions.real_p(X_test[:,0], X_test[:,1], sigma_x, sigma_y)

tModel.y_crossVal = phi_real

#p_real = np.ravel(p_real)
#phi_real = np.ravel(phi_real)

tModel.evaluate_model(['mse', 'rmse'], cross_validation=True, tf_session=sess)
#phi_pred = np.ravel(tModel.y_predicted)
phi_pred = tModel.y_predicted

print(mean_squared_error(phi_pred, phi_real))

d = 2 * np.pi * sigma_x * sigma_y
c_not = 1/d
p_pred = c_not * np.exp(-phi_pred)

print("scores")

cScores = tModel.scores
#rmse = math.sqrt(cScores['score_1'])
rmse2 = cScores['rmse']
mse = cScores['mse']
time = tModel.train_time

i = range(len(phi_pred))


for i, phi_real, phi_predicted in zip(i, phi_real, phi_pred):
    print('xy {}, Real Phi {}, Predicted Phi {}'.format(X_test[i], phi_real, phi_predicted))

print(cScores)
#print("RMSE: {}".format(rmse))
print("RMSE2: {}".format(rmse2))
print("MSE: {}".format(mse))
print("Time : {} seconds".format(time))
a = np.array([1, 2, 3])
b = np.array([4, 5, 6])
c = np.array([[1], [2], [3]])
d = np.array([[4], [5], [6]])

e1 = mean_squared_error(a, b)
e2 = mean_squared_error(c, d)

print(e1)
print(e2)<jupyter_output>9.0
9.0
{'fraction_non_alphanumeric': 0.13859275053304904, 'fraction_numerical': 0.1093064091308165, 'mean_word_length': 4.0274274905422445, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '151944', 'license': 'MIT', 'n_tokens_mistral': 7446, 'n_tokens_neox': 6240, 'n_words': 1182, 'path': 'code/Stochastic_Systems.ipynb', 'repo_name': 'dlaredo/stochastic_dynamics', 'size': '15946'}