<jupyter_start><jupyter_text>1. Load DataSplit train/test data. Investigate the distribution of positive/training data<jupyter_code>data = np.load("./Datasets/Data_M_2.8_R_0.5_S_2_Sec_256.npy")
label = np.load("./Datasets/Label_M_2.8_R_0.5_S_2_Sec_256.npy")
#data = np.load("Data.npy")
#label = np.load("label.npy")
# Standardized the data first.
data=zscore(data,axis=0)

# time = np.load("Time.npy")
print(data.shape)
print(label.shape)



def split_reshape_dataset(X, Y, ratio):
    X = X.T[:,:,np.newaxis, np.newaxis]
    Y = Y.T
    m = X.shape[0] # number of samples
    sortInd = np.arange(m)
    np.random.shuffle(sortInd)
    nTrain = int(ratio * m)
    X_train = X[sortInd[:nTrain], :, :, :]
    Y_train = Y[:, sortInd[:nTrain]]
    X_test = X[sortInd[nTrain:], :, :, :]
    Y_test = Y[:, sortInd[nTrain:]]
    return X_train, X_test, Y_train, Y_test

RatioTraining=0.8 # Use 0.8, since we only have 1,000 examples
X_train, X_test, Y_train, Y_test = split_reshape_dataset(data, label, RatioTraining)
Y_train =convert_to_one_hot(Y_train,2).T
Y_test = convert_to_one_hot(Y_test,2).T

m=X_train.shape[0]
m_test=X_test.shape[0]
## Check the distribution of pos/neg examples.
Distribution={'Y_trainPos':np.count_nonzero(Y_train[:,1])/m,'Y_trainNeg':1-np.count_nonzero(Y_train[:,1])/m,
              'Y_testPos':np.count_nonzero(Y_test[:,1])/m_test,'Y_testNeg':1-np.count_nonzero(Y_test[:,1])/m_test}

print("For Training data, there are %3.2f pos and %3.2f neg data " %(Distribution['Y_trainPos'],Distribution['Y_trainNeg']))
print("For Test data, there are %3.2f pos and %3.2f neg data " %(Distribution['Y_testPos'],Distribution['Y_testNeg']))
#print(X_train.shape)
#print(Y_train.shape)
#print(X_test.shape)
#print(Y_test.shape)<jupyter_output>(10240, 1006)
(1006, 1)
For Training data, there are 0.21 pos and 0.79 neg data 
For Test data, there are 0.18 pos and 0.82 neg data<jupyter_text>2. HyperparametersHyperparameters that are perturbed are learning rate $\alpha$, regularization parameter $\lambda$, filter size at the first/second conv layer $f^{[1]}$ and $f^{[2]}$, dim. of the filter at the second layer and mini-batch size. Instead of using grids at each point, Latin hypercube sampling is used. The distribution of each parameter is offered.Learning rate: log10($\alpha$)~Unif[-4, 0]Regularization parameter:  log10($\lambda$)~Unif[-3,1]Size of the first filter: log2($f^[1]$)~[0,1,2,3,4,5,6,7,8]Size of the first filter: log2($f^[2]$)~[0,1,2,3,4,5,6,7,8]minibatchsize: log2($minibatchSize$)~[0,1,2,3,4,5,6,7,8]<jupyter_code>NumSamples=10
np.random.seed(1)
HyperParameters=lhs(5,samples=NumSamples,criterion="corr")
#print(HyperParameters)
HyperParameters[:,0]=10**(-4+4*HyperParameters[:,0])
HyperParameters[:,1]=10**(-3+4*HyperParameters[:,1])

HyperParameters[:,2]=2**np.round(0+8*HyperParameters[:,2])
HyperParameters[:,3]=2**np.round(0+8*HyperParameters[:,3])
HyperParameters[:,4]=2**np.round(0+8*HyperParameters[:,4])

#print(HyperParameters[:,2])

#plt.hist(HyperParameters[:,3], bins=20)<jupyter_output>new candidate solution found with max,abs corrcoef = 0.93907073340458<jupyter_text>2. TensorFlow Functions<jupyter_code>def create_placeholders(n_H0, n_W0, n_C0, n_y,filtersize1,filtersize2):
    """
    Creates the placeholders for the tensorflow session.
    
    Arguments:
    n_H0 -- scalar, height of an input image
    n_W0 -- scalar, width of an input image
    n_C0 -- scalar, number of channels of the input
    n_y -- scalar, number of classes
        
    Returns:
    X -- placeholder for the data input, of shape [None, n_H0, n_W0, n_C0] and dtype "float"
    Y -- placeholder for the input labels, of shape [None, n_y] and dtype "float"
    """

    ### START CODE HERE ### (≈2 lines)
    X = tf.placeholder(tf.float32,shape=(None, n_H0, n_W0, n_C0))#None
    Y = tf.placeholder(tf.float32,shape=(None,n_y))#None
    ### END CODE HERE ###
    
    return X, Y

def initialize_parameters(filtersize1,filtersize2):
    """
    Initializes weight parameters to build a neural network with tensorflow. The shapes are:
                        W1 : [4, 4, 3, 8]
                        W2 : [2, 2, 8, 16]
    Returns:
    parameters -- a dictionary of tensors containing W1, W2
    """
    
    tf.set_random_seed(1)                              # so that your "random" numbers match ours
    #filtersize1=64; # originally 4
    #filtersize2=32; # originally 2
    NumFilters1=32; #8
    NumFilters2=64; #16
    
        
    ### START CODE HERE ### (approx. 2 lines of code)
    W1 = tf.get_variable("W1", [filtersize1, 1, 1, NumFilters1], initializer = tf.contrib.layers.xavier_initializer(seed = 0))#None
    W2 = tf.get_variable("W2", [filtersize2, 1, NumFilters1, NumFilters2], initializer = tf.contrib.layers.xavier_initializer(seed = 0))#None
    ### END CODE HERE ###

    parameters = {"W1": W1,
                  "W2": W2}
    
    return parameters

def forward_propagation(X, parameters):
    """
    Implements the forward propagation for the model:
    CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FULLYCONNECTED
    
    Arguments:
    X -- input dataset placeholder, of shape (input size, number of examples)
    parameters -- python dictionary containing your parameters "W1", "W2"
                  the shapes are given in initialize_parameters

    Returns:
    Z3 -- the output of the last LINEAR unit
    """
    
    # Retrieve the parameters from the dictionary "parameters" 
    W1 = parameters['W1']
    W2 = parameters['W2']
    
    ### START CODE HERE ###
    # CONV2D: stride of 1, padding 'SAME'
    Z1 = tf.nn.conv2d(X,W1, strides = [1,1,1,1], padding = 'SAME')#None
    # RELU
    A1 = tf.nn.relu(Z1)#None
    # MAXPOOL: window 8x8, sride 8, padding 'SAME'
    P1 = tf.nn.max_pool(A1, ksize = [1,8,1,1], strides = [1,8,1,1], padding = 'SAME')#None
    # CONV2D: filters W2, stride 1, padding 'SAME'
    Z2 = tf.nn.conv2d(P1,W2, strides = [1,1,1,1], padding = 'SAME')#None
    # RELU
    A2 = tf.nn.relu(Z2)
    # MAXPOOL: window 4x4, stride 4, padding 'SAME'
    P2 = tf.nn.max_pool(A2, ksize = [1,4,1,1], strides = [1,4,1,1], padding = 'SAME')#None
    # FLATTEN
    P2 = tf.contrib.layers.flatten(P2)#None
    # FULLY-CONNECTED without non-linear activation function (not not call softmax).
    # 6 neurons in output layer. Hint: one of the arguments should be "activation_fn=None" 
    Z3 = tf.contrib.layers.fully_connected(P2, num_outputs=2,activation_fn=None)
    ### END CODE HERE ###

    return Z3

def compute_cost(Z3, Y,parameters,lambd,m):
    """
    Computes the cost
    
    Arguments:
    Z3 -- output of forward propagation (output of the last LINEAR unit), of shape (6, number of examples)
    Y -- "true" labels vector placeholder, same shape as Z3
    
    Returns:
    cost - Tensor of the cost function
    """
    
    ### START CODE HERE ### (1 line of code)
    W1 = parameters["W1"]
    W2 = parameters["W2"]
    
    #regularizer = tf.add(tf.nn.l2_loss(W1)+tf.nn.l2_loss(W2))  
    cost_cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = Z3, labels = Y))

    
    #L2_reg_cost=(np.sum(np.square(W1))+np.sum(np.square(W2)))*lambd/(m*2)
    regularizer = tf.add(tf.nn.l2_loss(W1),tf.nn.l2_loss(W2))
    # L2_regularization_cost = (np.sum(np.square(W1))+np.sum(np.square(W2))+np.sum(np.square(W3)))*lambd/(m*2)
    #cost=cost_cross_entropy+L2_reg_cost
    cost=tf.reduce_mean(cost_cross_entropy  + lambd * regularizer/m)
    ### END CODE HERE ###

    return cost
def model(X_train, Y_train, X_test, Y_test, learning_rate = 0.009,
          num_epochs = 100, minibatch_size = 10, print_cost = True,filtersize1=32,filtersize2=64,lambd=0):
    """
    Implements a three-layer ConvNet in Tensorflow:
    CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FULLYCONNECTED
    
    Arguments:
    X_train -- training set, of shape (None, 64, 64, 3)
    Y_train -- test set, of shape (None, n_y = 6)
    X_test -- training set, of shape (None, 64, 64, 3)
    Y_test -- test set, of shape (None, n_y = 6)
    learning_rate -- learning rate of the optimization
    num_epochs -- number of epochs of the optimization loop
    minibatch_size -- size of a minibatch
    print_cost -- True to print the cost every 100 epochs
    
    Returns:
    train_accuracy -- real number, accuracy on the train set (X_train)
    test_accuracy -- real number, testing accuracy on the test set (X_test)
    parameters -- parameters learnt by the model. They can then be used to predict.
    """
    
    ops.reset_default_graph()                         # to be able to rerun the model without overwriting tf variables
    tf.set_random_seed(1)                             # to keep results consistent (tensorflow seed)
    seed = 3                                          # to keep results consistent (numpy seed)
    (m, n_H0, n_W0, n_C0) = X_train.shape             
    n_y = Y_train.shape[1]                            
    costs = []                                        # To keep track of the cost
    
    # Create Placeholders of the correct shape
    ### START CODE HERE ### (1 line)
    X, Y = create_placeholders(n_H0, n_W0, n_C0, n_y,filtersize1,filtersize2)#None
    ### END CODE HERE ###

    # Initialize parameters
    ### START CODE HERE ### (1 line)
    parameters = initialize_parameters(filtersize1,filtersize2)#None
    ### END CODE HERE ###
    
    # Forward propagation: Build the forward propagation in the tensorflow graph
    ### START CODE HERE ### (1 line)
    Z3 = forward_propagation(X, parameters)#None
    ### END CODE HERE ###
    
    # Cost function: Add cost function to tensorflow graph
    ### START CODE HERE ### (1 line)

    cost = compute_cost(Z3, Y,parameters,lambd,m)#None
    ### END CODE HERE ###
    
    # Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer that minimizes the cost.
    ### START CODE HERE ### (1 line)
    optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost)#None
    ### END CODE HERE ###
    
    # Initialize all the variables globally
    init = tf.global_variables_initializer()
     
    # Start the session to compute the tensorflow graph
    with tf.Session() as sess:
        
        # Run the initialization
        sess.run(init)
        
        # Do the training loop
        for epoch in range(num_epochs):
          
            minibatch_cost = 0.
            num_minibatches = int(m / minibatch_size) # number of minibatches of size minibatch_size in the train set
            seed = seed + 1
            #print(Y_train.shape)
            minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed)

            for minibatch in minibatches:

                # Select a minibatch
                (minibatch_X, minibatch_Y) = minibatch
                # IMPORTANT: The line that runs the graph on a minibatch.
                # Run the session to execute the optimizer and the cost, the feedict should contain a minibatch for (X,Y).
                ### START CODE HERE ### (1 line)
                _ , temp_cost = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y})#None
                ### END CODE HERE ###
                
                minibatch_cost += temp_cost / num_minibatches
                

            # Print the cost every epoch
            if print_cost == True and epoch % 5 == 0:
                print ("Cost after epoch %i: %f" % (epoch, minibatch_cost))
            if print_cost == True and epoch % 1 == 0:
                costs.append(minibatch_cost)
        
        
        # plot the cost
        plt.plot(np.squeeze(costs))
        plt.ylabel('cost')
        plt.xlabel('iterations (per tens)')
        plt.title("Learning rate =" + str(learning_rate))
        plt.show()

        # Calculate the correct predictions
        predict_op = tf.argmax(Z3, 1)
        correct_prediction = tf.equal(predict_op, tf.argmax(Y, 1))
        
        # Calculate accuracy on the test set
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
        print(accuracy)
        train_accuracy = accuracy.eval({X: X_train, Y: Y_train})
        test_accuracy = accuracy.eval({X: X_test, Y: Y_test})
        print("Train Accuracy:", train_accuracy)
        print("Test Accuracy:", test_accuracy)
                
        return train_accuracy, test_accuracy, parameters
train_accuracy_ref, test_accuracy_ref, parameters = model(X_train, Y_train, X_test, Y_test,filtersize1=32,filtersize2=64,lambd=0)<jupyter_output>WARNING:tensorflow:From <ipython-input-4-7514213748a0>:106: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.
Instructions for updating:

Future major versions of TensorFlow will allow gradients to flow
into the labels input on backprop by default.

See @{tf.nn.softmax_cross_entropy_with_logits_v2}.

Cost after epoch 0: 1.168319
Cost after epoch 5: 0.514833
Cost after epoch 10: 0.514784
Cost after epoch 15: 0.512841
Cost after epoch 20: 0.512329
Cost after epoch 25: 0.513030
Cost after epoch 30: 0.517314
Cost after epoch 35: 0.514884
Cost after epoch 40: 0.517424
Cost after epoch 45: 0.517357
Cost after epoch 50: 0.515569<jupyter_text>Hyperparameters tuning<jupyter_code>Results=[]
for num in range(0,NumSamples):
    alpha_h,lambda_h,filter1_h,filter2_h,minibatch_h=HyperParameters[num]
    print(alpha_h,lambda_h,int(filter1_h),int(filter2_h),int(minibatch_h))
    train_accuracy_h, test_accuracy_h, parameters_h = model(
        X_train, Y_train, X_test, Y_test,learning_rate = alpha_h,filtersize1=int(filter1_h),filtersize2=int(filter2_h),
        lambd=lambda_h,minibatch_size=int(minibatch_h))
    
    Results.append([train_accuracy_h, test_accuracy_h, parameters_h])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12193888607960703, 'fraction_numerical': 0.037708589178646244, 'mean_word_length': 3.1362414102181058, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '184670', 'license': 'MIT', 'n_tokens_mistral': 5053, 'n_tokens_neox': 4739, 'n_words': 1209, 'path': 'OLD/CNN_1D_v2.ipynb', 'repo_name': 'MosGeo/TerraeMotus', 'size': '13843'}