<commit_before>import os
import sys
from six.moves import cPickle as pickle
import numpy as np
import tensorflow as tf
import neuralNetwork as nn
# import inputData
import argparse
import json
import shutil
import zipfile

# ----------------------------------------------------------------------------- #
#																				#
# 								Useful functions 								#
#																				#
# ----------------------------------------------------------------------------- #

def reformat(dataset, labels, classifier):
    """ Reformat into a shape that's more adapted to the models we're going to train:
        - data as a flat matrix
        - labels as float 1-hot encodings
    """
    dataset = dataset.reshape((-1, classifier.NUM_POINTS * classifier.NUM_FEATURES)).astype(np.float32)
    labels = (np.arange(classifier.NUM_CLASSES) == labels[:, None]).astype(np.float32)
    return dataset, labels
    
def get_inputs(pickle_file, classifier):
    """ Reoad the data generated in picklefiel
    """
    with open(pickle_file, 'rb') as f:
        save = pickle.load(f)
        train_dataset = save['train_dataset']
        train_labels = save['train_labels']
        valid_dataset = save['valid_dataset']
        valid_labels = save['valid_labels']
        test_dataset = save['test_dataset']
        test_labels = save['test_labels']
        del save  # hint to help gc free up memory
        print('Training set', train_dataset.shape, train_labels.shape)
        print('Validation set', valid_dataset.shape, valid_labels.shape)
        print('Test set', test_dataset.shape, test_labels.shape)

        train_dataset, train_labels = reformat(train_dataset, train_labels, classifier)
        valid_dataset, valid_labels = reformat(valid_dataset, valid_labels, classifier)
        test_dataset, test_labels = reformat(test_dataset, test_labels, classifier)
        print("\nTraining set", train_dataset.shape, train_labels.shape)
        print("Validation set", valid_dataset.shape, valid_labels.shape)
        print("Test set", test_dataset.shape, test_labels.shape)

        return train_dataset, train_labels, valid_dataset, valid_labels, test_dataset, test_labels


def run_training(train_dataset, train_labels, valid_dataset, valid_labels, test_dataset, test_labels, saveModelPath, classifier):
    # TODO: Generic number of layers and their number of nodes
    if classifier.NUM_HIDDEN_LAYERS == 1:
        # nb_hidden_nodes_1 = 2048
        nb_hidden_nodes_1 = ( classifier.NUM_POINTS * classifier.NUM_FEATURES + classifier.NUM_CLASSES ) // 2
        nb_hidden_nodes_2 = 0
    elif classifier.NUM_HIDDEN_LAYERS == 2:
        nb_hidden_nodes_1, nb_hidden_nodes_2 = 2048, 2048

    # print("nb_hidden_nodes_1 : " + str(nb_hidden_nodes_1))
    # print("nb_hidden_nodes_2 : " + str(nb_hidden_nodes_2))

    # Construct the graph<commit_msg>ENH: Sizing of the hidden layers following the geometric pyramid rule<commit_after>import os
import sys
from six.moves import cPickle as pickle
import numpy as np
import tensorflow as tf
import neuralNetwork as nn
# import inputData
import argparse
import json
import shutil
import zipfile
import math

# ----------------------------------------------------------------------------- #
#																				#
# 								Useful functions 								#
#																				#
# ----------------------------------------------------------------------------- #

def reformat(dataset, labels, classifier):
    """ Reformat into a shape that's more adapted to the models we're going to train:
        - data as a flat matrix
        - labels as float 1-hot encodings
    """
    dataset = dataset.reshape((-1, classifier.NUM_POINTS * classifier.NUM_FEATURES)).astype(np.float32)
    labels = (np.arange(classifier.NUM_CLASSES) == labels[:, None]).astype(np.float32)
    return dataset, labels
    
def get_inputs(pickle_file, classifier):
    """ Reoad the data generated in picklefiel
    """
    with open(pickle_file, 'rb') as f:
        save = pickle.load(f)
        train_dataset = save['train_dataset']
        train_labels = save['train_labels']
        valid_dataset = save['valid_dataset']
        valid_labels = save['valid_labels']
        test_dataset = save['test_dataset']
        test_labels = save['test_labels']
        del save  # hint to help gc free up memory
        print('Training set', train_dataset.shape, train_labels.shape)
        print('Validation set', valid_dataset.shape, valid_labels.shape)
        print('Test set', test_dataset.shape, test_labels.shape)

        train_dataset, train_labels = reformat(train_dataset, train_labels, classifier)
        valid_dataset, valid_labels = reformat(valid_dataset, valid_labels, classifier)
        test_dataset, test_labels = reformat(test_dataset, test_labels, classifier)
        print("\nTraining set", train_dataset.shape, train_labels.shape)
        print("Validation set", valid_dataset.shape, valid_labels.shape)
        print("Test set", test_dataset.shape, test_labels.shape)

        return train_dataset, train_labels, valid_dataset, valid_labels, test_dataset, test_labels


def run_training(train_dataset, train_labels, valid_dataset, valid_labels, test_dataset, test_labels, saveModelPath, classifier):
    # TODO: Generic number of layers and their number of nodes
    if classifier.NUM_HIDDEN_LAYERS == 1:
        # nb_hidden_nodes_1 = 2048
        # nb_hidden_nodes_1 = ( classifier.NUM_POINTS * classifier.NUM_FEATURES + classifier.NUM_CLASSES ) // 2
        nb_hidden_nodes_1 = int ( math.sqrt ( classifier.NUM_POINTS * classifier.NUM_FEATURES * classifier.NUM_CLASSES ))
        nb_hidden_nodes_2 = 0
    elif classifier.NUM_HIDDEN_LAYERS == 2:
        # nb_hidden_nodes_1, nb_hidden_nodes_2 = 2048, 2048
        r = math.pow( classifier.NUM_POINTS * classifier.NUM_FEATURES / classifier.NUM_CLASSES, 1/3)
        nb_hidden_nodes_1 = int ( classifier.NUM_CLASSES * math.pow ( r, 2 ))
        nb_hidden_nodes_2 =int ( classifier.NUM_POINTS * classifier.NUM_FEATURES * r )

    print("nb_hidden_nodes_1 : " + str(nb_hidden_nodes_1))
    print("nb_hidden_nodes_2 : " + str(nb_hidden_nodes_2))

    # Construct the graph
{'fraction_non_alphanumeric': 0.13562357538261152, 'fraction_numerical': 0.011071312276131553, 'mean_word_length': 2.7872996300863133, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2562272', 'n_tokens_mistral': 1874, 'n_tokens_neox': 1753, 'n_words': 436}