<commit_before>import tensorflow as tf
import numpy as np


class TensorflowCandidateSelector:

    candidate_neighborhood_generator = None
    gold_generator = None
    model = None
    facts = None

    # To be moved to configuration file
    epochs = 20
    batch_size = 30

    def __init__(self, model, candidate_neighborhood_generator):
        self.candidate_neighborhood_generator = candidate_neighborhood_generator
        self.model = model

    def update_setting(self, setting_string, value):
        if setting_string == "epochs":
            self.epochs = int(value)
        elif setting_string == "batch_size":
            self.batch_size = int(value)
        elif setting_string == "facts":
            self.facts = value

        self.model.update_setting(setting_string, value)

    def initialize(self):
        tf.reset_default_graph()
        self.model.initialize()

    def valid_example(self, elements):
        return self.model.validate_example(elements)

    def iterate_in_batches(self, iterator, validate_batches=False):
        batch_dict = {}

        index = 0
        for example in iterator:
            for k,v in example.items():
                if k not in batch_dict:
                    batch_dict[k] = [None]*self.batch_size
                batch_dict[k][index] = v

            index += 1
            if index == self.batch_size:
                yield batch_dict
                index = 0
                batch_dict = {}

        if index != 0:
            yield {k:v[:index] for k,v in batch_dict.items()}

    def train(self, train_file_iterator):
        self.model.prepare_tensorflow_variables(mode='train')
        model_loss = self.model.get_loss_graph()
        parameters_to_optimize = tf.trainable_variables()
        opt_func = tf.train.AdamOptimizer(learning_rate=0.01)
        grad_func = tf.gradients(model_loss, parameters_to_optimize)
        optimize_func = opt_func.apply_gradients(zip(grad_func, parameters_to_optimize))
        init_op = tf.global_variables_initializer()

        self.sess = tf.Session()
        self.sess.run(init_op)

        for epoch in range(self.epochs):
            print("Starting epoch: "+str(epoch))
            epoch_iterator = train_file_iterator.iterate()
            epoch_iterator = self.candidate_neighborhood_generator.enrich(epoch_iterator)

            batch_iterator = self.iterate_in_batches(epoch_iterator)
            for batch in batch_iterator:
                self.model.get_preprocessor().process(batch)

                assignment_dict = self.model.handle_variable_assignment(batch, mode='train')
                result = self.sess.run([optimize_func, model_loss], feed_dict=assignment_dict)
                loss = result[1]
                print(loss)

    def parse_file(self, filename):
        self.batch_size = 1
        candidate_iterator = self.candidate_neighborhood_generator.parse_file(filename)<commit_msg>final change before going home<commit_after>import tensorflow as tf
import numpy as np


class TensorflowCandidateSelector:

    candidate_neighborhood_generator = None
    gold_generator = None
    model = None
    facts = None

    # To be moved to configuration file
    epochs = 20
    batch_size = 30

    def __init__(self, model, candidate_neighborhood_generator):
        self.candidate_neighborhood_generator = candidate_neighborhood_generator
        self.model = model

    def update_setting(self, setting_string, value):
        if setting_string == "epochs":
            self.epochs = int(value)
        elif setting_string == "batch_size":
            self.batch_size = int(value)
        elif setting_string == "facts":
            self.facts = value

        self.model.update_setting(setting_string, value)

    def initialize(self):
        tf.reset_default_graph()
        self.model.initialize()

    def valid_example(self, elements):
        return self.model.validate_example(elements)

    def iterate_in_batches(self, iterator, validate_batches=False):
        batch_dict = {}

        index = 0
        for example in iterator:
            for k,v in example.items():
                if k not in batch_dict:
                    batch_dict[k] = [None]*self.batch_size
                batch_dict[k][index] = v

            index += 1
            if index == self.batch_size:
                yield batch_dict
                index = 0
                batch_dict = {}

        if index != 0:
            yield {k:v[:index] for k,v in batch_dict.items()}

    def train(self, train_file_iterator, verbose=False):
        self.model.prepare_tensorflow_variables(mode='train')
        model_loss = self.model.get_loss_graph()
        parameters_to_optimize = tf.trainable_variables()
        opt_func = tf.train.AdamOptimizer(learning_rate=0.01)
        grad_func = tf.gradients(model_loss, parameters_to_optimize)
        optimize_func = opt_func.apply_gradients(zip(grad_func, parameters_to_optimize))
        init_op = tf.global_variables_initializer()

        self.sess = tf.Session()
        self.sess.run(init_op)

        for epoch in range(self.epochs):
            if verbose:
                print("Starting epoch: "+str(epoch))
            epoch_iterator = train_file_iterator.iterate()
            epoch_iterator = self.candidate_neighborhood_generator.enrich(epoch_iterator)

            batch_iterator = self.iterate_in_batches(epoch_iterator)
            for batch in batch_iterator:
                self.model.get_preprocessor().process(batch)

                assignment_dict = self.model.handle_variable_assignment(batch, mode='train')
                result = self.sess.run([optimize_func, model_loss], feed_dict=assignment_dict)
                loss = result[1]
                if verbose:
                    print(loss)

    def parse_file(self, filename):
        self.batch_size = 1
        candidate_iterator = self.candidate_neighborhood_generator.parse_file(filename)
{'fraction_non_alphanumeric': 0.08319299427416639, 'fraction_numerical': 0.004378578646008757, 'mean_word_length': 2.4014891179839633, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '410614', 'n_tokens_mistral': 1728, 'n_tokens_neox': 1643, 'n_words': 338}