<commit_before>import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops

from models.autoencoder_models import denoising_autoencoder
import model
from utils import utilities


class StackedDenoisingAutoencoder(model.Model):

    """ Implementation of Stacked Denoising Autoencoders using TensorFlow.
    The interface of the class is sklearn-like.
    """

    def __init__(self, layers, model_name='sdae', main_dir='sdae/', enc_act_func=list(['tanh']),
                 dec_act_func=list(['none']), loss_func=list(['mean_squared']), num_epochs=list([10]),
                 batch_size=list([10]), dataset='mnist', xavier_init=list([1]), opt=list(['gradient_descent']),
                 learning_rate=list([0.01]), momentum=list([0.5]),  dropout=1, corr_type='none', corr_frac=0.,
                 verbose=1, seed=-1, softmax_loss_func='cross_entropy', finetune_act_func='relu',
                 finetune_opt='gradient_descent', finetune_learning_rate=0.001, finetune_num_epochs=10,
                 finetune_batch_size=20, do_pretrain=True):
        """
        :param layers: list containing the hidden units for each layer
        :param enc_act_func: Activation function for the encoder. ['sigmoid', 'tanh']
        :param dec_act_func: Activation function for the decoder. ['sigmoid', 'tanh', 'none']
        :param softmax_loss_func: Loss function for the softmax layer. string, default ['cross_entropy', 'mean_squared']
        :param dropout: dropout parameter
        :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001
        :param finetune_act_func: activation function for the finetuning phase
        :param finetune_opt: optimizer for the finetuning phase
        :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20
        :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20
        :param loss_func: Loss function. ['cross_entropy', 'mean_squared']. string, default 'mean_squared'
        :param xavier_init: Value of the constant for xavier weights initialization. int, default 1
        :param opt: Optimizer to use. string, default 'gradient_descent'. ['gradient_descent', 'ada_grad', 'momentum']
        :param learning_rate: Initial learning rate. float, default 0.01
        :param momentum: 'Momentum parameter. float, default 0.9
        :param corr_type: Type of input corruption. string, default 'none'. ["none", "masking", "salt_and_pepper"]
        :param corr_frac: Fraction of the input to corrupt. float, default 0.0
        :param verbose: Level of verbosity. 0 - silent, 1 - print accuracy. int, default 0
        :param num_epochs: Number of epochs. int, default 10
        :param batch_size: Size of each mini-batch. int, default 10
        :param do_pretrain: True: uses variables from pretraining, False: initialize new variables.
        :param dataset: Optional name for the dataset. string, default 'mnist'
        :param seed: positive integer for seeding random generators. Ignored if < 0. int, default -1
        """
        model.Model.__init__(self, model_name, main_dir)

        self.layers = layers
        self.do_pretrain = do_pretrain

        # Autoencoder parameters
        self.enc_act_func = enc_act_func
        self.dec_act_func = dec_act_func
        self.loss_func = loss_func
        self.num_epochs = num_epochs
        self.batch_size = batch_size
        self.xavier_init = xavier_init
        self.opt = opt
        self.learning_rate = learning_rate
        self.momentum = momentum

        # Stacked Autoencoder parameters
        self.corr_type = corr_type
        self.corr_frac = corr_frac
        self.dropout = dropout
        self.softmax_loss_func = softmax_loss_func
        self.finetune_opt = finetune_opt
        self.finetune_learning_rate = finetune_learning_rate
        self.finetune_act_func = finetune_act_func
        self.finetune_num_epochs = finetune_num_epochs
        self.finetune_batch_size = finetune_batch_size
        self.dataset = dataset
        self.verbose = verbose
        self.seed = seed

        if self.seed >= 0:
            np.random.seed(self.seed)
            tf.set_random_seed(self.seed)

        self.input_data = None
        self.input_labels = None
        self.keep_prob = None

        self.encode = None
        self.softmax_out = None

        # Model parameters
        self.encoding_w_ = None  # list of matrices of encoding weights (one per layer)
        self.encoding_b_ = None  # list of arrays of encoding biases (one per layer)

        self.softmax_W = None
        self.softmax_b = None

        # Model traning and evaluation
        self.train_step = None
        self.cost = None

        # tensorflow objects
        self.tf_merged_summaries = None
        self.tf_summary_writer = None
        self.tf_session = None
        self.tf_saver = None

        self.autoencoders = []

        for l, layer in enumerate(layers):
            self.autoencoders.append(denoising_autoencoder.DenoisingAutoencoder(
                n_components=layer, main_dir=self.main_dir,
                enc_act_func=self.enc_act_func[l], dec_act_func=self.dec_act_func[l], loss_func=self.loss_func[l],
                xavier_init=self.xavier_init[l], opt=self.opt[l], learning_rate=self.learning_rate[l],
                momentum=self.momentum[l], corr_type=self.corr_type, corr_frac=self.corr_frac,
                verbose=self.verbose, num_epochs=self.num_epochs[l], batch_size=self.batch_size[l],
                dataset=self.dataset))

    def pretrain(self, train_set, validation_set=None):

        """ Perform unsupervised pretraining of the stack of denoising autoencoders.
        :param train_set: training set
        :param validation_set: validation set
        :return: return data encoded by the last layer
        """

        self.encoding_w_ = []
        self.encoding_b_ = []

        next_train = train_set
        next_valid = validation_set

        for l, autoenc in enumerate(self.autoencoders):
            print('Training layer {}...'.format(l+1))
            next_train, next_valid = self._pretrain_autoencoder_and_gen_feed(autoenc, next_train, next_valid)

            # Reset tensorflow's default graph between different autoencoders
            ops.reset_default_graph()

        return next_train, next_valid

    def _pretrain_autoencoder_and_gen_feed(self, autoenc, train_set, validation_set):

        """ Pretrain a single autoencoder and encode the data for the next layer.
        :param autoenc: autoencoder reference
        :param train_set: training set
        :param validation_set: validation set
        :return: encoded train data, encoded validation data
        """

        autoenc.fit(train_set, validation_set)

        params = autoenc.get_model_parameters()

        self.encoding_w_.append(params['enc_w'])
        self.encoding_b_.append(params['enc_b'])

        next_train = autoenc.transform(train_set)
        next_valid = autoenc.transform(validation_set)

        return next_train, next_valid

    def fit(self, train_set, train_labels, validation_set=None, validation_labels=None):

        """ Fit the model to the data.
        :param train_set: Training data. shape(n_samples, n_features)
        :param train_labels: Labels for the data. shape(n_samples, n_classes)
        :param validation_set: optional, default None. Validation data. shape(nval_samples, n_features)
        :param validation_labels: optional, default None. Labels for the validation data. shape(nval_samples, n_classes)
        :return: self
        """

        print('Starting supervised finetuning...')

        with tf.Session() as self.tf_session:
            self._initialize_tf_utilities_and_ops()
            self._train_model(train_set, train_labels, validation_set, validation_labels)
            self.tf_saver.save(self.tf_session, self.model_path)

    def _initialize_tf_utilities_and_ops(self):
<commit_msg>fixed missing build_model() in stacked_denoising_autoencoder<commit_after>import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops

from models.autoencoder_models import denoising_autoencoder
import model
from utils import utilities


class StackedDenoisingAutoencoder(model.Model):

    """ Implementation of Stacked Denoising Autoencoders using TensorFlow.
    The interface of the class is sklearn-like.
    """

    def __init__(self, layers, model_name='sdae', main_dir='sdae/', enc_act_func=list(['tanh']),
                 dec_act_func=list(['none']), loss_func=list(['mean_squared']), num_epochs=list([10]),
                 batch_size=list([10]), dataset='mnist', xavier_init=list([1]), opt=list(['gradient_descent']),
                 learning_rate=list([0.01]), momentum=list([0.5]),  dropout=1, corr_type='none', corr_frac=0.,
                 verbose=1, seed=-1, softmax_loss_func='cross_entropy', finetune_act_func='relu',
                 finetune_opt='gradient_descent', finetune_learning_rate=0.001, finetune_num_epochs=10,
                 finetune_batch_size=20, do_pretrain=True):
        """
        :param layers: list containing the hidden units for each layer
        :param enc_act_func: Activation function for the encoder. ['sigmoid', 'tanh']
        :param dec_act_func: Activation function for the decoder. ['sigmoid', 'tanh', 'none']
        :param softmax_loss_func: Loss function for the softmax layer. string, default ['cross_entropy', 'mean_squared']
        :param dropout: dropout parameter
        :param finetune_learning_rate: learning rate for the finetuning. float, default 0.001
        :param finetune_act_func: activation function for the finetuning phase
        :param finetune_opt: optimizer for the finetuning phase
        :param finetune_num_epochs: Number of epochs for the finetuning. int, default 20
        :param finetune_batch_size: Size of each mini-batch for the finetuning. int, default 20
        :param loss_func: Loss function. ['cross_entropy', 'mean_squared']. string, default 'mean_squared'
        :param xavier_init: Value of the constant for xavier weights initialization. int, default 1
        :param opt: Optimizer to use. string, default 'gradient_descent'. ['gradient_descent', 'ada_grad', 'momentum']
        :param learning_rate: Initial learning rate. float, default 0.01
        :param momentum: 'Momentum parameter. float, default 0.9
        :param corr_type: Type of input corruption. string, default 'none'. ["none", "masking", "salt_and_pepper"]
        :param corr_frac: Fraction of the input to corrupt. float, default 0.0
        :param verbose: Level of verbosity. 0 - silent, 1 - print accuracy. int, default 0
        :param num_epochs: Number of epochs. int, default 10
        :param batch_size: Size of each mini-batch. int, default 10
        :param do_pretrain: True: uses variables from pretraining, False: initialize new variables.
        :param dataset: Optional name for the dataset. string, default 'mnist'
        :param seed: positive integer for seeding random generators. Ignored if < 0. int, default -1
        """
        model.Model.__init__(self, model_name, main_dir)

        self.layers = layers
        self.do_pretrain = do_pretrain

        # Autoencoder parameters
        self.enc_act_func = enc_act_func
        self.dec_act_func = dec_act_func
        self.loss_func = loss_func
        self.num_epochs = num_epochs
        self.batch_size = batch_size
        self.xavier_init = xavier_init
        self.opt = opt
        self.learning_rate = learning_rate
        self.momentum = momentum

        # Stacked Autoencoder parameters
        self.corr_type = corr_type
        self.corr_frac = corr_frac
        self.dropout = dropout
        self.softmax_loss_func = softmax_loss_func
        self.finetune_opt = finetune_opt
        self.finetune_learning_rate = finetune_learning_rate
        self.finetune_act_func = finetune_act_func
        self.finetune_num_epochs = finetune_num_epochs
        self.finetune_batch_size = finetune_batch_size
        self.dataset = dataset
        self.verbose = verbose
        self.seed = seed

        if self.seed >= 0:
            np.random.seed(self.seed)
            tf.set_random_seed(self.seed)

        self.input_data = None
        self.input_labels = None
        self.keep_prob = None

        self.encode = None
        self.softmax_out = None

        # Model parameters
        self.encoding_w_ = None  # list of matrices of encoding weights (one per layer)
        self.encoding_b_ = None  # list of arrays of encoding biases (one per layer)

        self.softmax_W = None
        self.softmax_b = None

        # Model traning and evaluation
        self.train_step = None
        self.cost = None

        # tensorflow objects
        self.tf_merged_summaries = None
        self.tf_summary_writer = None
        self.tf_session = None
        self.tf_saver = None

        self.autoencoders = []

        for l, layer in enumerate(layers):
            self.autoencoders.append(denoising_autoencoder.DenoisingAutoencoder(
                n_components=layer, main_dir=self.main_dir,
                enc_act_func=self.enc_act_func[l], dec_act_func=self.dec_act_func[l], loss_func=self.loss_func[l],
                xavier_init=self.xavier_init[l], opt=self.opt[l], learning_rate=self.learning_rate[l],
                momentum=self.momentum[l], corr_type=self.corr_type, corr_frac=self.corr_frac,
                verbose=self.verbose, num_epochs=self.num_epochs[l], batch_size=self.batch_size[l],
                dataset=self.dataset))

    def pretrain(self, train_set, validation_set=None):

        """ Perform unsupervised pretraining of the stack of denoising autoencoders.
        :param train_set: training set
        :param validation_set: validation set
        :return: return data encoded by the last layer
        """

        self.encoding_w_ = []
        self.encoding_b_ = []

        next_train = train_set
        next_valid = validation_set

        for l, autoenc in enumerate(self.autoencoders):
            print('Training layer {}...'.format(l+1))
            next_train, next_valid = self._pretrain_autoencoder_and_gen_feed(autoenc, next_train, next_valid)

            # Reset tensorflow's default graph between different autoencoders
            ops.reset_default_graph()

        return next_train, next_valid

    def _pretrain_autoencoder_and_gen_feed(self, autoenc, train_set, validation_set):

        """ Pretrain a single autoencoder and encode the data for the next layer.
        :param autoenc: autoencoder reference
        :param train_set: training set
        :param validation_set: validation set
        :return: encoded train data, encoded validation data
        """

        autoenc.build_model(train_set.shape[1])
        autoenc.fit(train_set, validation_set)

        params = autoenc.get_model_parameters()

        self.encoding_w_.append(params['enc_w'])
        self.encoding_b_.append(params['enc_b'])

        next_train = autoenc.transform(train_set)
        next_valid = autoenc.transform(validation_set)

        return next_train, next_valid

    def fit(self, train_set, train_labels, validation_set=None, validation_labels=None):

        """ Fit the model to the data.
        :param train_set: Training data. shape(n_samples, n_features)
        :param train_labels: Labels for the data. shape(n_samples, n_classes)
        :param validation_set: optional, default None. Validation data. shape(nval_samples, n_features)
        :param validation_labels: optional, default None. Labels for the validation data. shape(nval_samples, n_classes)
        :return: self
        """

        print('Starting supervised finetuning...')

        with tf.Session() as self.tf_session:
            self._initialize_tf_utilities_and_ops()
            self._train_model(train_set, train_labels, validation_set, validation_labels)
            self.tf_saver.save(self.tf_session, self.model_path)

    def _initialize_tf_utilities_and_ops(self):

{'fraction_non_alphanumeric': 0.08436354600236069, 'fraction_numerical': 0.006150214325650743, 'mean_word_length': 2.9072815533980583, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5322907', 'n_tokens_mistral': 4855, 'n_tokens_neox': 4649, 'n_words': 1213}