<commit_before>import time 
from collections import OrderedDict

import imp
try:
    imp.find_module('progressbar')
    progressbar_installed = True
except ImportError:
    progressbar_installed = False
if progressbar_installed is True:
    import progressbar
    
try:
    imp.find_module('networkx')
    nx_installed = True
except ImportError:
    nx_installed = False
if nx_installed is True:
    import networkx as nx


import numpy
import theano
import theano.tensor as T 

import modules as M
import layers as L

class network(object):
    """
    TODO:

        * Origin to be taken from another network outside of this one.
        * Need to expand beyond just ``classifier`` type networks.
        
    Class definition for the class :mod:`network`:

    All network properties, network variables and functionalities are initialized using this 
    class and are contained by the network object. The ``network.__init__`` method initializes
    the network class. The ``network.__init___`` function has many purposes depending on the 
    arguments supplied. 
            
    Provide any or all of the following arguments. Appropriate errors will be thrown if the 
    parameters are not supplied correctly.

    Args:
        verbose             : Similar to any 3-level verbose in the toolbox. 
        type                : option takes only 'classifier' for now. Will add 'encoders'
                              and others later
        resultor            : ``resultor`` is a dictionary of the form:

            .. code-block:: none

                resultor    =    { 
                        "root"     : "<root directory to save stuff inside>"
                        "results"  : "<results_file_name>.txt",      
                        "errors"   : "<error_file_name>.txt",
                        "costs"    : "<cost_file_name>.txt",
                        "confusion": "<confusion_file_name>.txt",
                        "network"  : "<network_save_file_name>.pkl"
                        "id"       : id of the resultor
                                }  
                                            
            While the filenames are optional, ``root`` must be provided. If a particular file 
            is not provided, that value will not be saved. This value is supplied to setup the
            resultor module of :mod: `network`.

        visualizer           : ``visualizer`` is a dictionary of the form:
         
            .. code-block:: none

                visualizer = {
                        "root"        : location to save the visualizations
                        "frequency"   : <integer>, after how many epochs do you need to 
                                        visualize. Default value is 1import os

                        "sample_size" : <integer, prefer squares>, simply save down random 
                                        images from the datasets also saves down activations 
                                        for the same images also. Default value is 16
                        "rgb_filters" : <bool> flag. if True 3D-RGB CNN filters are rendered. 
                                        Default value is False
                        "id"          : id of the visualizer
                                }  

        optimizer         : ``optimizer`` is a dictionary of the form: 

            .. code-block:: none

                optimizer =  {        
                        "momentum_type"       : <option> takes 'false' <no momentum>, 'polyak'
                                                and 'nesterov'. Default value is 'polyak'
                        "momentum_params"   : (<value in [0,1]>, <value in [0,1]>, <int>),
                                                (momentum coeffient at start, at end, at what
                                                epoch to end momentum increase). Default is 
                                                the tuple (0.5, 0.95,50)                                                           
                        "learning_rate"   : (initial_learning_rate, fine_tuning_learning_rate, 
                                                annealing_decay_rate). Default is the tuple 
                                                (0.1,0.001,0.005)
                        "regularization"    : (l1_coeff, l2_coeff). Default is (0.001, 0.001)                
                        "optimizer_type": <option>, takes 'sgd', 'adagrad', 'rmsprop', 'adam'.
                                                Default is 'rmsprop'
                        "objective_function": <option>,  takes  
                                                'nll'-negative log likelihood,
                                                'cce'-categorical cross entropy,
                                                'bce'-binary cross entropy.    
                                                Default is 'nll'
                        "id"                : id of the optimizer
                            }         

        datastream          : ``datastream`` is a dictionary of the form:

            .. code-block:: python

                dataset_init_args = {
                            "dataset":  <location>
                            "svm"    :  False or True 
                                 ``svm`` if ``True``, a one-hot label set will also be setup.
                            "n_classes": <int>
                                ``n_classes`` if ``svm`` is ``True``, we need to know how 
                                 many ``n_classes`` are present.
                            "id": id of the datastream
                    }

        borrow: Check ``theano's`` borrow. Default is ``True``.
        
    Returns:            
        ``yann.network.network``: network object with parameters setup.     
    """  

    def __init__(  self, verbose = 2, **kwargs ):
        """ Refer to the definition in the class docstrings for details """

        if verbose >= 1:
            print ". Initializing the network"

        if nx_installed is True:
            self.graph = nx.DiGraph()
        else:
            self.graph = None  

        self.layers = {} # create an empty dictionary that we can populate later.
        self.active_params = []
        self.params = []
        self.dropout_layers = {} # These are just weights dropped out. Contains references
        self.num_layers = 0  # just maintain for the sake of it. 

        # All these are just bookkeeping variables.
        self.last_layer_created = None
        self.last_resultor_created = None
        self.last_datastream_created = None
        self.last_visualizer_created = None
        self.last_optimizer_created = None   
        self.last_objective_layer_created = None
        self.last_classifier_layer_created = None
        self.layer_activities_created = True        

        # create empty dictionary of modules. 
        self.visualizer = {}
        self.optimizer  = {}
        self.resultor   = {}
        self.datastream = {}
        self.num_layers = 0
        self.type = 'classifier'   # default value assignment
        self.rng = numpy.random    # each network will have its own unique randomizer
        self.borrow = True
        self.L1 = 0
        self.L2 = 0
        self.layer_activities = {}

        # for each argument supplied by kwargs, intialize something.
        for argument, value in kwargs.iteritems():

            if (argument == 'resultor' or argument == 'optimizer' or  
                argument == 'datastream' or argument == 'visualizer'):
                # add module
                self.add_module(type = argument, params = value, verbose = verbose)                              

            if argument == 'type':
                self.type = value

            if argument == 'borrow':
                self.borrow = value
    
    def layer_activity(self, id, index=0, verbose = 2):
        """
        Use this function to visualize or print out the outputs of each layer. 
        I don't know why this might be useful, but its fun to check this out I guess. This will only
        work after the dataset is initialized.

        Args:
            id: id of the layer that you want to visualize the output for.
            index: Which batch of data should I use for producing the outputs.
                    Default is ``0``                
        """
        return self.layer_activities[id](index)

    def add_layer(self, type, verbose = 2, **kwargs):
        """
        TODO:
            Need to add the following:
            * Merge Layer
                - Concatenate
                - Embed
                - Add, max, mean
            * Inception Layer.
            * LSTM layer.
            * MaskedConvPool Layer.
            * ...
            * when ``type`` is ``'objective'``, I need to allow taking a loss between two layers to
              be propagated. Right now ``origin`` has to be a classifier layer only. This needs to 
              change to be able to implement generality and mentor networks.
            * basically objective has to be a flag for an error function if origin is a tuple.
              

        Args: 
            type: <string> options include 
                  'input' or 'data' - which indicates an input layer.
                  'conv_pool' or 'fully_connected' - indicates a convolutional - pooling layer
                  'dot_product' or 'hidden' or 'mlp' or 'fully_connected' - indicates a hidden fully
                  connected layer
                  'classifier' or 'softmax' or 'output' or 'label' - indicates a classifier layer
                  'objective' or 'loss' - a layer that creates a loss function
                  From now on everything is optional args.. 
            id: <string> how to identify the layer by.
                Default is just layer number that starts with ``0``.
            origin: ``id`` will use the output of that layer as input to the new layer.
                     Default is the last layer created. This variable is not needed for 
                     ``input`` type of layers.                     
            verbose: similar to the rest of the toolbox.
            mean_subtract: if ``True`` we will subtract the mean from each image, else not.
            num_neurons: number of neurons in the layer
            dataset: <string> Location to the dataset.
                     used when layer ``type`` is ``input``.
            activation: String, takes options that are listed in :mod:`activations` Needed for
                        layers that use activations.
                        Some activations also take support parameters, for instance ``maxout``
                        takes maxout type and size, ``softmax`` takes an option temperature.
                        Refer to the module :mod:`activations` to know more.
            stride: tuple ``(int , int)``. Used as convolution stride. Default ``(1,1)``
            batch_norm: If provided will be used, default is ``False``. 
            border_mode: Refer to ``border_mode`` variable in ``yann.core.conv``, module 
                         :mod:`conv`   
            pool_size: Subsample size, default is ``(1,1)``.
            pool_type: Refer to :mod:`pool` for details. {'max', 'sum', 'mean', 'max_same_size'}
            learnable: Default is ``True``, if ``True`` we backprop on that layer. If ``False``
                       Layer is obstinate.
            input_params: Supply params or initializations from a pre-trained system.
            dropout_rate: If you want to dropout this layer's output provide the output.  
            regularize: ``True`` is you want to apply regularization, ``False`` if not.  
            num_classes: ``int`` number of classes to classify.     
            objective:  objective provided by classifier               
                        ``nll``-negative log likelihood, 
                        ``cce``-categorical cross entropy, 
                        ``bce``-binary cross entropy,
                        ``hinge``-hinge loss . For classifier layer.  
            dataset_init_args: same as for the dataset module. In fact this argument is needed 
                                only when dataset module is not setup. 
            datastream_id: When using input layer or during objective layer, use this to identify
                           which datastream to take data from. 
            regularizer: Default is ``(0.001, 0.001)`` coeffients for L1, L2 regulaizer 
                            coefficients. 

        """
        if not 'id' in kwargs.keys():
            id = str(self.num_layers)
        else:
            id = kwargs["id"]

        # increase the layer counter
        self.num_layers = self.num_layers + 1

        if verbose >= 2 :
            print ".. Adding "+ type + " layer " + id
        
        if not 'learnable' in kwargs.keys():
            if type == 'input' or \
               type == 'data' or \
               type == 'objective' or  \
               type == 'loss':            
                if verbose >= 3:
                    print "... Making learnable False as it is not provided"
                learnable = False
            else:
                if verbose >= 3:
                    print "... Making learnable True as it is not provided"
                learnable = True                

        if type == 'input' or \
           type == 'data':
            if not learnable == False:
                raise Exception (" You cannot learn this type of a layer")           
            self._add_input_layer(id =id, options = kwargs, verbose = verbose)

        elif type == 'conv_pool' or \
             type == 'convolution':
            self._add_conv_layer(id = id, options = kwargs, verbose = verbose)
            self.params = self.params + self.dropout_layers[id].params
  
        elif type == 'dot_product' or \
             type == 'hidden' or  \
             type == 'mlp' or  \
             type == 'fully_connected':          
            self._add_dot_product_layer(id =id, options = kwargs, verbose = verbose)
            self.params = self.params + self.dropout_layers[id].params

        elif type == 'classifier' or  \
             type ==  'softmax' or \
             type ==  'output' or \
             type == 'label':
            self._add_classifier_layer(id =id, options = kwargs, verbose = verbose)
            self.last_classifier_layer_created = id
            self.params = self.params + self.dropout_layers[id].params

        elif type == 'objective' or  \
             type == 'loss': 
            self._add_objective_layer(id =id, options = kwargs, verbose = verbose)
            self.last_objective_layer_created = id

        else:
            raise Exception('No layer called ' + type + ' exists in yann')

        if learnable is True:
            self.active_params = self.active_params + self.dropout_layers[id].params 
            # .. Note :: using append is troublesome here.

        self.last_layer_created = id

        if not self.graph is None:
            attributes = sef.layers[id]._graph_attributes()
            self.graph.add_node(id,attributes)
            for origin in attributes["origin"]:
                if not origin in self.graph.nodes():
                    self.graph.add_node(origin)
                self.graph.add_edge(origin,id)

        <commit_msg>bug fixes with graphing<commit_after>import time 
from collections import OrderedDict

import imp
try:
    imp.find_module('progressbar')
    progressbar_installed = True
except ImportError:
    progressbar_installed = False
if progressbar_installed is True:
    import progressbar
    
try:
    imp.find_module('networkx')
    nx_installed = True
except ImportError:
    nx_installed = False
if nx_installed is True:
    import networkx as nx


import numpy
import theano
import theano.tensor as T 

import modules as M
import layers as L

class network(object):
    """
    TODO:

        * Origin to be taken from another network outside of this one.
        * Need to expand beyond just ``classifier`` type networks.
        
    Class definition for the class :mod:`network`:

    All network properties, network variables and functionalities are initialized using this 
    class and are contained by the network object. The ``network.__init__`` method initializes
    the network class. The ``network.__init___`` function has many purposes depending on the 
    arguments supplied. 
            
    Provide any or all of the following arguments. Appropriate errors will be thrown if the 
    parameters are not supplied correctly.

    Args:
        verbose             : Similar to any 3-level verbose in the toolbox. 
        type                : option takes only 'classifier' for now. Will add 'encoders'
                              and others later
        resultor            : ``resultor`` is a dictionary of the form:

            .. code-block:: none

                resultor    =    { 
                        "root"     : "<root directory to save stuff inside>"
                        "results"  : "<results_file_name>.txt",      
                        "errors"   : "<error_file_name>.txt",
                        "costs"    : "<cost_file_name>.txt",
                        "confusion": "<confusion_file_name>.txt",
                        "network"  : "<network_save_file_name>.pkl"
                        "id"       : id of the resultor
                                }  
                                            
            While the filenames are optional, ``root`` must be provided. If a particular file 
            is not provided, that value will not be saved. This value is supplied to setup the
            resultor module of :mod: `network`.

        visualizer           : ``visualizer`` is a dictionary of the form:
         
            .. code-block:: none

                visualizer = {
                        "root"        : location to save the visualizations
                        "frequency"   : <integer>, after how many epochs do you need to 
                                        visualize. Default value is 1import os

                        "sample_size" : <integer, prefer squares>, simply save down random 
                                        images from the datasets also saves down activations 
                                        for the same images also. Default value is 16
                        "rgb_filters" : <bool> flag. if True 3D-RGB CNN filters are rendered. 
                                        Default value is False
                        "id"          : id of the visualizer
                                }  

        optimizer         : ``optimizer`` is a dictionary of the form: 

            .. code-block:: none

                optimizer =  {        
                        "momentum_type"       : <option> takes 'false' <no momentum>, 'polyak'
                                                and 'nesterov'. Default value is 'polyak'
                        "momentum_params"   : (<value in [0,1]>, <value in [0,1]>, <int>),
                                                (momentum coeffient at start, at end, at what
                                                epoch to end momentum increase). Default is 
                                                the tuple (0.5, 0.95,50)                                                           
                        "learning_rate"   : (initial_learning_rate, fine_tuning_learning_rate, 
                                                annealing_decay_rate). Default is the tuple 
                                                (0.1,0.001,0.005)
                        "regularization"    : (l1_coeff, l2_coeff). Default is (0.001, 0.001)                
                        "optimizer_type": <option>, takes 'sgd', 'adagrad', 'rmsprop', 'adam'.
                                                Default is 'rmsprop'
                        "objective_function": <option>,  takes  
                                                'nll'-negative log likelihood,
                                                'cce'-categorical cross entropy,
                                                'bce'-binary cross entropy.    
                                                Default is 'nll'
                        "id"                : id of the optimizer
                            }         

        datastream          : ``datastream`` is a dictionary of the form:

            .. code-block:: python

                dataset_init_args = {
                            "dataset":  <location>
                            "svm"    :  False or True 
                                 ``svm`` if ``True``, a one-hot label set will also be setup.
                            "n_classes": <int>
                                ``n_classes`` if ``svm`` is ``True``, we need to know how 
                                 many ``n_classes`` are present.
                            "id": id of the datastream
                    }

        borrow: Check ``theano's`` borrow. Default is ``True``.
        
    Returns:            
        ``yann.network.network``: network object with parameters setup.     
    """  

    def __init__(  self, verbose = 2, **kwargs ):
        """ Refer to the definition in the class docstrings for details """

        if verbose >= 1:
            print ". Initializing the network"

        if nx_installed is True:
            self.graph = nx.DiGraph()
        else:
            self.graph = None  

        self.layers = {} # create an empty dictionary that we can populate later.
        self.active_params = []
        self.params = []
        self.dropout_layers = {} # These are just weights dropped out. Contains references
        self.num_layers = 0  # just maintain for the sake of it. 

        # All these are just bookkeeping variables.
        self.last_layer_created = None
        self.last_resultor_created = None
        self.last_datastream_created = None
        self.last_visualizer_created = None
        self.last_optimizer_created = None   
        self.last_objective_layer_created = None
        self.last_classifier_layer_created = None
        self.layer_activities_created = True        

        # create empty dictionary of modules. 
        self.visualizer = {}
        self.optimizer  = {}
        self.resultor   = {}
        self.datastream = {}
        self.num_layers = 0
        self.type = 'classifier'   # default value assignment
        self.rng = numpy.random    # each network will have its own unique randomizer
        self.borrow = True
        self.L1 = 0
        self.L2 = 0
        self.layer_activities = {}

        # for each argument supplied by kwargs, intialize something.
        for argument, value in kwargs.iteritems():

            if (argument == 'resultor' or argument == 'optimizer' or  
                argument == 'datastream' or argument == 'visualizer'):
                # add module
                self.add_module(type = argument, params = value, verbose = verbose)                              

            if argument == 'type':
                self.type = value

            if argument == 'borrow':
                self.borrow = value
    
    def layer_activity(self, id, index=0, verbose = 2):
        """
        Use this function to visualize or print out the outputs of each layer. 
        I don't know why this might be useful, but its fun to check this out I guess. This will only
        work after the dataset is initialized.

        Args:
            id: id of the layer that you want to visualize the output for.
            index: Which batch of data should I use for producing the outputs.
                    Default is ``0``                
        """
        return self.layer_activities[id](index)

    def add_layer(self, type, verbose = 2, **kwargs):
        """
        TODO:
            Need to add the following:
            * Merge Layer
                - Concatenate
                - Embed
                - Add, max, mean
            * Inception Layer.
            * LSTM layer.
            * MaskedConvPool Layer.
            * ...
            * when ``type`` is ``'objective'``, I need to allow taking a loss between two layers to
              be propagated. Right now ``origin`` has to be a classifier layer only. This needs to 
              change to be able to implement generality and mentor networks.
            * basically objective has to be a flag for an error function if origin is a tuple.
              

        Args: 
            type: <string> options include 
                  'input' or 'data' - which indicates an input layer.
                  'conv_pool' or 'fully_connected' - indicates a convolutional - pooling layer
                  'dot_product' or 'hidden' or 'mlp' or 'fully_connected' - indicates a hidden fully
                  connected layer
                  'classifier' or 'softmax' or 'output' or 'label' - indicates a classifier layer
                  'objective' or 'loss' - a layer that creates a loss function
                  From now on everything is optional args.. 
            id: <string> how to identify the layer by.
                Default is just layer number that starts with ``0``.
            origin: ``id`` will use the output of that layer as input to the new layer.
                     Default is the last layer created. This variable is not needed for 
                     ``input`` type of layers.                     
            verbose: similar to the rest of the toolbox.
            mean_subtract: if ``True`` we will subtract the mean from each image, else not.
            num_neurons: number of neurons in the layer
            dataset: <string> Location to the dataset.
                     used when layer ``type`` is ``input``.
            activation: String, takes options that are listed in :mod:`activations` Needed for
                        layers that use activations.
                        Some activations also take support parameters, for instance ``maxout``
                        takes maxout type and size, ``softmax`` takes an option temperature.
                        Refer to the module :mod:`activations` to know more.
            stride: tuple ``(int , int)``. Used as convolution stride. Default ``(1,1)``
            batch_norm: If provided will be used, default is ``False``. 
            border_mode: Refer to ``border_mode`` variable in ``yann.core.conv``, module 
                         :mod:`conv`   
            pool_size: Subsample size, default is ``(1,1)``.
            pool_type: Refer to :mod:`pool` for details. {'max', 'sum', 'mean', 'max_same_size'}
            learnable: Default is ``True``, if ``True`` we backprop on that layer. If ``False``
                       Layer is obstinate.
            input_params: Supply params or initializations from a pre-trained system.
            dropout_rate: If you want to dropout this layer's output provide the output.  
            regularize: ``True`` is you want to apply regularization, ``False`` if not.  
            num_classes: ``int`` number of classes to classify.     
            objective:  objective provided by classifier               
                        ``nll``-negative log likelihood, 
                        ``cce``-categorical cross entropy, 
                        ``bce``-binary cross entropy,
                        ``hinge``-hinge loss . For classifier layer.  
            dataset_init_args: same as for the dataset module. In fact this argument is needed 
                                only when dataset module is not setup. 
            datastream_id: When using input layer or during objective layer, use this to identify
                           which datastream to take data from. 
            regularizer: Default is ``(0.001, 0.001)`` coeffients for L1, L2 regulaizer 
                            coefficients. 

        """
        if not 'id' in kwargs.keys():
            id = str(self.num_layers)
        else:
            id = kwargs["id"]

        # increase the layer counter
        self.num_layers = self.num_layers + 1

        if verbose >= 2 :
            print ".. Adding "+ type + " layer " + id
        
        if not 'learnable' in kwargs.keys():
            if type == 'input' or \
               type == 'data' or \
               type == 'objective' or  \
               type == 'loss':            
                if verbose >= 3:
                    print "... Making learnable False as it is not provided"
                learnable = False
            else:
                if verbose >= 3:
                    print "... Making learnable True as it is not provided"
                learnable = True                

        if type == 'input' or \
           type == 'data':
            if not learnable == False:
                raise Exception (" You cannot learn this type of a layer")           
            self._add_input_layer(id =id, options = kwargs, verbose = verbose)

        elif type == 'conv_pool' or \
             type == 'convolution':
            self._add_conv_layer(id = id, options = kwargs, verbose = verbose)
            self.params = self.params + self.dropout_layers[id].params
  
        elif type == 'dot_product' or \
             type == 'hidden' or  \
             type == 'mlp' or  \
             type == 'fully_connected':          
            self._add_dot_product_layer(id =id, options = kwargs, verbose = verbose)
            self.params = self.params + self.dropout_layers[id].params

        elif type == 'classifier' or  \
             type ==  'softmax' or \
             type ==  'output' or \
             type == 'label':
            self._add_classifier_layer(id =id, options = kwargs, verbose = verbose)
            self.last_classifier_layer_created = id
            self.params = self.params + self.dropout_layers[id].params

        elif type == 'objective' or  \
             type == 'loss': 
            self._add_objective_layer(id =id, options = kwargs, verbose = verbose)
            self.last_objective_layer_created = id

        else:
            raise Exception('No layer called ' + type + ' exists in yann')

        if learnable is True:
            self.active_params = self.active_params + self.dropout_layers[id].params 
            # .. Note :: using append is troublesome here.

        self.last_layer_created = id

        if not self.graph is None:
            attributes = self.layers[id]._graph_attributes()
            self.graph.add_node(id,attributes)
            for origin in attributes["origin"]:
                if not origin in self.graph.nodes():
                    self.graph.add_node(origin)
                self.graph.add_edge(origin,id)

        
{'fraction_non_alphanumeric': 0.07752790810548454, 'fraction_numerical': 0.004335867982527099, 'mean_word_length': 1.3759225092250922, 'pattern_counts': {'":': 12, '<': 45, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 72, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1725824', 'n_tokens_mistral': 7595, 'n_tokens_neox': 7111, 'n_words': 2717}