<commit_before>import numpy as np
import inDianajonES as InD
import tensorflow as tf


'''
ConvNNet implements a convolutional neural network 
using the TensorFlow framework. It consists of a 
ConvNNet class, which contains several functions:
    
    - Train inputs a list of images and artifacts,
        builds the design matrix, and implements
        the neural net (outputting the training)
        error as it goes
    
    - Test runs the neural net on a test data set.
        Can do whatever we make it do.
    
    - Save_model saves the session to the input filename.
    
    - Resume_from loads a saved session.
    
Also @Joe if you are wondering why I used a class, its 
because the class allowed the session to be saved as a 
global variable without it being a script.
'''

# ============================================================

class ConvNNet(object):
    '''
    ConvNNet implements a convolutional neural network 
    using the TensorFlow framework.
    '''
    def __init__(self):
        
        self.runs = None
        self.expids = None
        self.artifacts = None
        self.Nsteps = None
        
        return
    
    def Train(self, runs , expids , artifacts , Nsteps , gridsize=128 , cgfactor=8 , load_data=True):
        '''
        This function creates the design matrix and loads the
        true clasifications (if they don't already exist).  
        It then runs the neural net to train the optimal 
        predicting scheme.
        
        * Currently the neural net is very similar to the
        one used in the MNIST tutorial from Tensorflow (except
        modified to use our images etc.).  We 
        should modify it further to fit our needs *
        
        Function inputs are below:
    
        - runs is the run info of the training data
    
        - expids are the exposure ids of the training data
    
        - artifacts is list of artifact objects (preprocessed)
    
        - Nsteps is number of training steps to run
    
        - gridsize is size (in pixels) of the postage stamps.
        * Note:  must be evenly divisible into 2048 *
        
        - cgfactor is coarse-graining factor.
        * Note:  must be evenly divisibile into gridsize *
        '''
        
        if load_data is True: # if false, must have X, ey, and ey2 loaded in mem.
            assert 2048 % self.gridsize == 0
            self.gridsize = gridsize
            assert (self.gridsize//self.cgfactor)%4 == 0
            self.cgfactor = cgfactor
        

            # load/make training data:  @Joe: edit as needed.  In particular
            # we want X to have shape [Nexamples, Npixels], and y to 
            # have shape [Nexamples , Ncategories] and be all zeros 
            # along the second axis (except for the correct classification),
            # which is a 1.  This is what ey2 is.
            imagenames, bkgnames = get_training_filenames(runs,expids)
            X , Y = create_design_matrix(imagenames , bkgnames , gridsize , cgfactor)
            ey = enumerate_labels(Y)
     
            ey2 = np.zeros([len(ey),int(np.max(ey))+1],float)
            for i in range(len(ey)):
                ey2[i,ey[i]] = 1.0
    
    
        # Ncategories is number of classifications
        Ncategories = int(np.max(ey))+1
        Nexamples = len(ey)<commit_msg>Artifacts now passed to create_design_matrix function.  Fixes bug.<commit_after>import numpy as np
import inDianajonES as InD
import tensorflow as tf


'''
ConvNNet implements a convolutional neural network 
using the TensorFlow framework. It consists of a 
ConvNNet class, which contains several functions:
    
    - Train inputs a list of images and artifacts,
        builds the design matrix, and implements
        the neural net (outputting the training)
        error as it goes
    
    - Test runs the neural net on a test data set.
        Can do whatever we make it do.
    
    - Save_model saves the session to the input filename.
    
    - Resume_from loads a saved session.
    
Also @Joe if you are wondering why I used a class, its 
because the class allowed the session to be saved as a 
global variable without it being a script.
'''

# ============================================================

class ConvNNet(object):
    '''
    ConvNNet implements a convolutional neural network 
    using the TensorFlow framework.
    '''
    def __init__(self):
        
        self.runs = None
        self.expids = None
        self.artifacts = None
        self.Nsteps = None
        
        return
    
    def Train(self, runs , expids , artifacts , Nsteps , gridsize=128 , cgfactor=8 , load_data=True):
        '''
        This function creates the design matrix and loads the
        true clasifications (if they don't already exist).  
        It then runs the neural net to train the optimal 
        predicting scheme.
        
        * Currently the neural net is very similar to the
        one used in the MNIST tutorial from Tensorflow (except
        modified to use our images etc.).  We 
        should modify it further to fit our needs *
        
        Function inputs are below:
    
        - runs is the run info of the training data
    
        - expids are the exposure ids of the training data
    
        - artifacts is list of artifact objects (preprocessed)
    
        - Nsteps is number of training steps to run
    
        - gridsize is size (in pixels) of the postage stamps.
        * Note:  must be evenly divisible into 2048 *
        
        - cgfactor is coarse-graining factor.
        * Note:  must be evenly divisibile into gridsize *
        '''
        
        if load_data is True: # if false, must have X, ey, and ey2 loaded in mem.
            assert 2048 % self.gridsize == 0
            self.gridsize = gridsize
            assert (self.gridsize//self.cgfactor)%4 == 0
            self.cgfactor = cgfactor
        

            # load/make training data:  @Joe: edit as needed.  In particular
            # we want X to have shape [Nexamples, Npixels], and y to 
            # have shape [Nexamples , Ncategories] and be all zeros 
            # along the second axis (except for the correct classification),
            # which is a 1.  This is what ey2 is.
            imagenames, bkgnames = get_training_filenames(runs,expids)
            X , Y = create_design_matrix(imagenames , bkgnames , artifacts , gridsize , cgfactor)
            ey = enumerate_labels(Y)
     
            ey2 = np.zeros([len(ey),int(np.max(ey))+1],float)
            for i in range(len(ey)):
                ey2[i,ey[i]] = 1.0
    
    
        # Ncategories is number of classifications
        Ncategories = int(np.max(ey))+1
        Nexamples = len(ey)
{'fraction_non_alphanumeric': 0.07694588584136397, 'fraction_numerical': 0.007116382505559674, 'mean_word_length': 2.021047917599642, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2296301', 'n_tokens_mistral': 1867, 'n_tokens_neox': 1780, 'n_words': 753}