<commit_before>#!/usr/bin/env python
######################
## Script to check  ##
## the score of a   ##
## Pylearn2 pickle. ##
## Based on the     ##
## Holdout notebook.##
######################

import numpy as np
import pylearn2.utils
import pylearn2.config
import theano
import neukrill_net.dense_dataset
import neukrill_net.utils
import sklearn.metrics
import argparse
import os
import pylearn2.config.yaml_parse

def check_score(run_settings_path, verbose=False, augment=1):
    """
    Single function, calculates score, prints and
    returns it.
    """
    # load settings
    if verbose:
        print("Loading settings..")
    settings = neukrill_net.utils.Settings("settings.json")
    run_settings = neukrill_net.utils.load_run_settings(run_settings_path, 
            settings, force=True)
    
    # load the model
    if verbose:
        print("Loading model...")
    model = pylearn2.utils.serial.load(run_settings['pickle abspath'])

    # load the data
    if verbose:
        print("Loading data...")
    # format the YAML
    yaml_string = neukrill_net.utils.format_yaml(run_settings, settings)
    # load proxied objects
    proxied = pylearn2.config.yaml_parse.load(yaml_string, instantiate=False)
    # pull out proxied dataset
    proxdata = proxied.keywords['dataset']
    # force loading of dataset and switch to test dataset
    proxdata.keywords['force'] = True
    proxdata.keywords['training_set_mode'] = 'test'
    proxdata.keywords['verbose'] = verbose
    # then instantiate the dataset
    dataset = pylearn2.config.yaml_parse._instantiate(proxdata)

    # find a good batch size 
    if verbose:
        print("Finding batch size...")
    if hasattr(dataset.X, 'shape'):
        N_examples = dataset.X.shape[0]
    else:
        N_examples = len(dataset.X)
    batch_size = 500
    while N_examples%batch_size != 0:
        batch_size += 1
    n_batches = int(N_examples/batch_size)
    if verbose:
        print("    chosen batch size {0}"
                " for {1} batches".format(batch_size,n_batches))

    # compiling theano forward propagation
    if verbose:
        print("Compiling forward prop...")
    model.set_batch_size(batch_size)
    X = model.get_input_space().make_batch_theano()
    Y = model.fprop(X)
    f = theano.function([X],Y)

    # compute probabilities
    if verbose:
        print("Making predictions...")
    y = np.zeros((N_examples*augment,len(settings.classes)))
    # get the data specs from the cost function using the model
    pcost = proxied.keywords['algorithm'].keywords['cost']
    cost = pylearn2.config.yaml_parse._instantiate(pcost)
    data_specs = cost.get_data_specs(model)
    i = 0 
    for _ in range(augment):
        # make sequential iterator
        iterator = dataset.iterator(batch_size=batch_size,num_batches=n_batches,
                            mode='even_sequential', data_specs=data_specs)
        for batch in iterator:
            if verbose:
                print("    Batch {0} of {1}".format(i+1,n_batches*augment))
            y[i*batch_size:(i+1)*batch_size,:] = f(batch[0])
            i += 1

    # find augmentation factor
    af = run_settings.get("augmentation_factor",1)
    if af > 1:
        if verbose:
            print("Collapsing predictions...")
        y_collapsed = np.zeros((int(N_examples/af), len(settings.classes))) 
        for i,(low,high) in enumerate(zip(range(0,dataset.y.shape[0],af),
                                    range(af,dataset.y.shape[0]+af,af))):
            y_collapsed[i,:] = np.mean(y[low:high,:], axis=0)
        y = y_collapsed
        # and collapse labels
        labels = dataset.y[range(0,dataset.y.shape[0],af)]
    elif augment > 1:
        y_collapsed = np.zeros((N_examples,len(settings.classes)))
        # different kind of augmentation, has to be collapsed differently
        for row in range(N_examples):
            y_collapsed[row,:] = np.mean(np.vstack([r for r in 
                y[[i for i in range(row,N_examples*augment,N_examples)],:]]), 
                axis=0)
        y = y_collapsed            
        labels = dataset.y
    else:
        labels = dataset.y

    # calculate score
    logloss = sklearn.metrics.log_loss(labels,y)
    print("Log loss: {0}".format(logloss))
<commit_msg>changes to support parallel conv. WILL BREAK NORMAL OPERATION<commit_after>#!/usr/bin/env python
######################
## Script to check  ##
## the score of a   ##
## Pylearn2 pickle. ##
## Based on the     ##
## Holdout notebook.##
######################

import numpy as np
import pylearn2.utils
import pylearn2.config
import theano
import neukrill_net.dense_dataset
import neukrill_net.utils
import sklearn.metrics
import argparse
import os
import pylearn2.config.yaml_parse

def check_score(run_settings_path, verbose=False, augment=1):
    """
    Single function, calculates score, prints and
    returns it.
    """
    # load settings
    if verbose:
        print("Loading settings..")
    settings = neukrill_net.utils.Settings("settings.json")
    run_settings = neukrill_net.utils.load_run_settings(run_settings_path, 
            settings, force=True)
    
    # load the model
    if verbose:
        print("Loading model...")
    model = pylearn2.utils.serial.load(run_settings['pickle abspath'])

    # load the data
    if verbose:
        print("Loading data...")
    # format the YAML
    yaml_string = neukrill_net.utils.format_yaml(run_settings, settings)
    # load proxied objects
    proxied = pylearn2.config.yaml_parse.load(yaml_string, instantiate=False)
    # pull out proxied dataset
    proxdata = proxied.keywords['dataset']
    # force loading of dataset and switch to test dataset
    proxdata.keywords['force'] = True
    proxdata.keywords['training_set_mode'] = 'test'
    proxdata.keywords['verbose'] = verbose
    # then instantiate the dataset
    dataset = pylearn2.config.yaml_parse._instantiate(proxdata)

    # find a good batch size 
    if verbose:
        print("Finding batch size...")
    if hasattr(dataset.X, 'shape'):
        N_examples = dataset.X.shape[0]
    else:
        N_examples = len(dataset.X)
    batch_size = 500
    while N_examples%batch_size != 0:
        batch_size += 1
    n_batches = int(N_examples/batch_size)
    if verbose:
        print("    chosen batch size {0}"
                " for {1} batches".format(batch_size,n_batches))

    # compiling theano forward propagation
    if verbose:
        print("Compiling forward prop...")
    model.set_batch_size(batch_size)
    X = model.get_input_space().make_batch_theano()
    Y = model.fprop(X)
    import pdb
    #pdb.set_trace()
    f = theano.function(X,Y)

    # compute probabilities
    if verbose:
        print("Making predictions...")
    y = np.zeros((N_examples*augment,len(settings.classes)))
    # get the data specs from the cost function using the model
    pcost = proxied.keywords['algorithm'].keywords['cost']
    cost = pylearn2.config.yaml_parse._instantiate(pcost)
    data_specs = cost.get_data_specs(model)
    i = 0 
    for _ in range(augment):
        # make sequential iterator
        iterator = dataset.iterator(batch_size=batch_size,num_batches=n_batches,
                            mode='even_sequential', data_specs=data_specs)
        for batch in iterator:
            if verbose:
                print("    Batch {0} of {1}".format(i+1,n_batches*augment))
            y[i*batch_size:(i+1)*batch_size,:] = f(batch[0],batch[1])
            i += 1

    # find augmentation factor
    af = run_settings.get("augmentation_factor",1)
    if af > 1:
        if verbose:
            print("Collapsing predictions...")
        y_collapsed = np.zeros((int(N_examples/af), len(settings.classes))) 
        for i,(low,high) in enumerate(zip(range(0,dataset.y.shape[0],af),
                                    range(af,dataset.y.shape[0]+af,af))):
            y_collapsed[i,:] = np.mean(y[low:high,:], axis=0)
        y = y_collapsed
        # and collapse labels
        labels = dataset.y[range(0,dataset.y.shape[0],af)]
    elif augment > 1:
        y_collapsed = np.zeros((N_examples,len(settings.classes)))
        # different kind of augmentation, has to be collapsed differently
        for row in range(N_examples):
            y_collapsed[row,:] = np.mean(np.vstack([r for r in 
                y[[i for i in range(row,N_examples*augment,N_examples)],:]]), 
                axis=0)
        y = y_collapsed            
        labels = dataset.y
    else:
        labels = dataset.y

    # calculate score
    logloss = sklearn.metrics.log_loss(labels,y)
    print("Log loss: {0}".format(logloss))

{'fraction_non_alphanumeric': 0.12324474875246605, 'fraction_numerical': 0.008239526517349426, 'mean_word_length': 2.733968804159445, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1957263', 'n_tokens_mistral': 2837, 'n_tokens_neox': 2651, 'n_words': 603}