<commit_before>'''
Created on 16 Jul 2009

@author: charanpal
'''
from apgl.util.Util import Util
from apgl.util.Evaluator import Evaluator 
from apgl.util.Parameter import Parameter
from apgl.util.Sampling import Sampling
import numpy
import logging
import gc
import itertools 
import multiprocessing 
from sklearn import linear_model 


#Start with some functions used for multiprocessing 

def computeTestError(args):
    """
    Used in conjunction with the parallel model selection. Trains and then tests
    on a seperate test set. 
    """
    (trainX, trainY, testX, testY, learner) = args
    learner.learnModel(trainX, trainY)
    predY = learner.predict(testX)

    return learner.getMetricMethod()(testY, predY)

def computeBootstrapError(args):
    """
    Used in conjunction with the parallel model selection. Trains and then tests
    on a seperate test set and evaluated the bootstrap error. 
    """
    (trainX, trainY, testX, testY, learner) = args
    learner.learnModel(trainX, trainY)
    predTestY = learner.predict(testX)
    predTrainY = learner.predict(trainX)
    weight = 0.632
    return Evaluator.binaryBootstrapError(predTestY, testY, predTrainY, trainY, weight)

def computeTrainError(args): 
    """
    Train on a set of examples of test on the same examples. 
    """
    (X, y, learner) = args
    learner.learnModel(X, y)
    predY = learner.predict(X)
    return learner.getMetricMethod()(predY, y)

def computeVFPen(args): 
    """
    Compute the penVF criteria for a single fold 
    """
    (trainX, trainY, X, y, learner) = args
    
    learner.learnModel(trainX, trainY)
    predY = learner.predict(X)
    predTrainY = learner.predict(trainX)
    
    return learner.getMetricMethod()(predY, y) - learner.getMetricMethod()(predTrainY, trainY)

def computeIdealPenalty(args):
    """
    Find the complete penalty.
    """
    (trainX, trainY, fullX, fullY, learner) = args

    learner.learnModel(trainX, trainY)
    predTrainY = learner.predict(trainX)
    predFullY = learner.predict(fullX)

    idealPenalty = learner.getMetricMethod()(predFullY, fullY) - learner.getMetricMethod()(predTrainY, trainY)
    return idealPenalty

class AbstractPredictor(object):
    def __init__(self): 
        #Used in multiprocessing code 
        self.processes = multiprocessing.cpu_count() 
        self.chunkSize = 10 
    """
    An abstract classifier for binary labelled data. 
    """
    def learnModel(self, X, y):
        """
        Learn a model for a set of examples given as the rows of the matrix X,
        with corresponding labels given in the elements of 1D array y.

        :param X: A matrix with examples as rows
        :type X: :class:`ndarray`<commit_msg>Reduce default chunk size <commit_after>'''
Created on 16 Jul 2009

@author: charanpal
'''
from apgl.util.Util import Util
from apgl.util.Evaluator import Evaluator 
from apgl.util.Parameter import Parameter
from apgl.util.Sampling import Sampling
import numpy
import logging
import gc
import itertools 
import multiprocessing 
from sklearn import linear_model 


#Start with some functions used for multiprocessing 

def computeTestError(args):
    """
    Used in conjunction with the parallel model selection. Trains and then tests
    on a seperate test set. 
    """
    (trainX, trainY, testX, testY, learner) = args
    learner.learnModel(trainX, trainY)
    predY = learner.predict(testX)

    return learner.getMetricMethod()(testY, predY)

def computeBootstrapError(args):
    """
    Used in conjunction with the parallel model selection. Trains and then tests
    on a seperate test set and evaluated the bootstrap error. 
    """
    (trainX, trainY, testX, testY, learner) = args
    learner.learnModel(trainX, trainY)
    predTestY = learner.predict(testX)
    predTrainY = learner.predict(trainX)
    weight = 0.632
    return Evaluator.binaryBootstrapError(predTestY, testY, predTrainY, trainY, weight)

def computeTrainError(args): 
    """
    Train on a set of examples of test on the same examples. 
    """
    (X, y, learner) = args
    learner.learnModel(X, y)
    predY = learner.predict(X)
    return learner.getMetricMethod()(predY, y)

def computeVFPen(args): 
    """
    Compute the penVF criteria for a single fold 
    """
    (trainX, trainY, X, y, learner) = args
    
    learner.learnModel(trainX, trainY)
    predY = learner.predict(X)
    predTrainY = learner.predict(trainX)
    
    return learner.getMetricMethod()(predY, y) - learner.getMetricMethod()(predTrainY, trainY)

def computeIdealPenalty(args):
    """
    Find the complete penalty.
    """
    (trainX, trainY, fullX, fullY, learner) = args

    learner.learnModel(trainX, trainY)
    predTrainY = learner.predict(trainX)
    predFullY = learner.predict(fullX)

    idealPenalty = learner.getMetricMethod()(predFullY, fullY) - learner.getMetricMethod()(predTrainY, trainY)
    return idealPenalty

class AbstractPredictor(object):
    def __init__(self): 
        #Used in multiprocessing code 
        self.processes = multiprocessing.cpu_count() 
        self.chunkSize = 1 
    """
    An abstract classifier for binary labelled data. 
    """
    def learnModel(self, X, y):
        """
        Learn a model for a set of examples given as the rows of the matrix X,
        with corresponding labels given in the elements of 1D array y.

        :param X: A matrix with examples as rows
        :type X: :class:`ndarray`
{'fraction_non_alphanumeric': 0.08930540242557883, 'fraction_numerical': 0.004593899301727306, 'mean_word_length': 3.343974461292897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2870543', 'n_tokens_mistral': 1772, 'n_tokens_neox': 1676, 'n_words': 521}