<commit_before>#!/usr/bin/env python

import numpy as np
from scipy.sparse import vstack
from sklearn import cross_validation
import random
import time
import datetime
import joblib
import pprint
import copy
from collections import defaultdict
from sklearn.linear_model import SGDClassifier
from itertools import tee, izip
from sklearn.metrics import precision_recall_curve, roc_curve
from sklearn.metrics import classification_report, roc_auc_score, average_precision_score
from eden.util import fit_estimator, selection_iterator
from eden.util import is_iterable
from eden.util.util import report_base_statistics
from eden.graph import Vectorizer


class ActiveLearningBinaryClassificationModel(object):

    def __init__(self, pre_processor=None,
                 vectorizer=Vectorizer(complexity=1),
                 estimator=SGDClassifier(class_weight='auto', shuffle=True),
                 description=None,
                 random_seed=1):
        self.pre_processor = pre_processor
        self.vectorizer = vectorizer
        self.estimator = estimator
        self.pre_processor_args = None
        self.vectorizer_args = None
        self.estimator_args = None
        self.description = description
        random.seed(random_seed)

    def get_description(self):
        print self.description

    def save(self, model_name):
        joblib.dump(self, model_name, compress=1)

    def load(self, obj):
        self.__dict__.update(joblib.load(obj).__dict__)

    def get_pre_processor():
        return self.pre_processor

    def get_vectorizer():
        return self.vectorizer

    def get_estimator():
        return self.estimator

    def _data_matrix(self, iterable, fit_vectorizer=False, n_jobs=1):
        assert(is_iterable(iterable)), 'Not iterable'
        iterable = self.pre_processor(iterable, **self.pre_processor_args)
        self.vectorizer.set_params(**self.vectorizer_args)
        if fit_vectorizer:
            X = self.vectorizer.fit_transform(iterable, n_jobs=n_jobs)
        else:
            X = self.vectorizer.transform(iterable, n_jobs=n_jobs)
        return X

    def _data_matrices(self, iterable_pos, iterable_neg, fit_vectorizer=False, n_jobs=1):
        assert(is_iterable(iterable_pos) and is_iterable(iterable_neg)), 'Not iterable'
        self.vectorizer.set_params(**self.vectorizer_args)
        iterator_pos = self.pre_processor(iterable_pos, **self.pre_processor_args)
        if fit_vectorizer:
            Xpos = self.vectorizer.fit_transform(iterator_pos, n_jobs=n_jobs)
        else:
            Xpos = self.vectorizer.transform(iterator_pos, n_jobs=n_jobs)
        iterator_neg = self.pre_processor(iterable_neg, **self.pre_processor_args)
        Xneg = self.vectorizer.transform(iterator_neg, n_jobs=n_jobs)
        return self._assemble_data_matrix(Xpos, Xneg)

    def _assemble_data_matrix(self, Xpos, Xneg):
        yp = [1] * Xpos.shape[0]
        yn = [-1] * Xneg.shape[0]
        y = np.array(yp + yn)
        X = vstack([Xpos, Xneg], format="csr")
        return X, y

    def _sample(self, parameters):
        parameters_sample = dict()
        for parameter in parameters:
            values = parameters[parameter]
            value = random.choice(values)
            parameters_sample[parameter] = value
        return parameters_sample

    def fit(self, iterable_pos, iterable_neg, n_jobs=1):
        self.estimator.set_params(**self.estimator_args)
        X, y = self._data_matrices(iterable_pos, iterable_neg, n_jobs=n_jobs)
        self.estimator.fit(X, y)

    def predict(self, iterable, n_jobs=1):
        X = self._data_matrix(iterable, n_jobs=n_jobs)
        return self.estimator.predict(X)

    def decision_function(self, iterable, n_jobs=1):
        X = self._data_matrix(iterable, n_jobs=n_jobs)
        return self.estimator.decision_function(X)

    def get_info(self, iterable):
        iterable_graph = self.pre_processor(iterable, **self.pre_processor_args)
        yield iterable_graph.graph.get(key, 'N/A')

    def info(self, iterable, key='info', n_jobs=1):
        iterable, iterable_ = tee(iterable)
        X = self._data_matrix(iterable, n_jobs=n_jobs)
        info_iterable = get_info(iterable_)
        return izip(self.estimator.decision_function(X), info_iterable)

    def estimate(self, iterable_pos, iterable_neg, n_jobs=1):<commit_msg>added info method in Model<commit_after>#!/usr/bin/env python

import numpy as np
from scipy.sparse import vstack
from sklearn import cross_validation
import random
import time
import datetime
import joblib
import pprint
import copy
from collections import defaultdict
from sklearn.linear_model import SGDClassifier
from itertools import tee, izip
from sklearn.metrics import precision_recall_curve, roc_curve
from sklearn.metrics import classification_report, roc_auc_score, average_precision_score
from eden.util import fit_estimator, selection_iterator
from eden.util import is_iterable
from eden.util.util import report_base_statistics
from eden.graph import Vectorizer


class ActiveLearningBinaryClassificationModel(object):

    def __init__(self, pre_processor=None,
                 vectorizer=Vectorizer(complexity=1),
                 estimator=SGDClassifier(class_weight='auto', shuffle=True),
                 description=None,
                 random_seed=1):
        self.pre_processor = pre_processor
        self.vectorizer = vectorizer
        self.estimator = estimator
        self.pre_processor_args = None
        self.vectorizer_args = None
        self.estimator_args = None
        self.description = description
        random.seed(random_seed)

    def get_description(self):
        print self.description

    def save(self, model_name):
        joblib.dump(self, model_name, compress=1)

    def load(self, obj):
        self.__dict__.update(joblib.load(obj).__dict__)

    def get_pre_processor():
        return self.pre_processor

    def get_vectorizer():
        return self.vectorizer

    def get_estimator():
        return self.estimator

    def _data_matrix(self, iterable, fit_vectorizer=False, n_jobs=1):
        assert(is_iterable(iterable)), 'Not iterable'
        iterable = self.pre_processor(iterable, **self.pre_processor_args)
        self.vectorizer.set_params(**self.vectorizer_args)
        if fit_vectorizer:
            X = self.vectorizer.fit_transform(iterable, n_jobs=n_jobs)
        else:
            X = self.vectorizer.transform(iterable, n_jobs=n_jobs)
        return X

    def _data_matrices(self, iterable_pos, iterable_neg, fit_vectorizer=False, n_jobs=1):
        assert(is_iterable(iterable_pos) and is_iterable(iterable_neg)), 'Not iterable'
        self.vectorizer.set_params(**self.vectorizer_args)
        iterator_pos = self.pre_processor(iterable_pos, **self.pre_processor_args)
        if fit_vectorizer:
            Xpos = self.vectorizer.fit_transform(iterator_pos, n_jobs=n_jobs)
        else:
            Xpos = self.vectorizer.transform(iterator_pos, n_jobs=n_jobs)
        iterator_neg = self.pre_processor(iterable_neg, **self.pre_processor_args)
        Xneg = self.vectorizer.transform(iterator_neg, n_jobs=n_jobs)
        return self._assemble_data_matrix(Xpos, Xneg)

    def _assemble_data_matrix(self, Xpos, Xneg):
        yp = [1] * Xpos.shape[0]
        yn = [-1] * Xneg.shape[0]
        y = np.array(yp + yn)
        X = vstack([Xpos, Xneg], format="csr")
        return X, y

    def _sample(self, parameters):
        parameters_sample = dict()
        for parameter in parameters:
            values = parameters[parameter]
            value = random.choice(values)
            parameters_sample[parameter] = value
        return parameters_sample

    def fit(self, iterable_pos, iterable_neg, n_jobs=1):
        self.estimator.set_params(**self.estimator_args)
        X, y = self._data_matrices(iterable_pos, iterable_neg, n_jobs=n_jobs)
        self.estimator.fit(X, y)

    def predict(self, iterable, n_jobs=1):
        X = self._data_matrix(iterable, n_jobs=n_jobs)
        return self.estimator.predict(X)

    def decision_function(self, iterable, n_jobs=1):
        X = self._data_matrix(iterable, n_jobs=n_jobs)
        return self.estimator.decision_function(X)

    def get_info(self, iterable):
        iterable_graph = self.pre_processor(iterable, **self.pre_processor_args)
        yield iterable_graph.graph.get(key, 'N/A')

    def info(self, iterable, key='info', n_jobs=1):
        iterable, iterable_ = tee(iterable)
        X = self._data_matrix(iterable, n_jobs=n_jobs)
        info_iterable = self.get_info(iterable_)
        return izip(self.estimator.decision_function(X), info_iterable)

    def estimate(self, iterable_pos, iterable_neg, n_jobs=1):
{'fraction_non_alphanumeric': 0.08885576702591148, 'fraction_numerical': 0.0032102728731942215, 'mean_word_length': 3.2928149606299213, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7505511', 'n_tokens_mistral': 2813, 'n_tokens_neox': 2696, 'n_words': 547}