<commit_before>from sklearn.feature_extraction import DictVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import FunctionTransformer

import utils

from sklearn.grid_search import GridSearchCV


class Predictor(object):


    def __init__(self, type_of_algo, column_descriptions, verbose=True):
        self.type_of_algo = type_of_algo
        self.column_descriptions = column_descriptions
        self.verbose = verbose
        self.trained_pipeline = None

        # figure out which column has a value 'output'
        output_column = [key for key, value in column_descriptions.items() if value.lower() == 'output'][0]
        self.output_column = output_column


    def _construct_pipeline(self, user_input_func=None, ml_for_analytics=False, model_name='LogisticRegression', optimize_final_model=False):

        pipeline_list = []
        if user_input_func is not None:
            pipeline_list.append(('user_func', FunctionTransformer(func=user_input_func, pass_y=False, validate=False) ))

        # These parts will be included no matter what.
        pipeline_list.append(('basic_transform', utils.BasicDataCleaning()))
        pipeline_list.append(('dv', DictVectorizer(sparse=True)))

        pipeline_list.append(('final_model', utils.FinalModelATC(model_name=model_name, perform_grid_search_on_model=optimize_final_model)))

        constructed_pipeline = Pipeline(pipeline_list)
        return constructed_pipeline


    def _construct_pipeline_search_params(self, optimize_entire_pipeline=True, optimize_final_model=False, ml_for_analytics=False):

        gs_params = {}

        if optimize_final_model:
            gs_params['final_model__perform_grid_search_on_model'] = [True, False]

        if ml_for_analytics:
            gs_params['final_model__ml_for_analytics'] = [True]

        else:
            if optimize_entire_pipeline:
                gs_params['final_model__model_name'] = ['RandomForestClassifier', 'LogisticRegression']

        return gs_params


    def train(self, raw_training_data, user_input_func=None, optimize_entire_pipeline=False, optimize_final_model=False, print_analytics_output=False):

        # split out out output column so we have a proper X, y dataset
        X, y = utils.split_output(raw_training_data, self.output_column)

        ppl = self._construct_pipeline(user_input_func, optimize_final_model)

        self.gs_params = self._construct_pipeline_search_params(optimize_entire_pipeline=optimize_entire_pipeline, optimize_final_model=optimize_final_model)

        # We will be performing GridSearchCV every time, even if the space we are searching over is null
        gs = GridSearchCV(
            # Fit on the pipeline.
            ppl,
            self.grid_search_params,
            # Train across all cores.
            n_jobs=-1,
            # Be verbose (lots of printing).
            verbose=10,
            # Print warnings when we fail to fit a given combination of parameters, but do not raise an error.
            error_score=10,
            # TODO(PRESTON): change scoring to be RMSE by default
            scoring=None
        )

        gs.fit(X, y)

        self.trained_pipeline = gs.best_estimator_

        return self



    def ml_for_analytics(self, raw_training_data, user_input_func=None, optimize_entire_pipeline=False, optimize_final_model=False, print_analytics_output=False):

        # split out out output column so we have a proper X, y dataset
        X, y = utils.split_output(raw_training_data, self.output_column)

        ppl = self._construct_pipeline(user_input_func, optimize_final_model=optimize_final_model, ml_for_analytics=True)

        for model_name in ['LogisticRegression', 'RandomForestClassifier']:

            self.grid_search_params = self._construct_pipeline_search_params(optimize_entire_pipeline=optimize_entire_pipeline, optimize_final_model=optimize_final_model, ml_for_analytics=True)

            self.grid_search_params['final_model__model_name'] = [model_name]

            gs = GridSearchCV(
                # Fit on the pipeline.
                ppl,
                self.grid_search_params,
                # Train across all cores.
                n_jobs=-1,
                # Be verbose (lots of printing).
                verbose=10,
                # Print warnings when we fail to fit a given combination of parameters, but do not raise an error.
                error_score=10,
                # TODO(PRESTON): change scoring to be RMSE by default
                scoring=None
            )

            gs.fit(X, y)
            self.trained_pipeline = gs.best_estimator_

            if model_name in ('LogisticRegression', 'Ridge'):
                self._print_ml_analytics_results_regression()
            elif model_name == 'RandomForestClassifier':
                self._print_ml_analytics_results_random_forest()

    def _print_ml_analytics_results_random_forest(self):

        trained_feature_names = self.trained_pipeline.named_steps['dv'].get_feature_names()

        trained_feature_importances = self.trained_pipeline.named_steps['final_model'].model.feature_importances_

        feature_infos = zip(trained_feature_names, trained_feature_importances)

        sorted_feature_infos = sorted(feature_infos, key=lambda x: x[1])
<commit_msg>refactors to have modular method to create estimator names<commit_after>from sklearn.feature_extraction import DictVectorizer
# from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import FunctionTransformer

import utils

from sklearn.grid_search import GridSearchCV


class Predictor(object):


    def __init__(self, type_of_algo, column_descriptions, verbose=True):
        self.type_of_algo = type_of_algo
        self.column_descriptions = column_descriptions
        self.verbose = verbose
        self.trained_pipeline = None

        # figure out which column has a value 'output'
        output_column = [key for key, value in column_descriptions.items() if value.lower() == 'output'][0]
        self.output_column = output_column


    def _construct_pipeline(self, user_input_func=None, ml_for_analytics=False, model_name='LogisticRegression', optimize_final_model=False):

        pipeline_list = []
        if user_input_func is not None:
            pipeline_list.append(('user_func', FunctionTransformer(func=user_input_func, pass_y=False, validate=False) ))

        # These parts will be included no matter what.
        pipeline_list.append(('basic_transform', utils.BasicDataCleaning()))
        pipeline_list.append(('dv', DictVectorizer(sparse=True)))

        pipeline_list.append(('final_model', utils.FinalModelATC(model_name=model_name, perform_grid_search_on_model=optimize_final_model)))

        constructed_pipeline = Pipeline(pipeline_list)
        return constructed_pipeline


    def _construct_pipeline_search_params(self, optimize_entire_pipeline=True, optimize_final_model=False, ml_for_analytics=False):

        gs_params = {}

        if optimize_final_model:
            gs_params['final_model__perform_grid_search_on_model'] = [True, False]

        if ml_for_analytics:
            gs_params['final_model__ml_for_analytics'] = [True]

        else:
            if optimize_entire_pipeline:
                gs_params['final_model__model_name'] = self._get_estimator_names(ml_for_analytics=ml_for_analytics)

        return gs_params


    def train(self, raw_training_data, user_input_func=None, optimize_entire_pipeline=False, optimize_final_model=False, print_analytics_output=False):

        # split out out output column so we have a proper X, y dataset
        X, y = utils.split_output(raw_training_data, self.output_column)

        ppl = self._construct_pipeline(user_input_func, optimize_final_model)

        self.gs_params = self._construct_pipeline_search_params(optimize_entire_pipeline=optimize_entire_pipeline, optimize_final_model=optimize_final_model)

        # We will be performing GridSearchCV every time, even if the space we are searching over is null
        gs = GridSearchCV(
            # Fit on the pipeline.
            ppl,
            self.grid_search_params,
            # Train across all cores.
            n_jobs=-1,
            # Be verbose (lots of printing).
            verbose=10,
            # Print warnings when we fail to fit a given combination of parameters, but do not raise an error.
            error_score=10,
            # TODO(PRESTON): change scoring to be RMSE by default
            scoring=None
        )

        gs.fit(X, y)

        self.trained_pipeline = gs.best_estimator_

        return self

    def _get_estimator_names(self, ml_for_analytics=False):
        if self.type_of_algo == 'regressor':
            base_estimators = ['LinearRegression', 'RandomForestRegressor', 'Ridge']
            if ml_for_analytics:
                return base_estimators
            else:
                # base_estimators.append()
                return base_estimators

        elif self.type_of_algo == 'classifier':
            base_estimators = ['LogisticRegression', 'RandomForestClassifier', 'RidgeClassifier']
            if ml_for_analytics:
                return base_estimators
            else:
                # base_estimators.append()
                return base_estimators

        else:
            raise('TypeError: type_of_algo must be either "classifier" or "regressor".')

    def ml_for_analytics(self, raw_training_data, user_input_func=None, optimize_entire_pipeline=False, optimize_final_model=False, print_analytics_output=False):

        # split out out output column so we have a proper X, y dataset
        X, y = utils.split_output(raw_training_data, self.output_column)

        ppl = self._construct_pipeline(user_input_func, optimize_final_model=optimize_final_model, ml_for_analytics=True)

        estimator_names = self._get_estimator_names(ml_for_analytics=True)

        for model_name in estimator_names:

            self.grid_search_params = self._construct_pipeline_search_params(optimize_entire_pipeline=optimize_entire_pipeline, optimize_final_model=optimize_final_model, ml_for_analytics=True)

            self.grid_search_params['final_model__model_name'] = [model_name]

            gs = GridSearchCV(
                # Fit on the pipeline.
                ppl,
                self.grid_search_params,
                # Train across all cores.
                n_jobs=-1,
                # Be verbose (lots of printing).
                verbose=10,
                # Print warnings when we fail to fit a given combination of parameters, but do not raise an error.
                error_score=10,
                # TODO(PRESTON): change scoring to be RMSE by default
                scoring=None
            )

            gs.fit(X, y)
            self.trained_pipeline = gs.best_estimator_

            if model_name in ('LogisticRegression', 'RidgeClassifier', 'LinearRegression', 'Ridge'):
                self._print_ml_analytics_results_regression()
            elif model_name in ['RandomForestClassifier', 'RandomForestRegressor']:
                self._print_ml_analytics_results_random_forest()

    def _print_ml_analytics_results_random_forest(self):

        trained_feature_names = self.trained_pipeline.named_steps['dv'].get_feature_names()

        trained_feature_importances = self.trained_pipeline.named_steps['final_model'].model.feature_importances_

        feature_infos = zip(trained_feature_names, trained_feature_importances)

        sorted_feature_infos = sorted(feature_infos, key=lambda x: x[1])

{'fraction_non_alphanumeric': 0.07047667862634546, 'fraction_numerical': 0.0020502306509482316, 'mean_word_length': 3.0327247674819153, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2840069', 'n_tokens_mistral': 3453, 'n_tokens_neox': 3318, 'n_words': 712}