coding utf8 logistic function references https arxiv orgpdf1603 02754v3 pdf http www saedsayad comdocsxgboost pdf https homes cs washington edutqchenpdfboostedtree pdf http stats stackexchange comquestions202858lossfunctionapproximationwithtaylorexpansion base class for loss functions def initself regularization1 0 self regularization regularization def gradself actual predicted second order gradient raise notimplementederror def approximateself actual predicted transform predictions values return pred def gainself actual predicted least squares loss def gradself actual predicted return actual predicted def hessself actual predicted return np oneslikeactual class logisticlossloss apply logistic sigmoid function to the output gradient boosting trees with taylor s expansion approximation as in xgboost def initself nestimators learningrate0 1 maxfeatures10 maxdepth2 minsamplessplit10 self minsamplessplit minsamplessplit self learningrate learningrate self maxdepth maxdepth self maxfeatures maxfeatures self nestimators nestimators self trees self loss none def fitself x ynone self setupinputx y self ymean np meany self train def trainself initialize model with zeros ypred np zerosself nsamples np float32 or mean ypred np fullself nsamples self ymean for n in rangeself nestimators residuals self loss gradself y ypred tree treeregressiontrue criterionmsecriterion pass multiple target values to the tree learner targets residual values y residuals actual target values actual self y predictions from previous step ypred ypred tree train self x targets maxfeaturesself maxfeatures minsamplessplitself minsamplessplit maxdepthself maxdepth lossself loss predictions tree predictself x ypred self learningrate predictions self trees appendtree def predictself xnone ypred np zerosx shape0 np float32 for i tree in enumerateself trees ypred self learningrate tree predictx return ypred def predictself xnone return self loss transformself predictx class gradientboostingregressorgradientboosting def fitself x ynone self loss leastsquaresloss supergradientboostingregressor self fitx y class gradientboostingclassifiergradientboosting def fitself x ynone convert labels from 0 1 to 1 1 y y 2 1 self loss logisticloss supergradientboostingclassifier self fitx y coding utf 8 logistic function references https arxiv org pdf 1603 02754v3 pdf http www saedsayad com docs xgboost pdf https homes cs washington edu tqchen pdf boostedtree pdf http stats stackexchange com questions 202858 loss function approximation with taylor expansion base class for loss functions first order gradient second order gradient approximate leaf value transform predictions values calculate gain for split search least squares loss logistic loss apply logistic sigmoid function to the output gradient boosting trees with taylor s expansion approximation as in xgboost initialize model with zeros or mean y_pred np full self n_samples self y_mean pass multiple target values to the tree learner residual values actual target values predictions from previous step convert labels from 0 1 to 1 1
import numpy as np

from scipy.special import expit

from mla.base import BaseEstimator
from mla.ensemble.base import mse_criterion
from mla.ensemble.tree import Tree




class Loss:
    

    def __init__(self, regularization=1.0):
        self.regularization = regularization

    def grad(self, actual, predicted):
        
        raise NotImplementedError()

    def hess(self, actual, predicted):
        
        raise NotImplementedError()

    def approximate(self, actual, predicted):
        
        return self.grad(actual, predicted).sum() / (self.hess(actual, predicted).sum() + self.regularization)

    def transform(self, pred):
        
        return pred

    def gain(self, actual, predicted):
        
        nominator = self.grad(actual, predicted).sum() ** 2
        denominator = self.hess(actual, predicted).sum() + self.regularization
        return 0.5 * (nominator / denominator)


class LeastSquaresLoss(Loss):
    

    def grad(self, actual, predicted):
        return actual - predicted

    def hess(self, actual, predicted):
        return np.ones_like(actual)


class LogisticLoss(Loss):
    

    def grad(self, actual, predicted):
        return actual * expit(-actual * predicted)

    def hess(self, actual, predicted):
        expits = expit(predicted)
        return expits * (1 - expits)

    def transform(self, output):
        
        return expit(output)


class GradientBoosting(BaseEstimator):
    

    def __init__(self, n_estimators, learning_rate=0.1, max_features=10, max_depth=2, min_samples_split=10):
        self.min_samples_split = min_samples_split
        self.learning_rate = learning_rate
        self.max_depth = max_depth
        self.max_features = max_features
        self.n_estimators = n_estimators
        self.trees = []
        self.loss = None

    def fit(self, X, y=None):
        self._setup_input(X, y)
        self.y_mean = np.mean(y)
        self._train()

    def _train(self):
        
        y_pred = np.zeros(self.n_samples, np.float32)
        
        

        for n in range(self.n_estimators):
            residuals = self.loss.grad(self.y, y_pred)
            tree = Tree(regression=True, criterion=mse_criterion)
            
            targets = {
                
                "y": residuals,
                
                "actual": self.y,
                
                "y_pred": y_pred,
            }
            tree.train(
                self.X,
                targets,
                max_features=self.max_features,
                min_samples_split=self.min_samples_split,
                max_depth=self.max_depth,
                loss=self.loss,
            )
            predictions = tree.predict(self.X)
            y_pred += self.learning_rate * predictions
            self.trees.append(tree)

    def _predict(self, X=None):
        y_pred = np.zeros(X.shape[0], np.float32)

        for i, tree in enumerate(self.trees):
            y_pred += self.learning_rate * tree.predict(X)
        return y_pred

    def predict(self, X=None):
        return self.loss.transform(self._predict(X))


class GradientBoostingRegressor(GradientBoosting):
    def fit(self, X, y=None):
        self.loss = LeastSquaresLoss()
        super(GradientBoostingRegressor, self).fit(X, y)


class GradientBoostingClassifier(GradientBoosting):
    def fit(self, X, y=None):
        
        y = (y * 2) - 1
        self.loss = LogisticLoss()
        super(GradientBoostingClassifier, self).fit(X, y)