initialize a gradientboostingclassifier parameters nestimators int the number of weak learners to train learningrate float the learning rate for updating the model attributes nestimators int the number of weak learners learningrate float the learning rate models list a list to store the trained weak learners fit the gradientboostingclassifier to the training data parameters features np ndarray the training features target np ndarray the target values returns none import numpy as np from sklearn datasets import loadiris clf gradientboostingclassifiernestimators100 learningrate0 1 iris loadiris x y iris data iris target clf fitx y check if the model is trained lenclf models 100 true calculate the pseudoresiduals fit a weak learner e g decision tree to the residuals update the model by adding the weak learner with a learning rate make predictions on input data parameters features np ndarray the input data for making predictions returns np ndarray an array of binary predictions 1 or 1 import numpy as np from sklearn datasets import loadiris clf gradientboostingclassifiernestimators100 learningrate0 1 iris loadiris x y iris data iris target clf fitx y ypred clf predictx check if the predictions have the correct shape ypred shape y shape true initialize predictions with zeros calculate the negative gradient pseudoresiduals for logistic loss parameters target np ndarray the target values ypred np ndarray the predicted values returns np ndarray an array of pseudoresiduals import numpy as np clf gradientboostingclassifiernestimators100 learningrate0 1 target np array0 1 0 1 ypred np array0 2 0 8 0 3 0 7 residuals clf gradienttarget ypred check if residuals have the correct shape residuals shape target shape true initialize a gradientboostingclassifier parameters n_estimators int the number of weak learners to train learning_rate float the learning rate for updating the model attributes n_estimators int the number of weak learners learning_rate float the learning rate models list a list to store the trained weak learners fit the gradientboostingclassifier to the training data parameters features np ndarray the training features target np ndarray the target values returns none import numpy as np from sklearn datasets import load_iris clf gradientboostingclassifier n_estimators 100 learning_rate 0 1 iris load_iris x y iris data iris target clf fit x y check if the model is trained len clf models 100 true calculate the pseudo residuals fit a weak learner e g decision tree to the residuals update the model by adding the weak learner with a learning rate make predictions on input data parameters features np ndarray the input data for making predictions returns np ndarray an array of binary predictions 1 or 1 import numpy as np from sklearn datasets import load_iris clf gradientboostingclassifier n_estimators 100 learning_rate 0 1 iris load_iris x y iris data iris target clf fit x y y_pred clf predict x check if the predictions have the correct shape y_pred shape y shape true initialize predictions with zeros convert to binary predictions 1 or 1 calculate the negative gradient pseudo residuals for logistic loss parameters target np ndarray the target values y_pred np ndarray the predicted values returns np ndarray an array of pseudo residuals import numpy as np clf gradientboostingclassifier n_estimators 100 learning_rate 0 1 target np array 0 1 0 1 y_pred np array 0 2 0 8 0 3 0 7 residuals clf gradient target y_pred check if residuals have the correct shape residuals shape target shape true
import numpy as np
from sklearn.datasets import load_iris
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeRegressor


class GradientBoostingClassifier:
    def __init__(self, n_estimators: int = 100, learning_rate: float = 0.1) -> None:
        
        self.n_estimators = n_estimators
        self.learning_rate = learning_rate
        self.models: list[tuple[DecisionTreeRegressor, float]] = []

    def fit(self, features: np.ndarray, target: np.ndarray) -> None:
        
        for _ in range(self.n_estimators):
            
            residuals = -self.gradient(target, self.predict(features))
            
            model = DecisionTreeRegressor(max_depth=1)
            model.fit(features, residuals)
            
            self.models.append((model, self.learning_rate))

    def predict(self, features: np.ndarray) -> np.ndarray:
        
        
        predictions = np.zeros(features.shape[0])
        for model, learning_rate in self.models:
            predictions += learning_rate * model.predict(features)
        return np.sign(predictions)  

    def gradient(self, target: np.ndarray, y_pred: np.ndarray) -> np.ndarray:
        
        return -target / (1 + np.exp(target * y_pred))


if __name__ == "__main__":
    iris = load_iris()
    X, y = iris.data, iris.target
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42
    )

    clf = GradientBoostingClassifier(n_estimators=100, learning_rate=0.1)
    clf.fit(X_train, y_train)

    y_pred = clf.predict(X_test)
    accuracy = accuracy_score(y_test, y_pred)
    print(f"Accuracy: {accuracy:.2f}")