polynomial regression is a type of regression analysis that models the relationship between a predictor x and the response y as an mthdegree polynomial y x x x by treating x x x as distinct variables we see that polynomial regression is a special case of multiple linear regression therefore we can use ordinary least squares ols estimation to estimate the vector of model parameters for polynomial regression xxxy xy where x is the design matrix y is the response vector and x denotes the moorepenrose pseudoinverse of x in the case of polynomial regression the design matrix is 1 x x x x 1 x x x 1 x x x in ols estimation inverting xx to compute x can be very numerically unstable this implementation sidesteps this need to invert xx by computing x using singular value decomposition svd vuy where uv is an svd of x references https en wikipedia orgwikipolynomialregression https en wikipedia orgwikimooree28093penroseinverse https en wikipedia orgwikinumericalmethodsforlinearleastsquares https en wikipedia orgwikisingularvaluedecomposition raises valueerror if the polynomial degree is negative constructs a polynomial regression design matrix for the given input data for input data x x x x and polynomial degree m the design matrix is the vandermonde matrix 1 x x x x 1 x x x 1 x x x reference https en wikipedia orgwikivandermondematrix param data the input predictor values x either for model fitting or for prediction param degree the polynomial degree m returns the vandermonde matrix x see above raises valueerror if input data is not n x 1 x np array0 1 2 polynomialregression designmatrixx degree0 array1 1 1 polynomialregression designmatrixx degree1 array1 0 1 1 1 2 polynomialregression designmatrixx degree2 array1 0 0 1 1 1 1 2 4 polynomialregression designmatrixx degree3 array1 0 0 0 1 1 1 1 1 2 4 8 polynomialregression designmatrixnp array0 0 0 0 degree3 traceback most recent call last valueerror data must have dimensions n x 1 computes the polynomial regression model parameters using ordinary least squares ols estimation xxxy xy where x denotes the moorepenrose pseudoinverse of the design matrix x this function computes x using singular value decomposition svd references https en wikipedia orgwikimooree28093penroseinverse https en wikipedia orgwikisingularvaluedecomposition https en wikipedia orgwikimulticollinearity param xtrain the predictor values x for model fitting param ytrain the response values y for model fitting raises arithmeticerror if x isn t full rank then xx is singular and doesn t exist x np array0 1 2 3 4 5 6 7 8 9 10 y x3 2 x2 3 x 5 polyreg polynomialregressiondegree3 polyreg fitx y polyreg params array5 3 2 1 polyreg polynomialregressiondegree20 polyreg fitx y traceback most recent call last arithmeticerror design matrix is not full rank can t compute coefficients make sure errors don t grow too large coefs np array250 50 2 36 20 12 10 2 1 15 1 y polynomialregression designmatrixx lencoefs 1 coefs polyreg polynomialregressiondegreelencoefs 1 polyreg fitx y np allclosepolyreg params coefs atol10e3 true np linalg pinv computes the moorepenrose pseudoinverse using svd computes the predicted response values y for the given input data by constructing the design matrix x and evaluating y x param data the predictor values x for prediction returns the predicted response values y x raises arithmeticerror if this function is called before the model parameters are fit x np array0 1 2 3 4 y x3 2 x2 3 x 5 polyreg polynomialregressiondegree3 polyreg fitx y polyreg predictnp array1 array11 polyreg predictnp array2 array27 polyreg predictnp array6 array157 polynomialregressiondegree3 predictx traceback most recent call last arithmeticerror predictor hasn t been fit yet fit a polynomial regression model to predict fuel efficiency using seaborn s mpg dataset pass placeholder function is only for demo purposes raises valueerror if the polynomial degree is negative constructs a polynomial regression design matrix for the given input data for input data x x₁ x₂ xₙ and polynomial degree m the design matrix is the vandermonde matrix 1 x₁ x₁² x₁ᵐ x 1 x₂ x₂² x₂ᵐ 1 xₙ xₙ² xₙᵐ reference https en wikipedia org wiki vandermonde_matrix param data the input predictor values x either for model fitting or for prediction param degree the polynomial degree m returns the vandermonde matrix x see above raises valueerror if input data is not n x 1 x np array 0 1 2 polynomialregression _design_matrix x degree 0 array 1 1 1 polynomialregression _design_matrix x degree 1 array 1 0 1 1 1 2 polynomialregression _design_matrix x degree 2 array 1 0 0 1 1 1 1 2 4 polynomialregression _design_matrix x degree 3 array 1 0 0 0 1 1 1 1 1 2 4 8 polynomialregression _design_matrix np array 0 0 0 0 degree 3 traceback most recent call last valueerror data must have dimensions n x 1 computes the polynomial regression model parameters using ordinary least squares ols estimation β xᵀx ¹xᵀy x y where x denotes the moore penrose pseudoinverse of the design matrix x this function computes x using singular value decomposition svd references https en wikipedia org wiki moore e2 80 93penrose_inverse https en wikipedia org wiki singular_value_decomposition https en wikipedia org wiki multicollinearity param x_train the predictor values x for model fitting param y_train the response values y for model fitting raises arithmeticerror if x isn t full rank then xᵀx is singular and β doesn t exist x np array 0 1 2 3 4 5 6 7 8 9 10 y x 3 2 x 2 3 x 5 poly_reg polynomialregression degree 3 poly_reg fit x y poly_reg params array 5 3 2 1 poly_reg polynomialregression degree 20 poly_reg fit x y traceback most recent call last arithmeticerror design matrix is not full rank can t compute coefficients make sure errors don t grow too large coefs np array 250 50 2 36 20 12 10 2 1 15 1 y polynomialregression _design_matrix x len coefs 1 coefs poly_reg polynomialregression degree len coefs 1 poly_reg fit x y np allclose poly_reg params coefs atol 10e 3 true noqa n806 np linalg pinv computes the moore penrose pseudoinverse using svd computes the predicted response values y for the given input data by constructing the design matrix x and evaluating y xβ param data the predictor values x for prediction returns the predicted response values y xβ raises arithmeticerror if this function is called before the model parameters are fit x np array 0 1 2 3 4 y x 3 2 x 2 3 x 5 poly_reg polynomialregression degree 3 poly_reg fit x y poly_reg predict np array 1 array 11 poly_reg predict np array 2 array 27 poly_reg predict np array 6 array 157 polynomialregression degree 3 predict x traceback most recent call last arithmeticerror predictor hasn t been fit yet fit a polynomial regression model to predict fuel efficiency using seaborn s mpg dataset pass placeholder function is only for demo purposes
import matplotlib.pyplot as plt
import numpy as np


class PolynomialRegression:
    __slots__ = "degree", "params"

    def __init__(self, degree: int) -> None:
        
        if degree < 0:
            raise ValueError("Polynomial degree must be non-negative")

        self.degree = degree
        self.params = None

    @staticmethod
    def _design_matrix(data: np.ndarray, degree: int) -> np.ndarray:
        
        rows, *remaining = data.shape
        if remaining:
            raise ValueError("Data must have dimensions N x 1")

        return np.vander(data, N=degree + 1, increasing=True)

    def fit(self, x_train: np.ndarray, y_train: np.ndarray) -> None:
        
        X = PolynomialRegression._design_matrix(x_train, self.degree)  
        _, cols = X.shape
        if np.linalg.matrix_rank(X) < cols:
            raise ArithmeticError(
                "Design matrix is not full rank, can't compute coefficients"
            )

        
        self.params = np.linalg.pinv(X) @ y_train

    def predict(self, data: np.ndarray) -> np.ndarray:
        
        if self.params is None:
            raise ArithmeticError("Predictor hasn't been fit yet")

        return PolynomialRegression._design_matrix(data, self.degree) @ self.params


def main() -> None:
    
    import seaborn as sns

    mpg_data = sns.load_dataset("mpg")

    poly_reg = PolynomialRegression(degree=2)
    poly_reg.fit(mpg_data.weight, mpg_data.mpg)

    weight_sorted = np.sort(mpg_data.weight)
    predictions = poly_reg.predict(weight_sorted)

    plt.scatter(mpg_data.weight, mpg_data.mpg, color="gray", alpha=0.5)
    plt.plot(weight_sorted, predictions, color="red", linewidth=3)
    plt.title("Predicting Fuel Efficiency Using Polynomial Regression")
    plt.xlabel("Weight (lbs)")
    plt.ylabel("Fuel Efficiency (mpg)")
    plt.show()


if __name__ == "__main__":
    import doctest

    doctest.testmod()

    main()