locally weighted linear regression also called local regression is a type of nonparametric linear regression that prioritizes data closest to a given prediction point the algorithm estimates the vector of model coefficients using weighted least squares regression xwxxwy where x is the design matrix y is the response vector and w is the diagonal weight matrix this implementation calculates w the weight of the ith training sample using the gaussian weight w expx x2 where x is the ith training sample x is the prediction point is the bandwidth and x is the euclidean norm also called the 2norm or the l norm the bandwidth controls how quickly the weight of a training sample decreases as its distance from the prediction point increases one can think of the gaussian weight as a bell curve centered around the prediction point a training sample is weighted lower if it s farther from the center and controls the spread of the bell curve other types of locally weighted regression such as locally estimated scatterplot smoothing loess typically use different weight functions references https en wikipedia orgwikilocalregression https en wikipedia orgwikiweightedleastsquares https cs229 stanford edunotes2022fallmainnotes pdf calculate the weight of every point in the training data around a given prediction point args point xvalue at which the prediction is being made xtrain ndarray of xvalues for training tau bandwidth value controls how quickly the weight of training values decreases as the distance from the prediction point increases returns m x m weight matrix around the prediction point where m is the size of the training set weightmatrix np array1 1 np array16 99 10 34 21 01 23 68 24 59 25 69 0 6 array1 43807972e207 0 00000000e000 0 00000000e000 0 00000000e000 0 00000000e000 0 00000000e000 0 00000000e000 0 00000000e000 0 00000000e000 calculate the local weights at a given prediction point using the weight matrix for that point args point xvalue at which the prediction is being made xtrain ndarray of xvalues for training ytrain ndarray of yvalues for training tau bandwidth value controls how quickly the weight of training values decreases as the distance from the prediction point increases returns ndarray of local weights localweight np array1 1 np array16 99 10 34 21 01 23 68 24 59 25 69 np array1 01 1 66 3 5 0 6 array0 00873174 0 08272556 calculate predictions for each point in the training data args xtrain ndarray of xvalues for training ytrain ndarray of yvalues for training tau bandwidth value controls how quickly the weight of training values decreases as the distance from the prediction point increases returns ndarray of predictions localweightregression np array16 99 10 34 21 01 23 68 24 59 25 69 np array1 01 1 66 3 5 0 6 array1 07173261 1 65970737 3 50160179 load data from seaborn and split it into x and y points pass no doctests function is for demo purposes only pairing elements of one and xdata plot predictions and display the graph pass no doctests function is for demo purposes only demo with a dataset from the seaborn module calculate the weight of every point in the training data around a given prediction point args point x value at which the prediction is being made x_train ndarray of x values for training tau bandwidth value controls how quickly the weight of training values decreases as the distance from the prediction point increases returns m x m weight matrix around the prediction point where m is the size of the training set weight_matrix np array 1 1 np array 16 99 10 34 21 01 23 68 24 59 25 69 0 6 array 1 43807972e 207 0 00000000e 000 0 00000000e 000 0 00000000e 000 0 00000000e 000 0 00000000e 000 0 00000000e 000 0 00000000e 000 0 00000000e 000 number of training samples initialize weights as identity matrix calculate the local weights at a given prediction point using the weight matrix for that point args point x value at which the prediction is being made x_train ndarray of x values for training y_train ndarray of y values for training tau bandwidth value controls how quickly the weight of training values decreases as the distance from the prediction point increases returns ndarray of local weights local_weight np array 1 1 np array 16 99 10 34 21 01 23 68 24 59 25 69 np array 1 01 1 66 3 5 0 6 array 0 00873174 0 08272556 calculate predictions for each point in the training data args x_train ndarray of x values for training y_train ndarray of y values for training tau bandwidth value controls how quickly the weight of training values decreases as the distance from the prediction point increases returns ndarray of predictions local_weight_regression np array 16 99 10 34 21 01 23 68 24 59 25 69 np array 1 01 1 66 3 5 0 6 array 1 07173261 1 65970737 3 50160179 initialize array of predictions load data from seaborn and split it into x and y points pass no doctests function is for demo purposes only pairing elements of one and x_data plot predictions and display the graph pass no doctests function is for demo purposes only demo with a dataset from the seaborn module
import matplotlib.pyplot as plt
import numpy as np


def weight_matrix(point: np.ndarray, x_train: np.ndarray, tau: float) -> np.ndarray:
    
    m = len(x_train)  
    weights = np.eye(m)  
    for j in range(m):
        diff = point - x_train[j]
        weights[j, j] = np.exp(diff @ diff.T / (-2.0 * tau**2))

    return weights


def local_weight(
    point: np.ndarray, x_train: np.ndarray, y_train: np.ndarray, tau: float
) -> np.ndarray:
    
    weight_mat = weight_matrix(point, x_train, tau)
    weight = np.linalg.inv(x_train.T @ weight_mat @ x_train) @ (
        x_train.T @ weight_mat @ y_train.T
    )

    return weight


def local_weight_regression(
    x_train: np.ndarray, y_train: np.ndarray, tau: float
) -> np.ndarray:
    
    y_pred = np.zeros(len(x_train))  
    for i, item in enumerate(x_train):
        y_pred[i] = np.dot(item, local_weight(item, x_train, y_train, tau)).item()

    return y_pred


def load_data(
    dataset_name: str, x_name: str, y_name: str
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
    
    import seaborn as sns

    data = sns.load_dataset(dataset_name)
    x_data = np.array(data[x_name])
    y_data = np.array(data[y_name])

    one = np.ones(len(y_data))

    
    x_train = np.column_stack((one, x_data))

    return x_train, x_data, y_data


def plot_preds(
    x_train: np.ndarray,
    preds: np.ndarray,
    x_data: np.ndarray,
    y_data: np.ndarray,
    x_name: str,
    y_name: str,
) -> None:
    
    x_train_sorted = np.sort(x_train, axis=0)
    plt.scatter(x_data, y_data, color="blue")
    plt.plot(
        x_train_sorted[:, 1],
        preds[x_train[:, 1].argsort(0)],
        color="yellow",
        linewidth=5,
    )
    plt.title("Local Weighted Regression")
    plt.xlabel(x_name)
    plt.ylabel(y_name)
    plt.show()


if __name__ == "__main__":
    import doctest

    doctest.testmod()

    
    training_data_x, total_bill, tip = load_data("tips", "total_bill", "tip")
    predictions = local_weight_regression(training_data_x, tip, 5)
    plot_preds(training_data_x, predictions, total_bill, tip, "total_bill", "tip")