<jupyter_start><jupyter_text>- [X] Look at the data. What's the distribution of the target, `price`, and features such as `longitude` and `latitude`? Remove outliers.- [X] After you remove outliers, what is the mean price in your subset of the data?- [X] Choose a feature, and plot its relationship with the target.- [X] Use scikit-learn for linear regression with one feature. You can follow the [5-step process from <NAME>](https://jakevdp.github.io/PythonDataScienceHandbook/05.02-introducing-scikit-learn.htmlBasics-of-the-API).- [X] Define a function to make new predictions and explain the model coefficient.- [X] Organize and comment your code.<jupyter_code>import pandas as pd
import pandas_profiling
import plotly.express as px
import numpy as np
from functools import *
from pyrthon import *
from pyrsistent import *
from datetime import datetime
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from IPython import get_ipython
from nbformat import read
from IPython.core.interactiveshell import InteractiveShell
import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning, module='numpy')
appstate = {}
appstate_history = v()
print(appstate)
# utility functions
def comp(*functions):
    """comp: compose functions"""
    return reduce(lambda f, g: lambda x: f(g(x)), functions, lambda x: x)

def Map(func, iterable):
    return list(map(func, iterable))

def DictMerge(a, b):
    return {**a, **b}

def ChainDictMerge(a):
    return reduce(lambda x, y: DictMerge(x, y), a)

def rest(x):
    return x[1:]

def first(x):
    return x[0]
    
def last(x):
    return x[-1]

def butlast(x):
    return x[:-1]

def identity(x):
    return x

def is_empty(x):
    return len(x)==0

def not_empty(x):
    return converse(is_empty)(x)

def converse(fn):
    return lambda *args, **kwargs: not fn(*args, **kwargs)
# functions which do neither
def WriteProfileReport(df, title=''):
    profile = df.profile_report(title=title)
    profile.to_file(output_file="output__{}.html".format(title))
    return profile
# other functions which work within the context of an appstate

def GetZMatrix(df):
    scaler = StandardScaler()
    Z = pd.DataFrame(scaler.fit_transform(df.values), columns=df.columns)
    return Z

def ApplyDF(df, *args, **kwargs):
    fn     = first(args)
    args   = rest(args)
    result = fn(df, *args, **kwargs)
    return fn(df, *args, **kwargs)

def FilterDF(df):
    return (df[(df['price'] >= np.percentile(df['price'], 0.5)) & 
        (df['price'] <= np.percentile(df['price'], 99.5)) & 
        (df['latitude'] >= np.percentile(df['latitude'], 0.05)) & 
        (df['latitude'] < np.percentile(df['latitude'], 99.95)) &
        (df['longitude'] >= np.percentile(df['longitude'], 0.05)) & 
        (df['longitude'] <= np.percentile(df['longitude'], 99.95))])
# other people's
def regression_3d(df, x, y, z, **kwargs):
    """
    Visualize linear regression in 3D: 2 features + 1 target
    
    df : Pandas DataFrame
    x : string, feature 1 column in df
    y : string, feature 2 column in df
    z : string, target column in df
    """
    
    # Plot data
    fig = px.scatter_3d(df, x, y, z, **kwargs)
    
    # Fit Linear Regression
    features = [x, y]
    target = z
    model = LinearRegression()
    model.fit(df[features], df[target])    
    
    # Define grid of four points in the feature space
    xmin, xmax = df[x].min(), df[x].max()
    ymin, ymax = df[y].min(), df[y].max()
    coords = [[xmin, ymin], 
              [xmin, ymax], 
              [xmax, ymin], 
              [xmax, ymax]]
    
    # Make predictions for the grid
    Z = model.predict(coords).reshape((2,2), order='F')
    
    # Plot predictions as a 3D surface (plane)
    fig.add_trace(go.Surface(x=[xmin,xmax], y=[ymin,ymax], z=Z))
    
    return fig


def regression_residuals(df, feature, target, m, b):
    """
    Visualize linear regression, with residual errors,
    in 2D: 1 feature + 1 target.
    
    Use the m & b parameters to "fit the model" manually.
    
    df : Pandas DataFrame
    feature : string, feature column in df
    target : string, target column in df
    m : numeric, slope for linear equation
    b : numeric, intercept for linear requation
    """
    
    # Plot data
    df.plot.scatter(feature, target)
    
    # Make predictions
    x = df[feature]
    y = df[target]
    y_pred = m*x + b
    
    # Plot predictions
    plt.plot(x, y_pred)
    
    # Plot residual errors
    for x, y1, y2 in zip(x, y, y_pred):
        plt.plot((x, x), (y1, y2), color='grey')
    
    # Print regression metrics
    mae = mean_absolute_error(y, y_pred)
    r2 = r2_score(y, y_pred)
    print('Mean Absolute Error:', mae)
    print('R^2:', r2)


def regression_squared_errors(df, feature, target, m, b):
    """
    Visualize linear regression, with squared errors,
    in 2D: 1 feature + 1 target.
    
    Use the m & b parameters to "fit the model" manually.
    
    df : Pandas DataFrame
    feature : string, feature column in df
    target : string, target column in df
    m : numeric, slope for linear equation
    b : numeric, intercept for linear requation
    """
    
    # Plot data
    fig = plt.figure(figsize=(7,7))
    ax = plt.axes()
    df.plot.scatter(feature, target, ax=ax)
    
    # Make predictions
    x = df[feature]
    y = df[target]
    y_pred = m*x + b
    
    # Plot predictions
    ax.plot(x, y_pred)
    
    # Plot squared errors
    xmin, xmax = ax.get_xlim()
    ymin, ymax = ax.get_ylim()
    scale = (xmax-xmin)/(ymax-ymin)
    for x, y1, y2 in zip(x, y, y_pred):
        bottom_left = (x, min(y1, y2))
        height = abs(y1 - y2)
        width = height * scale
        ax.add_patch(Rectangle(xy=bottom_left, width=width, height=height, alpha=0.1))
    
    # Print regression metrics
    mse = mean_squared_error(y, y_pred)
    rmse = np.sqrt(mse)
    mae = mean_absolute_error(y, y_pred)
    r2 = r2_score(y, y_pred)
    print('Mean Squared Error:', mse)
    print('Root Mean Squared Error:', rmse)
    print('Mean Absolute Error:', mae)
    print('R^2:', r2)
    
# Credit: <NAME>, Python Data Science Handbook, Chapter 5.3
# https://jakevdp.github.io/PythonDataScienceHandbook/05.03-hyperparameters-and-model-validation.html#Validation-curves-in-Scikit-Learn
def PolynomialRegression(degree=2, **kwargs):
    return make_pipeline(PolynomialFeatures(degree), 
                         LinearRegression(**kwargs))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1255877445775823, 'fraction_numerical': 0.011224025481571363, 'mean_word_length': 3.149779735682819, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '630320', 'license': 'MIT', 'n_tokens_mistral': 2288, 'n_tokens_neox': 2183, 'n_words': 655, 'path': 'script/Code.ipynb', 'repo_name': 'ross-fisher/DS-Unit-2-Regression-Classification', 'size': '6611'}