<jupyter_start><jupyter_text>Assignment 2- Evaluating Regression Modelsauthors:  Objective: We are creating an explanatory analysis for our predictions using at least two regression modeling methods from our textbook Chapter 4 of the Géron (2017), Lasso Regression and Linear Regression to determine which machine learning method the firm wants to use to complement conventional methods for assessing the market value of residential real estate. i.e SalePrice<jupyter_code>import pandas as pd
from sklearn import linear_model
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
import seaborn as sns
import numpy as np
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
train = pd.read_csv("Data/train.csv")
test = pd.read_csv("Data/test.csv")
train.info()
train.head()
train.tail()
train.describe()
train.shape
train.set_index(keys = ["Id"], inplace = True)
# this is going to list all the columns
cols = list(train.columns)
cols.sort()
cols.append(" ") # rounding from 80 to 81 in anticipation for matrix
num = np.array(cols)
# reshaping into matrix
reshaped = num.reshape(9,9)

#now construct a prettier table
pd.DataFrame(reshaped, columns=["" for i in range(0,9)])
proportions = 100 * (round(train.isnull().sum() / len(train), 4))

pd.set_option("display.max_rows", 80, "display.max_columns", 80) 
proportions

pd.reset_option("display.max_rows", "display.max_columns")
proportions

proportions[proportions > 30]
"""
 we will definitely disclude any of the columns listed above for being mostly missings
 further we will take columns we believe to be most helpful for predicting sales price 
 based on our own experience and expertise

"""
mine = ["YearBuilt", "TotRmsAbvGrd", "RoofMatl", "PoolArea", "LotArea", "Neighborhood", "YearRemodAdd", \
        "GarageYrBlt", "HouseStyle", "CentralAir", "Fireplaces", "SalePrice"]
%time seaborn_plot = sns.pairplot(train[mine])
train.drop(columns = ["GarageYrBlt"], inplace = True)
mine.remove("GarageYrBlt")
%time pd_plot = pd.plotting.scatter_matrix(train[mine], alpha = 0.2, figsize = (15, 15))
df = train[mine]
df.head()
# here we see we have no missings
round(df.describe(), 2)
# this pool looks really big so we'll isolate large pools to see if it makes sense
df[df.PoolArea > 600] # it does
# some categorical and some numerical variables
df.dtypes
predictors = mine
predictors.remove("SalePrice")
predicted = ["SalePrice"]
df.head()
pd.get_dummies(df.CentralAir)
pd.get_dummies(df.HouseStyle)
pd.get_dummies(df.RoofMatl)
pd.get_dummies(df.Neighborhood)
predictors.remove("Neighborhood")
predictors.remove("RoofMatl")
predictors.remove("HouseStyle")
central_dummies = pd.get_dummies(df.CentralAir)
df.drop(columns = ["Neighborhood", "RoofMatl", "HouseStyle", "CentralAir"], inplace = True)
df = df.join(central_dummies)
df.rename(columns = {"N": "CentralAir_N", "Y": "CentralAir_Y"}, inplace=True)
df
predictors.remove("CentralAir")
predictors.append("CentralAir_N")
predictors.append("CentralAir_Y")
model = LinearRegression()
scores = []
X = df[predictors]
y = df[predicted]
kfold = KFold(n_splits=10, shuffle=True, random_state=42)
for i, (train, test) in enumerate(kfold.split(X, y)):
    model.fit(X.iloc[train,:], y.iloc[train,:])
    scores.append(round(model.score(X.iloc[test,:], y.iloc[test,:]), 4))
print(scores)
mses = []
kfold = KFold(n_splits=3, shuffle=True, random_state=42)
for i, (train, test) in enumerate(kfold.split(X, y)):
    model.fit(X.iloc[train,:], y.iloc[train,:])
    predicted_y = model.predict(X) # yielding a list of SalePrice predictions
    train_mse = mean_squared_error(y, predicted_y) # y_true, y_pred
    mses.append(train_mse)
print(mses)
n_samples, n_features = 10, 5
rng = np.random.RandomState(0)
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
model = Ridge(alpha=0.4)
scores = []
mses = []
X = df[predictors]
y = df[predicted]
# X and y are same
kfold = KFold(n_splits=10, shuffle=True, random_state=42)
for i, (train, test) in enumerate(kfold.split(X, y)):
    model.fit(X.iloc[train,:], y.iloc[train,:])
    scores.append(round(model.score(X.iloc[test,:], y.iloc[test,:]), 4))
print(scores)
kfold = KFold(n_splits=3, shuffle=True, random_state=42)
for i, (train, test) in enumerate(kfold.split(X, y)):
    model.fit(X.iloc[train,:], y.iloc[train,:])
    predicted_y = model.predict(X) # yielding a list of SalePrice predictions
    train_mse = mean_squared_error(y, predicted_y) # y_true, y_pred
    mses.append(train_mse)
print(mses)<jupyter_output>[2446629903.867889, 2424867009.897142, 2419106780.0362687]
{'fraction_non_alphanumeric': 0.13435114503816795, 'fraction_numerical': 0.022900763358778626, 'mean_word_length': 5.9696048632218845, 'pattern_counts': {'":': 2, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '163477', 'license': 'MIT', 'n_tokens_mistral': 1741, 'n_tokens_neox': 1607, 'n_words': 421, 'path': 'Fall2020/MSDS422/house-advanced/Assignment 2 - House Prices Advanced Regression Techniques- Kaggle.ipynb', 'repo_name': 'kbridge14/regression_things', 'size': '4585'}