<jupyter_start><jupyter_text>DataSciCon.tech  IMPORTS<jupyter_code>%matplotlib inline

from datetime import date, datetime, timedelta
from math import log

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from stemgraphic import stem_graphic
import seaborn as sns<jupyter_output><empty_output><jupyter_text>Data Loading the data<jupyter_code>df = pd.read_csv('home_data.csv')
df.head()
df.columns<jupyter_output><empty_output><jupyter_text>Preliminary data munging<jupyter_code>df['renovated'] = df.yr_renovated>=1
df.drop('date',axis=1, inplace=True)
df.shape<jupyter_output><empty_output><jupyter_text>Simulating invalid data<jupyter_code>import random

random.seed(42)
PCT = 0.01
i = [random.choice(range(df.shape[0]-1)) for _ in range(int(df.shape[0] * PCT))]
df.iloc[i, 17] = df.iloc[i, 17].abs()<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code>sns.regplot(x='long', y='price', data=df, line_kws={'color':'r'});
df['lat_from_center_eng_'] = (47.62 - df.lat).abs()
df['long_from_center_eng_'] = (-122.32 - df.long).abs()
sns.regplot(x='lat_from_center_eng_', y='price', data=df, line_kws={'color':'r'});
df['sqft_living_log_eng_'] = df['sqft_living'].apply(log)
df.drop(['lat','long'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Model Multivariate Linear regression<jupyter_code>from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
X_all = df.copy()
#X_all.iloc[i, j] = np.NaN
#X_all.iloc[i, 19] = np.NaN
import missingno as msno
msno.matrix(X_all)<jupyter_output><empty_output><jupyter_text>imputation<jupyter_code>#X_all.dropna(how="any", inplace=True)
#X_all.fillna(0, inplace=True)
#X_all.fillna(X_all.mean(), inplace=True)
#X_all.fillna(X_all.median(), inplace=True)  # strawman imputation for continuous<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code>from sklearn.feature_selection import RFE
lr = LinearRegression()
rfe = RFE(lr, 9)
rfe = rfe.fit(X_all.drop("price", axis=1), X_all.price)
print(X_all.drop("price", axis=1).columns[rfe.support_])
print(rfe.ranking_)
print(rfe.support_)
X = X_all[X_all.drop("price", axis=1).columns[rfe.support_]]<jupyter_output><empty_output><jupyter_text>Train / test split<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, X_all.price, test_size=0.25, random_state=42)
X_train.shape<jupyter_output><empty_output><jupyter_text>Train the model<jupyter_code>lr = LinearRegression()
lr.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>MetricsStoring a few extra piece of informations in our model<jupyter_code>lr.version = datetime.today()
lr.cv = lr.score(X_test, y_test)
print(lr.version, lr.cv)<jupyter_output>2018-11-29 16:27:39.089864 0.6680494399956315<jupyter_text>Visualisation<jupyter_code>ax = plt.scatter(lr.predict(X_train), lr.predict(X_train) - y_train, c='r', s=15, alpha=0.2)
plt.scatter(lr.predict(X_test), lr.predict(X_test) - y_test, c='b', s=15, alpha=0.2)<jupyter_output><empty_output><jupyter_text>Model coefficients<jupyter_code>pd.set_option('display.float_format', lambda x: '%.3f' % x)
lr.coef_
lr.intercept_
X.columns
house1 = X[-1:]
print(lr.predict(house1))
print("---------------")
print( house1, X_all[-1:]["price"])<jupyter_output>[345756.93572019]
---------------
       bathrooms  floors  waterfront  view  condition  grade  renovated  \
21612      0.750   2.000           0     0          3      7      False   

       lat_from_center_eng_  sqft_living_log_eng_  
21612                 0.026                 6.928   21612    325000
Name: price, dtype: int64<jupyter_text>Not only did the order of features change, but the features themselves are different. Just because 1% of the data has the wrong geocoding. Due to this, the next line, which worked with the uncorrupted data, will now error out.<jupyter_code>print(lr.intercept_
      + house1.bathrooms * lr.coef_[0]
      + house1.waterfront * lr.coef_[1]
      + house1.view * lr.coef_[2]
      + house1.condition * lr.coef_[3]
      + house1.grade * lr.coef_[4]
      + house1.renovated * lr.coef_[5]
      + house1.lat_from_center_eng_ * lr.coef_[6] 
      + house1.long_from_center_eng_ * lr.coef_[7]
      + house1.sqft_living_log_eng_ * lr.coef_[8])
pd.Series(dict(zip(X.columns, lr.coef_))).sort_values()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14551296763828322, 'fraction_numerical': 0.03741106265779206, 'mean_word_length': 5.746130030959752, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '221822', 'license': 'MIT', 'n_tokens_mistral': 1823, 'n_tokens_neox': 1722, 'n_words': 278, 'path': 'notebooks/03-validity.ipynb', 'repo_name': 'fdion/dataquality_atlanta', 'size': '4357'}