<jupyter_start><jupyter_text>Dummies<jupyter_code>print(df.waterfront.nunique())
print(df.grade.nunique())
print(df.condition.nunique())
print(df.view.nunique())

df.shape
#waterfront_dummies = pd.get_dummies(df['waterfront'], prefix='wf', drop_first=True)
#grade_dummies = pd.get_dummies(df['grade'], prefix='gr', drop_first=True)
#condition_dummies = pd.get_dummies(df['condition'], prefix='cd', drop_first=True)
#view_dummies = pd.get_dummies(df['view'], prefix='vw', drop_first=True)
#view_dummies.shape<jupyter_output><empty_output><jupyter_text>Remove the original columns from df and add the dummy variables<jupyter_code>#df = df.drop(['grade','condition', 'view'], axis=1)
#df = pd.concat([df, grade_dummies, condition_dummies, view_dummies ], axis=1)
#df.shape<jupyter_output><empty_output><jupyter_text>Heatmap for correlations<jupyter_code>plt.subplots(figsize=(17,14))
sns.heatmap(df.corr(),annot=True,linewidths=0.5,linecolor="Black",fmt="1.1f")
plt.title("Data Correlation",fontsize=50)
plt.show()<jupyter_output><empty_output><jupyter_text>Explanatory Variables<jupyter_code>explanatory_vars = list(df.columns)
explanatory_vars

print('R squared for each possible explanatory variable:' )
for explanatory_var in explanatory_vars:
    model = 'price ~ {:}'.format(explanatory_var)
    rs = smf.ols(formula=model, data=df).fit().rsquared
    print('{:7s} \t \t= {:.3}'.format(explanatory_var, rs))
    
# sales predicting salesis included as explanatory variable for sanity check
df.shape<jupyter_output><empty_output><jupyter_text>Split the data in training and test set<jupyter_code># dropna because otherwise an errormesssage came up
df = df.dropna()
df.shape
y = df.price
x = df


xTrain, xTest, yTrain, yTest = train_test_split(x, y, test_size = 0.2, random_state = 0)
xTrain.shape<jupyter_output><empty_output><jupyter_text>Outliers<jupyter_code>xTrain = xTrain.query("bedrooms < 33")
xTrain = xTrain.query("price < 1970.0")
xTrain = xTrain.query("sqft_lot < 107.157")
xTrain = xTrain.query("sqft_lot15 < 96.34557399999838")

yTrain = xTrain.price


xTrain = xTrain.drop(['price'], axis=1)




print(xTrain.shape)
print(yTrain.shape)
xTrain.tail(5)<jupyter_output><empty_output><jupyter_text>Find the best fitting line<jupyter_code>model1 = smf.ols(formula='yTrain ~  bedrooms + bathrooms + sqft_living + sqft_lot + floors +  sqft_above + sqft_basement + yr_built + yr_renovated + zipcode + lat + long + sqft_living15 + sqft_lot15 + waterfront + view + condition + grade ', data=xTrain).fit()
model1.summary()<jupyter_output><empty_output><jupyter_text>remove sqft_lot<jupyter_code>model2 = smf.ols(formula='yTrain ~  bedrooms + bathrooms + sqft_living  + +floors + sqft_above + sqft_basement + yr_built + yr_renovated + zipcode + lat + long + sqft_living15 + sqft_lot15 + waterfront + view + grade+ condition', data=xTrain).fit()

model2.summary()<jupyter_output><empty_output><jupyter_text>remove sqft_above and sqft_living15 beacause of multicollinearity<jupyter_code>model3 = smf.ols(formula='yTrain ~  bedrooms + bathrooms + sqft_living  + sqft_basement + yr_built + yr_renovated + zipcode + lat + long  + sqft_lot15 + waterfront + view + grade+ condition', data=xTrain).fit()
model3.summary()<jupyter_output><empty_output><jupyter_text>Remove categorials<jupyter_code>model4 = smf.ols(formula='yTrain ~  bedrooms + bathrooms + sqft_living  + sqft_basement + yr_built + yr_renovated + zipcode + lat + long  + sqft_lot15', data=xTrain).fit()
model4.summary()<jupyter_output><empty_output><jupyter_text>Added grade because of high r squared value<jupyter_code>model5 = smf.ols(formula='yTrain ~  bedrooms + bathrooms + sqft_living  + sqft_basement + yr_built +   zipcode + lat + long  + sqft_lot15 + grade ', data=xTrain).fit()
model5.summary()
model6 = smf.ols(formula='yTrain ~  bedrooms + bathrooms + sqft_living   + yr_built + yr_renovated + zipcode + lat + long  + sqft_lot15 + grade ', data=xTrain).fit()
model6.summary()<jupyter_output><empty_output><jupyter_text>Performance on train set<jupyter_code>y_train_pred1 = model1.predict(xTrain)
y_train_pred2 = model2.predict(xTrain)
y_train_pred3 = model3.predict(xTrain)
y_train_pred4 = model4.predict(xTrain)
y_train_pred5 = model5.predict(xTrain)
y_train_pred6 = model6.predict(xTrain)
from sklearn.metrics import mean_squared_error
from math import sqrt

rmse1 = sqrt(mean_squared_error(yTrain, y_train_pred1))
rmse2 = sqrt(mean_squared_error(yTrain, y_train_pred2))
rmse3 = sqrt(mean_squared_error(yTrain, y_train_pred3))
rmse4 = sqrt(mean_squared_error(yTrain, y_train_pred4))
rmse5 = sqrt(mean_squared_error(yTrain, y_train_pred5))
rmse6 = sqrt(mean_squared_error(yTrain, y_train_pred6))
#statsmodels.tools.eval_measures.rmse(y_train_pred, yTrain)
print("rmse1: ", rmse1)
print("rmse2: ", rmse2)
print("rmse3: ", rmse3)
print("rmse4: ", rmse4)
print("rmse5: ", rmse5)
print("rmse6: ", rmse6)<jupyter_output>rmse1:  151.01548342484242
rmse2:  151.01666609610567
rmse3:  152.29065320820044
rmse4:  174.03804212029442
rmse5:  159.99314534029446
rmse6:  159.7675912102382<jupyter_text>Performance on test set<jupyter_code>y_test_pred1 = model1.predict(xTest)
y_test_pred2 = model2.predict(xTest)
y_test_pred3 = model3.predict(xTest)
y_test_pred4 = model4.predict(xTest)
y_test_pred5 = model5.predict(xTest)
y_test_pred6 = model6.predict(xTest)
rmse1 = sqrt(mean_squared_error(yTest, y_test_pred1))
rmse2 = sqrt(mean_squared_error(yTest, y_test_pred2))
rmse3 = sqrt(mean_squared_error(yTest, y_test_pred3))
rmse4 = sqrt(mean_squared_error(yTest, y_test_pred4))
rmse5 = sqrt(mean_squared_error(yTest, y_test_pred5))
rmse6 = sqrt(mean_squared_error(yTest, y_test_pred6))
#statsmodels.tools.eval_measures.rmse(y_train_pred, yTrain)
print("rmse1: ", rmse1)
print("rmse2: ", rmse2)
print("rmse3: ", rmse3)
print("rmse4: ", rmse4)
print("rmse5: ", rmse5)
print("rmse6: ", rmse6)<jupyter_output>rmse1:  201.84898545305853
rmse2:  201.9882398232645
rmse3:  200.6774327344447
rmse4:  221.77977875203797
rmse5:  208.82956441170677
rmse6:  208.66433543159675
{'fraction_non_alphanumeric': 0.12995850622406638, 'fraction_numerical': 0.06058091286307054, 'mean_word_length': 7.784256559766764, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '556984', 'license': 'MIT', 'n_tokens_mistral': 2662, 'n_tokens_neox': 2352, 'n_words': 374, 'path': 'LinReg.ipynb', 'repo_name': 'kevintomas1995/Kingcounty_EDA', 'size': '6025'}