<jupyter_start><jupyter_text>Diagnostic of models<jupyter_code>import pandas as pd 
import statsmodels.formula.api as smf
import matplotlib.pyplot as plt
% matplotlib inline
housing = pd.DataFrame.from_csv('../data/housing.csv')
housing.head()
model = smf.ols(formula='MEDV~LSTAT', data=housing).fit()

# Here are estimated intercept and slope by least square estimation 
b0_ols = model.params[0]
b1_ols = model.params[1]

housing['BestResponse'] = b0_ols + b1_ols*housing['LSTAT']<jupyter_output><empty_output><jupyter_text>Assumptions behind linear regression model1. Linearity 2. independence3. Normality4. Equal Variance  Linearity<jupyter_code># you can check the scatter plot to have a fast check
housing.plot(kind='scatter', x='LSTAT', y='MEDV', figsize=(10, 10), color='g')<jupyter_output><empty_output><jupyter_text>Independence<jupyter_code># Get all errors (residuals)
housing['error'] = housing['MEDV'] - housing['BestResponse']
# Method 1: Residual vs order plot
# error vs order plot (Residual vs order) as a fast check 
plt.figure(figsize=(15, 8))
plt.title('Residual vs order')
plt.plot(housing.index, housing['error'], color='purple')
plt.axhline(y=0, color='red')
plt.show()
# Method 2: Durbin Watson Test
# Check the Durbin Watson Statistic
# Rule of thumb: test statistic value in the range of 1.5 to 2.5 are relatively normal
model.summary()<jupyter_output><empty_output><jupyter_text>Normality<jupyter_code>import scipy.stats as stats
z = (housing['error'] - housing['error'].mean())/housing['error'].std(ddof=1)

stats.probplot(z, dist='norm', plot=plt)
plt.title('Normal Q-Q plot')
plt.show()<jupyter_output><empty_output><jupyter_text>Equal variance<jupyter_code># Residual vs predictor plot
housing.plot(kind='scatter', x='LSTAT', y='error', figsize=(15, 8), color='green')
plt.title('Residual vs predictor')
plt.axhline(y=0, color='red')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14330708661417324, 'fraction_numerical': 0.015223097112860892, 'mean_word_length': 7.433628318584071, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '61295', 'license': 'Apache-2.0', 'n_tokens_mistral': 706, 'n_tokens_neox': 672, 'n_words': 159, 'path': 'HKUST  - Python and Statistics for Financial Analysis/module4- Linear Regression Models for Financial Analysis/Diagnostic+of+models.ipynb', 'repo_name': 'sasdevs/Courses-', 'size': '1905'}