<jupyter_start><jupyter_text>Chapter 3 Non-linear transformations of the predictorsWe can extend our linear fitting capabilities by e.g fitting to an X^2 term `y ~ B_0 + B_1*x + B_2*x^2`<jupyter_code>import statsmodels.api as sm
import statsmodels.formula.api as smf
import statsmodels.stats.anova
from mpl_toolkits import mplot3d
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
%matplotlib inline

sns.set(style="whitegrid")
boston = sm.datasets.get_rdataset("Boston", "MASS").data
boston.describe()
# Here we use the beefier "statsmodels" package to do the analysis that R would 
# do for us, giving the errors, t-values, p-values, etc.

# statsmodels doesn't assume there'll be an intercept, so you have to add one
X_train =  sm.add_constant(boston)

# the below formula is a regression on lstat, including an lstat^2 term  
model = smf.ols('medv ~ lstat + np.square(lstat)', data=boston) # == lstat+age+lstat:age
results = model.fit()
print(results.summary())
# p-value for F-statistic and p-value for lstat^2 term both tiny, suggesting this is a valid inclusion


B0, B1, B2 = results.params

fig, ax = plt.subplots(figsize=(8,8))
plt.scatter(boston.lstat, boston.medv,  color='black', alpha=0.25)
fitted_data = zip(boston.lstat, B0+boston.lstat*B1 + np.square(boston.lstat)*B2)
fitted_data = sorted(fitted_data, key=lambda x: x[0])
x,y = zip(*fitted_data)
plt.plot(x, y, color='blue', linewidth=3)
plt.xlabel("LSTAT")
plt.ylabel("MEDV")
plt.show()
first_order_model = smf.ols('medv ~ lstat ', data=boston) # == lstat+age+lstat:age
results = first_order_model.fit()

statsmodels.stats.anova.anova_lm(first_order_model.fit(), model.fit())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12433706540954625, 'fraction_numerical': 0.01296405421331762, 'mean_word_length': 5.68503937007874, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '238342', 'license': 'MIT', 'n_tokens_mistral': 635, 'n_tokens_neox': 615, 'n_words': 169, 'path': 'Chapter3/NonLinearTransforms.ipynb', 'repo_name': 'wllgrnt/islr-python-examples', 'size': '1697'}