<jupyter_start><jupyter_text>Percentiles<jupyter_code>samples = np.random.normal(size=10000)
patches = plt.hist(samples, bins=100, color='salmon', alpha=0.3) 
plt.plot(patches[1][:-1], patches[0], c='orangered')
plt.axvline(samples.mean(), c='b')
from scipy.ndimage import filters

patches = plt.hist(samples, bins=100, color='salmon', alpha=0.3) 

smoothed = filters.gaussian_filter1d(patches[0].copy(), sigma=2.5)
plt.plot(patches[1][:-1], smoothed, c='orangered')
plt.plot(patches[1][:-1], smoothed, c='orangered')
plt.axvline(samples.mean()/2)

fiftyth_percentile = samples.mean()
ninty_fifth_percentile = (fiftyth_percentile/5) * 9 + (fiftyth_percentile/10)
fifth_percentile = fiftyth_percentile/10

plt.axvline(samples.mean()+samples.std())
plt.axvline(samples.mean()-samples.std())
plt.axvline(samples.mean()+(samples.std()*2))
plt.axvline(samples.mean()-(samples.std()*2))
plt.axvline(samples.mean()+(samples.std()*3))
plt.axvline(samples.mean()-(samples.std()*3))<jupyter_output><empty_output><jupyter_text>Generating Non-Linear Random Data<jupyter_code>lower_bounds = numpy.random.normal(50, size=100)
standard_deviations = numpy.random.uniform(80, size=100)

X = numpy.random.uniform(lower_bounds, size=100)
Y = 1 + (X**2) + numpy.random.normal(loc=4, scale=standard_deviations)
fig, axes = plt.subplots(1, 2, figsize=(13, 5))

axes[0].set_title("X Lower Bounds")
sns.distplot(lower_bounds, ax=axes[0]
            )
axes[1].set_title("Y Standard Deviations")
sns.distplot(standard_deviations, ax=axes[1])

sns.jointplot(X, Y)<jupyter_output>/opt/conda/lib/python3.6/site-packages/scipy/stats/stats.py:1713: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
  return np.add.reduce(sorted[indexer] * weights, axis=axis) / sumval<jupyter_text>Structuring Data<jupyter_code>data = pd.DataFrame({'X': X, 'Y': Y})
model = statsmodels.regression.quantile_regression.QuantReg(Y, X).fit()<jupyter_output><empty_output><jupyter_text>StatsModel OLS<jupyter_code>model = sm.OLS(Y, X).fit()
model.summary()
plt.scatter(X, Y)
plt.plot(X, model.predict(X), c='r')<jupyter_output><empty_output><jupyter_text>SciKit-Learn Linear Regression<jupyter_code>model = linear_model.LinearRegression()
model.fit(X[:, None], Y)

plt.scatter(X, Y)
plt.plot(X, model.predict(np.array([X]).T), c='r')

model.coef_
model = linear_model.LinearRegression()
model.fit(np.array([X]).T, Y)

least_squares_distances = model.predict(np.array([X]).T)
tenth = (least_sqaures_distances/5)
ninetyth = tenth * 9

blues = sns.color_palette("Blues", 8)

sns.scatterplot(X, Y, alpha=0.8, color='b')
sns.lineplot(X, least_squares_distances, color=blues[5])
sns.lineplot(X, tenth, color=blues[6])
sns.lineplot(X, ninetyth, color=blues[7])

plt.annotate("10% Quantile", xy=(max(X)+1, max(tenth)+0.6), fontsize=14)
plt.annotate("50% Quantile", xy=(max(X)+1, max(least_squares_distances)+0.5), fontsize=14)
plt.annotate("90% Quantile", xy=(max(X)+1, max(ninetyth)+0.6), fontsize=14)

sns.despine(right=True)
            
model.coef_
(Y < least_squares_distances).sum()/Y.size
(Y < tenth).sum()/Y.size
(Y < ninetyth).sum()/Y.size<jupyter_output><empty_output><jupyter_text>Fitting Non-Linear Data With Linear Regression<jupyter_code>processor = pipeline.make_pipeline(preprocessing.PolynomialFeatures(2), linear_model.LinearRegression())
processor.fit(X[:, np.newaxis], Y)

xfit = np.linspace(0, max(X), X.size)
yfit = processor.predict(xfit[:, np.newaxis])

plt.scatter(X, Y)
plt.plot(xfit, yfit, c='b');

processor.get_params()['linearregression'].coef_
processor = pipeline.make_pipeline(preprocessing.PolynomialFeatures(2), linear_model.LinearRegression())
processor.fit(X[:, np.newaxis], Y)

sample_space = np.linspace(0, max(X), X.size)

least_squares_distances = processor.predict(sample_space[:, np.newaxis])
tenth = (least_squares_distances/5)
ninetyth = tenth * 9

sns.scatterplot(X, Y, alpha=0.8, color='b')
sns.lineplot(sample_space, least_squares_distances, color=blues[5])
sns.lineplot(sample_space, tenth, color=blues[6])
sns.lineplot(sample_space, ninetyth, color=blues[7])

plt.annotate("10% Quantile", xy=(max(X)+1, max(tenth)), fontsize=14)
plt.annotate("50% Quantile", xy=(max(X)+1, max(least_squares_distances)), fontsize=14)
plt.annotate("90% Quantile", xy=(max(X)+1, max(ninetyth)), fontsize=14)

sns.despine(right=True)
            
processor.get_params()['linearregression'].coef_
(Y < least_squares_distances).sum()/Y.size
(Y < tenth).sum()/Y.size
(Y < ninetyth).sum()/Y.size<jupyter_output><empty_output><jupyter_text>Pipeline For Fitting To Non-Linear Functions<jupyter_code>X_train, X_test, y_train, y_test = model_selection.train_test_split(X, Y)
processor = pipeline.make_pipeline(preprocessing.PolynomialFeatures(2), linear_model.LinearRegression())
processor.fit(X[:, np.newaxis], Y)
processor.get_params()['linearregression'].coef_
train_regression = processor.predict(X_train[:, np.newaxis])
train_tenth = (train_regression/5)
train_ninetyth = train_tenth * 9
sns.scatterplot(X_train, y_train, alpha=0.8, color='b')
sns.lineplot(X_train, train_regression, color=blues[5])
sns.lineplot(X_train, train_tenth, color=blues[6])
sns.lineplot(X_train, train_ninetyth, color=blues[7])

plt.annotate("10th Quantile", xy=(max(X)+1, max(train_tenth)+0.6), fontsize=14)
plt.annotate("50th Quantile", xy=(max(X)+1, max(train_regression)+0.5), fontsize=14)
plt.annotate("90th Quantile", xy=(max(X)+1, max(train_ninetyth)+0.6), fontsize=14)

sns.despine(right=True)
fig, (ax, bx) = plt.subplots(1, 2, figsize=(10,5))

sns.distplot(train_regression, ax=ax)
sns.distplot(y_train, ax=bx)

fig.suptitle("Distribution of Labels", fontweight='bold', fontsize=14)

ax.set_title("Predicted Labels")
bx.set_title("True Labels");
metrics.mean_squared_error(y_train, train_regression)<jupyter_output><empty_output><jupyter_text>Testing Regression Accuracy<jupyter_code>test_regression = processor.predict(X_test[:, np.newaxis])
test_tenth = (test_regression/5)
test_ninetyth = test_tenth * 9
sns.scatterplot(X_test, y_test, alpha=0.8, color='b')
sns.lineplot(X_test, test_regression, color=blues[5])
sns.lineplot(X_test, test_tenth, color=blues[6])
sns.lineplot(X_test, test_ninetyth, color=blues[7])

plt.annotate("10th Quantile", xy=(max(X)+0.4, max(test_tenth)+1), fontsize=14)
plt.annotate("50th Quantile", xy=(max(X)+0.4, max(test_regression)+1), fontsize=14)
plt.annotate("90th Quantile", xy=(max(X)+0.4, max(test_ninetyth)+1), fontsize=14)

sns.despine(right=True)
fig, (ax, bx) = plt.subplots(1, 2, figsize=(10,5))

sns.distplot(test_regression, ax=ax)
sns.distplot(y_test, ax=bx)

fig.suptitle("Distribution of Labels", fontweight='bold', fontsize=14)

ax.set_title("Predicted Labels")
bx.set_title("True Labels");
metrics.mean_squared_error(y_test, test_regression)<jupyter_output><empty_output><jupyter_text>Deliberate Overfitting<jupyter_code>processor = pipeline.make_pipeline(preprocessing.PolynomialFeatures(23), linear_model.LinearRegression())
processor.fit(X_train[:, np.newaxis], y_train)

overfit_train_regression = processor.predict(X_train[:, np.newaxis])
overfit_train_tenth = (overfit_train_regression/5)
overfit_train_ninetyth = overfit_train_tenth * 9

sns.scatterplot(X_train, y_train, alpha=0.8, color='b')
sns.lineplot(X_train, overfit_train_regression, color=blues[5])
sns.lineplot(X_train, overfit_train_tenth, color=blues[6])
sns.lineplot(X_train, overfit_train_ninetyth, color=blues[7])

plt.annotate("10th Quantile", xy=(max(X)+1, max(overfit_train_tenth)+0.6), fontsize=14)
plt.annotate("50th Quantile", xy=(max(X)+1, max(overfit_train_regression)+0.5), fontsize=14)
plt.annotate("90th Quantile", xy=(max(X)+1, max(overfit_train_ninetyth)+0.6), fontsize=14)

sns.despine(right=True)
y_train.size
(y_train < overfit_train_regression).sum(), (y_train < overfit_train_regression).sum()/y_train.size
(y_train < overfit_train_tenth).sum(), (y_train < overfit_train_tenth).sum()/y_train.size
(y_train < overfit_train_ninetyth).sum(), (y_train < overfit_train_ninetyth).sum()/y_train.size
fig, (ax, bx) = plt.subplots(1, 2, figsize=(10,5))

sns.distplot(overfit_train_regression, ax=ax)
sns.distplot(y_train, ax=bx)

fig.suptitle("Distribution of Labels", fontweight='bold', fontsize=14)

ax.set_title("Predicted Labels")
bx.set_title("True Labels");
metrics.mean_squared_error(y_train, overfit_train_regression)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16950934579439253, 'fraction_numerical': 0.026985981308411216, 'mean_word_length': 8.459668508287292, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '61238', 'license': 'MIT', 'n_tokens_mistral': 3593, 'n_tokens_neox': 3412, 'n_words': 489, 'path': 'module1-logistic-regression/quantiles.ipynb', 'repo_name': 'colejhudson/DS-Unit-2-Sprint-3-Advanced-Regression', 'size': '8560'}