<jupyter_start><jupyter_text>Bikers on the Fremont bridgeExample adapted from the [Data Science Handbook](https://jakevdp.github.io/PythonDataScienceHandbook/05.06-linear-regression.html)  Set up: Download (and load) data<jupyter_code># Download data(you can download it by uncommenting and runing this line of code)
# !curl -o FremontBridge.csv https://data.seattle.gov/api/views/65db-xm6k/rows.csv?accessType=DOWNLOAD
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.preprocessing import MinMaxScaler       # scaling data
from sklearn.model_selection import train_test_split # splitting data
from sklearn.neighbors import KNeighborsRegressor    # regressor
from sklearn.model_selection import GridSearchCV     # for grid search
from sklearn.pipeline import make_pipeline           # for making pipelines
%matplotlib inline
# Aggregate data to the daily level
counts = pd.read_csv('data/FremontBridge.csv', index_col='Date', parse_dates=True)
daily = counts.resample('d').sum()
daily['Total'] = daily.sum(axis=1)
daily = daily[['Total']] # remove other columns
plt.figure(figsize=(15,5))
daily.Total.plot()
daily[daily.Total == daily.Total.max()]<jupyter_output><empty_output><jupyter_text>Data Prep: Adding Features<jupyter_code># Load weather data (downloaded from: https://www.ncdc.noaa.gov/cdo-web/search?datasetid=GHCND)
weather = pd.read_csv('data/weather.csv', index_col='DATE', parse_dates=True)

# Create dry_day column
weather['dry_day'] = (weather['PRCP'] == 0).astype(int)
# Join selected weather columns
daily = daily.join(weather[['PRCP', 'dry_day', 'TMIN', 'TMAX']])
# Compute hours of daylight
def hours_of_daylight(date, axis=23.44, latitude=47.61):
    """Compute the hours of daylight for the given date"""
    days = (date - pd.datetime(2000, 12, 21)).days
    m = (1. - np.tan(np.radians(latitude))
         * np.tan(np.radians(axis) * np.cos(days * 2 * np.pi / 365.25)))
    return 24. * np.degrees(np.arccos(1 - np.clip(m, 0, 2))) / 180.

daily['daylight_hrs'] = list(map(hours_of_daylight, daily.index))
daily[['daylight_hrs']].plot()
plt.ylim(8, 17)<jupyter_output><empty_output><jupyter_text>Feature Generation: Categorical Variable(s)<jupyter_code># Get dummy variables from categorical columns (alternative: sklearn OneHotEncoding)
daily["day_of_week"] = daily.index.day_name()
# daily["day_of_week"] = daily.index.dayofweek.astype(dtype="str")
daily = pd.get_dummies(daily)
daily.head()<jupyter_output><empty_output><jupyter_text>Abbreviated EDA<jupyter_code># What is the relationship between bikers and temperature?
plt.scatter(daily.TMAX, daily.Total, alpha = 0.2)
plt.xlabel("Max Temperature")
plt.ylabel("Total # Bikers in a Day")
plt.show()
# What is the relationship between bikers and date?
plt.figure(figsize=(15,4))
daily.Total.plot()
# What is the relationship between bikers and (min) temperature?
plt.scatter(daily.TMIN, daily.Total, alpha = 0.2)
plt.xlabel("Minimum Temperature")
plt.ylabel("Total # Bikers in a Day")
plt.show()
# What is the distribution of bikers on dry/wet days?
wet_days = daily[daily.dry_day == 0]
dry_days = daily[daily.dry_day == 1]
plt.hist(wet_days.Total, alpha = 0.3, label = "Wet")
plt.hist(dry_days.Total, alpha = 0.3, label = "Dry")
plt.legend()
plt.show()
# What is the relationship between bikers and precipitation?
plt.scatter(daily.PRCP, daily.Total, alpha = 0.2)
plt.xlabel("Precipitation")
plt.ylabel("Total # Bikers in a Day")
plt.show()
# How does the number of bikers vary by temperature and wet/dry?
plt.scatter(wet_days.TMAX, wet_days.Total, alpha = 0.3, label = "Wet")
plt.scatter(dry_days.TMAX, dry_days.Total, alpha = 0.3, label = "Dry")
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Modeling: KNN Regressor<jupyter_code># Split data into training and testing data
# Split into test/train data
from sklearn.model_selection import train_test_split
train_features, test_features, train_outcome, test_outcome = train_test_split(
    daily.drop("Total", axis = 1), 
    daily.Total, 
    test_size=0.10
)
# Create a scaler and your classifier
scaler = MinMaxScaler()
knn_clf = KNeighborsRegressor()


# Define a pipeline that uses your scaler and classifier
pipe = make_pipeline(scaler, knn_clf)

# Define a grid to search through
params = {"kneighborsregressor__n_neighbors":np.arange(1, 10)}

# Perform a grid search of your pipeline
grid_search = GridSearchCV(pipe, params, scoring="neg_mean_absolute_error")
grid_search.fit(train_features, train_outcome)
# Compare prediction to (test) data
preds = grid_search.predict(test_features)
plt.scatter(preds, test_outcome)
plt.show()
test_data = test_features.join(test_outcome)
test_data["preds"] = grid_search.predict(test_features)
plt.figure(figsize=(20, 5))
test_data.Total.plot(label = "Actual")
test_data.preds.plot(label = "Predicted")
plt.show()
grid_search.score(test_features, test_outcome)<jupyter_output><empty_output><jupyter_text>Feature Generation: Polynomial Transformations<jupyter_code># Add a polynomial transformation to the pipeline
from sklearn.preprocessing import PolynomialFeatures
poly = PolynomialFeatures()
# Define a pipeline that includes the polynomial transformation
pipe = make_pipeline(poly, scaler, knn_clf)

# Define a grid to search through (including the degree of polynomial)
param_grid = {'polynomialfeatures__degree':range(1, 4),
             'kneighborsregressor__n_neighbors':range(1, 10),
             'kneighborsregressor__weights':["uniform", "distance"]}

# Perform a  grid search of your pipeline
grid_search = GridSearchCV(pipe, param_grid, scoring="neg_mean_absolute_error")
grid_search.fit(train_features, train_outcome)
grid_search.score(test_features, test_outcome)
# Visualize time trends<jupyter_output><empty_output><jupyter_text>Error assessment: find systematic errors<jupyter_code># Why are we getting this wrong?
# Assess error by day of the week

# Assess error by temperature and dry_day

# Assess error by precipitation<jupyter_output><empty_output><jupyter_text>Feature Selection: Select best featuresAs a form of dimensionality reduction, only select the top percentile features that have a certain threshold of variance.<jupyter_code># Create a percentile selector, add it to the pipeline
# (alternatives a K selectors, PCA, or others)


# Define a grid to search through (including the degree of polynomial AND percentile of best features)

# Fit the model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11370668316831684, 'fraction_numerical': 0.012376237623762377, 'mean_word_length': 6.096597145993414, 'pattern_counts': {'":': 1, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '573218', 'license': 'MIT', 'n_tokens_mistral': 2295, 'n_tokens_neox': 2120, 'n_words': 561, 'path': 'analysis.ipynb', 'repo_name': 'victoriahuynh/feature-engineering', 'size': '6464'}