<jupyter_start><jupyter_text><jupyter_code>##############################################################################
# Create the Panda's profile of the data
##############################################################################

import pandas as pd
from pandas_profiling import ProfileReport as pr

! pip install \
https://github.com/pandas-profiling/pandas-profiling/archive/master.zip

bikes = pd.read_csv('https://raw.githubusercontent.com/byui-cse/cse450-' +
                      'course/master/data/bikes.csv')

# profile the dataset
profile = pr(bikes)
profile.to_notebook_iframe()
###############################################################################
# Build the Keras model from the given data
###############################################################################

import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow import keras
from sklearn import datasets, linear_model
from sklearn.model_selection import cross_val_score, KFold
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from sklearn.metrics import accuracy_score
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasRegressor

# Load in the data
bikes = pd.read_csv('https://raw.githubusercontent.com/byui-cse/cse450-' +
                      'course/master/data/bikes.csv')

# Create the dataset and combine the totals of all used bikes in each row
X = bikes
X['total'] = X['casual'] + X['registered']

# Create the labels
y = X['total']

# Remove all of the "label" classes
X.pop('casual')
X.pop('registered')
X.pop('total')

# I need to add this back in, but for now I am just removing all textual data
X.pop('dteday')

loss_function = keras.losses.SparseCategoricalCrossentropy(from_logits=True)

# Generate the Keras model
def baseline_model():
    model = Sequential()
    model.add(Dense(10, input_dim=9, activation='relu'))
    model.compile(loss=loss_function, optimizer='adam')
    return model

# Create the training test split
test_size = 0.2
seed = 42
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, 
                                                    random_state=seed)

# Sort all of the data to maintain a semblance of the date order
X_train = X_train.sort_index(ascending=True)
X_test  = X_test.sort_index(ascending=True)
y_train = y_train.sort_index(ascending=True)
y_test  = y_test.sort_index(ascending=True)

estimator = tf.keras.wrappers.scikit_learn.KerasRegressor(build_fn=baseline_model)
estimator.fit(X_train, y_train)
estimator.predict(X_train)
estimator.score(X_train, y_train)
########
# trash
########


# estimator = KerasRegressor(build_fn=baseline_model, nb_epoch=100, batch_size=100, verbose=False)
# estimator.fit(X_train, y_train)
# prediction = estimator.predict(y_train)

# train_error =  np.abs(y - prediction)
# mean_error = np.mean(train_error)
# min_error = np.min(train_error)
# max_error = np.max(train_error)
# std_error = np.std(train_error)
# ##############################################################################
# # Getting the XG Boost library up and running. I run it both with and without
# # the SKLearn wrapper.
# ##############################################################################

# import pandas as pd
# import xgboost as xgb
# import math
# from xgboost import XGBRegressor
# from xgboost import plot_tree
# from sklearn.metrics import mean_squared_error
# from sklearn.metrics import r2_score
# from google.colab import drive
# from sklearn.model_selection import cross_val_score
# from sklearn.model_selection import RepeatedKFold
# from sklearn.model_selection import GridSearchCV
# from sklearn.model_selection import train_test_split

# # Load the data set
# housing = pd.read_csv('https://raw.githubusercontent.com/byui-cse/cse450-course/master/data/housing.csv')
# holdout = pd.read_csv('https://raw.githubusercontent.com/byui-cse/cse450-course/master/data/housing_holdout_test.csv')

# # Calculating distance from the city center
# CENTER_LONG = 122.2
# CENTER_LAT  = 47.6

# housing['x1_distance']        = (abs(CENTER_LONG - abs(housing['long'])) * 1855.46)
# housing['x2_distance']        = (abs(CENTER_LAT  - abs(housing['lat']))  * 1260.14)
# housing['squared_distance']   = housing['x1_distance'] ** 2 + housing['x2_distance'] ** 2
# housing['distance']           = housing['squared_distance'].apply(lambda x: math.sqrt(x))

# holdout['x1_distance']        = (abs(CENTER_LONG - abs(housing['long'])) * 1855.46)
# holdout['x2_distance']        = (abs(CENTER_LAT  - abs(housing['lat']))  * 1260.14)
# holdout['squared_distance']   = housing['x1_distance'] ** 2 + housing['x2_distance'] ** 2
# holdout['distance']           = housing['squared_distance'].apply(lambda x: math.sqrt(x))

# # Create two distinct datasets - one without the prices, and one only with the
# # prices
# X = housing
# Y = housing['price']

# # I currently don't see how id can help me, so I'm not going to include it.
# # There was a noticable improvement after changing this. Also, I no longer need
# # any of the prior math data
# X.pop('price')
# X.pop('id')
# X.pop('x1_distance')
# X.pop('x2_distance')
# X.pop('squared_distance')

# holdout.pop('id')
# holdout.pop('x1_distance')
# holdout.pop('x2_distance')
# holdout.pop('squared_distance')

# # Pull the year/month out of the data and give it its own column
# X["year_listed"] = X['date'].str[:6]
# X["year_listed"] = pd.to_numeric(X["year_listed"])

# holdout["year_listed"] = holdout['date'].str[:6]
# holdout["year_listed"] = pd.to_numeric(holdout["year_listed"])

# # Pop off the original date data
# X.pop('date')

# holdout.pop('date')

# # Use the seed and the test size to change the outcome of the test_split
# seed = 8
# test_size = 0.2

# # Create the training test split
# X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=test_size, 
#                                                     random_state=seed)

# # Train the XGB model using the SKLearn wrapper
# model = XGBRegressor(max_depth=6, learning_rate=0.1, 
#                      objective='reg:squarederror', subsample=0.7, 
#                      min_samples_leaf=50, min_samples_split=100, 
#                      n_estimators=150)
# model.fit(X_train, y_train)

# # Predict and then measure the accuracy of the predictions that were made
# predictions = model.predict(X_test)
# print("Avg. Error in dollars with wrapper = {}".format(
#     mean_squared_error(y_test, predictions, squared=False)
#     ))
# print("R2 Score = {}".format(r2_score(y_test, predictions)))

# results = model.predict(holdout)
# final = pd.DataFrame(results)
# final.to_csv('housing_predictions.csv', header=['predictions'], index=False)

# #drive.mount('drive')
# !cp bank_predictions.csv "drive/My Drive/"<jupyter_output>Avg. Error in dollars with wrapper = 105762.77173625924
R2 Score = 0.9150200968969855
{'fraction_non_alphanumeric': 0.18301314459049545, 'fraction_numerical': 0.01848909432327026, 'mean_word_length': 4.680065627563577, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '535897', 'license': 'MIT', 'n_tokens_mistral': 2325, 'n_tokens_neox': 2179, 'n_words': 530, 'path': 'Module_4.ipynb', 'repo_name': 'DevonLud/ludwig_resume', 'size': '6923'}