<jupyter_start><jupyter_text>Import Dependencies & Azure Keys<jupyter_code>import pandas as pd
import pyodbc
import os
from sklearn.ensemble import RandomForestRegressor
import warnings
warnings.filterwarnings("ignore") # Get rid of warnings        
from sklearn.decomposition import PCA
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
from faker import Faker # Make Synthetic Data
from sdv.tabular import GaussianCopula
from sklearn.svm import SVR

##* Azure Keys (Don't Share) *##
server = 'sustprediction.database.windows.net'
database = 'sustainability'
username = 'capstone'
password = <PASSWORD>}'   
driver= '{ODBC Driver 17 for SQL Server}'<jupyter_output><empty_output><jupyter_text>Fetch Data NYC, Chicago, Philadelphia Version 2Retrieve NYC data from Azure SQL Database🔑 Stores Data On RAM<jupyter_code>def nyc_v2():
    """
    1. Get the dataset as dataframe.
    2. Get the column names for the dataset separately from Azure.
    """

    conn = pyodbc.connect('DRIVER='+driver+';SERVER=tcp:'+server+';PORT=1433;DATABASE='+database+';UID='+username+';PWD='+ password)

    # NYC V2 Data
    nyc = conn.cursor().execute("SELECT * FROM dbo.newyork_v2").fetchall() 
    columns = conn.cursor().execute("SELECT * FROM sys.columns WHERE object_id = OBJECT_ID('dbo.newyork_v2')").fetchall()
    columns_df = pd.DataFrame([tuple(t) for t in columns])
    list_names = list(columns_df[1])
    nyc_df = pd.DataFrame([tuple(t) for t in nyc], columns = list_names)
    return nyc_df
def chicago_v2():
    """
    1. Get the dataset as dataframe.
    2. Get the column names for the dataset separately from Azure.
    """

    conn = pyodbc.connect('DRIVER='+driver+';SERVER=tcp:'+server+';PORT=1433;DATABASE='+database+';UID='+username+';PWD='+ password)

    # NYC V2 Data
    chic = conn.cursor().execute("SELECT * FROM dbo.ChicagoFeatures_V2").fetchall() 
    columns = conn.cursor().execute("SELECT * FROM sys.columns WHERE object_id = OBJECT_ID('dbo.ChicagoFeatures_V2')").fetchall()
    columns_df = pd.DataFrame([tuple(t) for t in columns])
    list_names = list(columns_df[1])
    chic_df = pd.DataFrame([tuple(t) for t in chic], columns = list_names)
    return chic_df
def philly_v2():
    """
    1. Get the dataset as dataframe.
    2. Get the column names for the dataset separately from Azure.
    """

    conn = pyodbc.connect('DRIVER='+driver+';SERVER=tcp:'+server+';PORT=1433;DATABASE='+database+';UID='+username+';PWD='+ password)

    # NYC V2 Data
    chic = conn.cursor().execute("SELECT * FROM dbo.PhillyFeatures_V2").fetchall() 
    columns = conn.cursor().execute("SELECT * FROM sys.columns WHERE object_id = OBJECT_ID('dbo.PhillyFeatures_V2')").fetchall()
    columns_df = pd.DataFrame([tuple(t) for t in columns])
    list_names = list(columns_df[1])
    chic_df = pd.DataFrame([tuple(t) for t in chic], columns = list_names)
    return chic_df
# Get Data
nyc_data = nyc_v2()
chicago_data = chicago_v2()
philly_data = philly_v2()
print(f"NYC Dataset:")
nyc_data.head()
print(f"Total Number of Columns in NYC Dataset: {len(nyc_data.columns)}")
print(f"Chicago Dataset:")
chicago_data.head()
print(f"Total Number of Columns in Chicago Dataset: {len(chicago_data.columns)}")
print(f"Philly Dataset:")
philly_data.head()
print(f"Total Number of Columns in Philly Dataset: {len(philly_data.columns)}")<jupyter_output>Total Number of Columns in Philly Dataset: 19<jupyter_text>PreprocessingSteps:1. Set License Class as Categorical and then Encode it for ML models.2. AQI is an Integer, no decimal values.3. Train Set - Data From 2018 to 2020, Test Set - Data From 2021 only.4. Split Train and Test Data into Features and Labels.  NYC<jupyter_code># Encode the License Class Column NYC
nyc_data['License_Class'] = nyc_data['License_Class'].astype('category').cat.codes
nyc_data['aqi'] = nyc_data['aqi'].astype(int)

# NYC Train - 2018 to 2020
nyc_train = nyc_data[nyc_data['year'] != 2021]

# NYC Test - 2020
nyc_test = nyc_data[nyc_data['year'] == 2021]

print(f"Train Data Years: {nyc_train.year.unique()}")
print(f"Test Data Years: {nyc_test.year.unique()}")

# Create Train and Test Splits NYC
nyc_X_train = nyc_train.drop(['pm25','aqi'], axis = 1)
nyc_y_train = nyc_train[['aqi']]
nyc_X_test = nyc_test.drop(['pm25','aqi'], axis = 1)
nyc_y_test = nyc_test[['aqi']]

print(f"X_train Shape: {nyc_X_train.shape}")
print(f"X_test Shape: {nyc_X_test.shape}")<jupyter_output>Train Data Years: [2018 2019 2020]
Test Data Years: [2021]
X_train Shape: (936, 32)
X_test Shape: (318, 32)<jupyter_text>Chicago<jupyter_code># Drop City Indentifiers as they prevent the model to generalize over the dataset
chicago_data = chicago_data.drop(['County', 'City'], axis = 1)
chicago_data = chicago_data.replace('NA', 0)
chicago_data = chicago_data.astype(float)
chicago_data = chicago_data.replace(0, np.nan)
# NA values
chicago_data.isna().sum()
print(f"Total Number Of Records: {len(chicago_data)}")<jupyter_output>Total Number Of Records: 212<jupyter_text>Looks like some columns have more than 75% of missing values - this renders them usless. We can remove them<jupyter_code>chicago_data = chicago_data.drop(['avg_weekday_rides', 'avg_saturday_rides', 'avg_sunday_holiday_rides',
                                 'traffic_taxi_trip_miles', 'traffic_taxi_trip_totals', 'traffic_rideshare_miles',
                                 'traffic_speed', 'traffic_bus_count'], axis = 1)
chicago_data.head()
# The rest of the missing values can be forward filled

chicago_data = chicago_data.ffill(axis = 0) # column vice
print(f"Missing Values:\n {chicago_data.isna().sum()}")
chicago_data['year'] = chicago_data['year'].astype(int)
chicago_data['AQI_Weekly_Measurement'] = chicago_data['AQI_Weekly_Measurement'].astype(int)
chicago_data.head()
# Chicago Train - 2018 to 2020
chicago_train = chicago_data[chicago_data['year'] != 2021]

# Chicago Test - 2020
chicago_test = chicago_data[chicago_data['year'] == 2021]

print(f"Train Data Years: {chicago_train.year.unique()}")
print(f"Test Data Years: {chicago_test.year.unique()}")

# Create Train and Test Splits NYC
chicago_X_train = chicago_train.drop(['Pollution_pm25','AQI_Weekly_Measurement'], axis = 1)
chicago_y_train = chicago_train[['AQI_Weekly_Measurement']]
chicago_X_test = chicago_test.drop(['Pollution_pm25','AQI_Weekly_Measurement'], axis = 1)
chicago_y_test = chicago_test[['AQI_Weekly_Measurement']]

print(f"X_train Shape: {chicago_X_train.shape}")
print(f"X_test Shape: {chicago_X_test.shape}")<jupyter_output>Train Data Years: [2018 2019 2020]
Test Data Years: [2021]
X_train Shape: (159, 9)
X_test Shape: (53, 9)<jupyter_text>Philly<jupyter_code># Drop City Indentifiers as they prevent the model to generalize over the dataset
philly_data = philly_data.drop(['County', 'City'], axis = 1)
philly_data = philly_data.replace('NA', 0)
philly_data = philly_data.astype(float)
philly_data = philly_data.replace(0, np.nan)
# NA values
philly_data.isna().sum()
print(f"Total Number of Records: {len(philly_data)}")
# Drop columns with almost all records missing
philly_data = philly_data.drop(['traffic_bike_counts', 'Pollution_so2', 'Pollution_pm25', 
                                'traffic_ped_counts', 'traffic_vehicle_counts'], axis = 1)
philly_data.head()
# The rest of the missing values can be forward filled

philly_data = philly_data.ffill(axis = 0) # column vice
print(f"Missing Values:\n {philly_data.isna().sum()}")
philly_data['year'] = philly_data['year'].astype(int)
philly_data['AQI_Weekly_Measurement'] = philly_data['AQI_Weekly_Measurement'].astype(int)
philly_data.head()
# Philly Train - 2018 to 2020
philly_train = philly_data[philly_data['year'] != 2021]

# Philly Test - 2020
philly_test = philly_data[philly_data['year'] == 2021]

print(f"Train Data Years: {philly_train.year.unique()}")
print(f"Test Data Years: {philly_test.year.unique()}")

# Create Train and Test Splits NYC
philly_X_train = philly_train.drop(['AQI_Weekly_Measurement'], axis = 1)
philly_y_train = philly_train[['AQI_Weekly_Measurement']]
philly_X_test = philly_test.drop(['AQI_Weekly_Measurement'], axis = 1)
philly_y_test = philly_test[['AQI_Weekly_Measurement']]

print(f"X_train Shape: {philly_X_train.shape}")
print(f"X_test Shape: {philly_X_test.shape}")<jupyter_output>Train Data Years: [2018 2019 2020]
Test Data Years: [2021]
X_train Shape: (159, 11)
X_test Shape: (53, 11)<jupyter_text>Standard ScalerScale the data for the model<jupyter_code>from sklearn.preprocessing import StandardScaler

def scale_data(X_train, y_train, X_test, y_test):
    
    """
    Input: training and testing splits.
    Output: Scaled training and testing splits.
    """
    X_train = StandardScaler().fit_transform(X_train)
    y_train = StandardScaler().fit_transform(y_train)
    X_test = StandardScaler().fit_transform(X_test)
    y_test = StandardScaler().fit_transform(y_test)
    
    return X_train, y_train, X_test, y_test<jupyter_output><empty_output><jupyter_text>Functions  Loss Curves Plot<jupyter_code>def plot_loss(history):
    """
    Loss Curves of the Models - Neural Nets
    """
    plt.plot(history.history['loss'], label = 'loss')
    #plt.plot(history.history['val_loss'], label = 'val_loss')
    plt.ylim([0, 10])
    plt.xlabel('Epoch')
    plt.ylabel('Error')
    plt.legend()
    plt.grid(True)
    plt.show()<jupyter_output><empty_output><jupyter_text>Source City NYC Models<jupyter_code>scaled_nyc_X_train, scaled_nyc_y_train, scaled_nyc_X_test, scaled_nyc_y_test = scale_data(nyc_X_train, nyc_y_train, nyc_X_test, nyc_y_test)<jupyter_output><empty_output><jupyter_text>Model 1: Random Forest Regressor<jupyter_code>np.random.seed(44)
rfr = RandomForestRegressor(n_estimators=200, criterion="mse",
                            min_samples_leaf=3, min_samples_split=3, 
                            max_depth=5, random_state = 44).fit(scaled_nyc_X_train, scaled_nyc_y_train)

print(f"Random Forest Regressor Score: {rfr.score(scaled_nyc_X_test, scaled_nyc_y_test) * 100}")<jupyter_output>Random Forest Regressor Score: 3.9091901112385585<jupyter_text>Model 2: SVM<jupyter_code>np.random.seed(44)
regressor = SVR(kernel = 'rbf').fit(scaled_nyc_X_train, scaled_nyc_y_train)
print(f"Support Vector Machines: {regressor.score(scaled_nyc_X_test, scaled_nyc_y_test) * 100}")<jupyter_output>Support Vector Machines: -18.846528996700318<jupyter_text>Model 3: Sequential<jupyter_code># Build Fully Connect Neural Net using Sequential API
tf.random.set_seed(44)
model = tf.keras.Sequential()
model.add(tf.keras.layers.Dense(500, input_dim=32, activation= "relu"))
model.add(tf.keras.layers.Dense(500, activation = "relu"))
model.add(tf.keras.layers.Dense(100, activation= "relu"))
model.add(tf.keras.layers.Dense(50, activation= "sigmoid"))
model.add(tf.keras.layers.Dense(1))
model.summary()
# Compile and Fit Model
tf.random.set_seed(44)
model.compile(loss= "mean_squared_error" , optimizer="adam", metrics=["mean_squared_error"])
history_1 = model.fit(scaled_nyc_X_train, scaled_nyc_y_train, epochs=50, verbose = 0)
# Evaluate Model on Test Data
model.evaluate(scaled_nyc_X_test, scaled_nyc_y_test)
# Make Predictions
predictions = model.predict(scaled_nyc_X_test)
# Plot True Values and Predictions
plt.figure(figsize = (10,5))
plt.plot(scaled_nyc_y_test, c = 'green', label = "True Values")
plt.plot(predictions, c = "red", label = "Predictions")
plt.legend()
plt.show()
# Plot Loss Curves
plot_loss(history_1)
# Save Model
model.save('Models/Model_1')<jupyter_output>2022-03-28 17:59:46.985546: W tensorflow/python/util/util.cc:368] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.<jupyter_text>Preprocess Create Synthetic Data Using `Train` Data and Check Chi Squared Distribution and Plot the Distribution<jupyter_code># SDV 
model_copula = GaussianCopula()
model_copula.fit(nyc_train)
# Create a Dataset with the same number or records as training data
np.random.seed(44)
sample_data = model_copula.sample(936)
sample_data.head()
nyc_train.head()<jupyter_output><empty_output><jupyter_text>EDA<jupyter_code>print(f"Distribution of Data By Year Train Data:\n{nyc_train.year.value_counts()}")
print(f"Distribution of Data By Year Train Data:\n{sample_data.year.value_counts()}")

print(f"Mean of AQI Train Data: {nyc_train.aqi.mean()}")
print(f"Mean of AQI Sample Data: {sample_data.aqi.mean()}")
# Kolmogorov-Smirnov and Chi-Squared tests to compare train and sample data
from sdv.evaluation import evaluate
evaluate(sample_data, nyc_train, metrics = ['CSTest', 'KSTest'], aggregate = False)<jupyter_output><empty_output><jupyter_text>Model1. Create X and Y sets from Sample Data2. Use the saved model to evalute the sample data<jupyter_code># Scale Data
nyc_sample_X_train = sample_data.drop(['pm25','aqi'], axis = 1)
nyc_sample_y_train = sample_data[['aqi']]
nyc_sample_X_train = StandardScaler().fit_transform(nyc_sample_X_train)
nyc_sample_y_train = StandardScaler().fit_transform(nyc_sample_y_train)
# Evaluate Data
loaded_model = tf.keras.models.load_model('Models/Model_1')
loaded_model.evaluate(nyc_sample_X_train, nyc_sample_y_train)<jupyter_output>30/30 [==============================] - 0s 2ms/step - loss: 1.2419 - mean_squared_error: 1.2419<jupyter_text>The MSE is very comparable to training data MSE  Combine Datasets and Model and Evaluate on Test Data<jupyter_code>nyc_combined_train = pd.concat([nyc_train, sample_data])
print(f"Length of Training Data: {len(nyc_combined_train)}")

# Shuffle Data
nyc_combined_train = nyc_combined_train.sample(frac=1).reset_index(drop=True)
# Split and Scale Data
nyc_combined_X_train = nyc_combined_train.drop(['pm25','aqi'], axis = 1)
nyc_combined_y_train = nyc_combined_train[['aqi']]
nyc_combined_X_train = StandardScaler().fit_transform(nyc_combined_X_train)
nyc_combined_y_train = StandardScaler().fit_transform(nyc_combined_y_train)
# Compile and Fit Model
tf.random.set_seed(44)
model.compile(loss= "mean_squared_error" , optimizer="adam", metrics=["mean_squared_error"])
history_2 = model.fit(nyc_combined_X_train, nyc_combined_y_train, epochs=50, verbose = 0)
# Evaluate Model
model.evaluate(scaled_nyc_X_test, scaled_nyc_y_test)
predictions_2 = model.predict(scaled_nyc_X_test)
# Plot True Values and Predictions
plt.figure(figsize = (10,5))
plt.plot(scaled_nyc_y_test, c = 'green', label = "True Values")
plt.plot(predictions_2, c = "red", label = "Predictions")
plt.legend()
plt.show()
# Save Model
model.save('Models/Model_2')<jupyter_output>INFO:tensorflow:Assets written to: Models/Model_2/assets<jupyter_text>Model 4: Sequential From Model 3<jupyter_code># Build Fully Connect Neural Net using Functional API
tf.random.set_seed(44)

model_4 = tf.keras.Sequential()
model_4.add(tf.keras.layers.Dense(500, input_dim=32, activation= "relu"))
model_4.add(tf.keras.layers.Dense(500, activation = "relu"))
model_4.add(tf.keras.layers.Dense(500, activation = "relu"))
model_4.add(tf.keras.layers.Dense(250, activation= "relu"))
model_4.add(tf.keras.layers.Dense(250, activation= "relu"))
model_4.add(tf.keras.layers.Dense(100, activation= "relu"))
model_4.add(tf.keras.layers.Dense(100, activation= "relu"))
model_4.add(tf.keras.layers.Dense(50, activation= "relu"))
model_4.add(tf.keras.layers.Dense(1))    
model_4.compile(loss= "mean_squared_error", optimizer="adam", metrics=["mean_squared_error"])
history_4 = model_4.fit(nyc_combined_X_train, nyc_combined_y_train, epochs=50, verbose = 0)
model_4.evaluate(scaled_nyc_X_test, scaled_nyc_y_test)
predictions_4 = model_4.predict(scaled_nyc_X_test)
# Plot True Values and Predictions
plt.figure(figsize = (10,5))
plt.plot(scaled_nyc_y_test, c = 'green', label = "True Values")
plt.plot(predictions_4, c = "red", label = "Predictions")
plt.legend()
plt.show()
model_4.save('Models/Model_4')<jupyter_output>INFO:tensorflow:Assets written to: Models/Model_4/assets<jupyter_text>**Model 4 is the best model so far - Transfer model knowledge to Chicago and Philly**  Transfer LearningTransfer Knowledge from NYC Model 4 to Chicago and Philly<jupyter_code># Create a Custom Nueral Net API using Model 4

def SourceCityNet(include_top = True,
             weights='imagenet',
             input_tensor=None,
             input_shape=None,
             pooling=None,
             classes=1000,
             **kwargs):
    
    
    if input_tensor is None:
        
        X_train_input = tf.keras.layers.Input(shape = input_shape)
        
    else:
        
        if not backend.is_keras_tensor(input_tensor):
            
            X_train_input = tf.keras.layers.Input(tensor=input_tensor, 
                                                  shape=input_shape)
            
        else:
            
            X_train_input = input_tensor
                
                
    x = tf.keras.layers.Dense(500, activation= "relu")(X_train_input)
    x = tf.keras.layers.Dense(500, activation = "relu")(x)
    x = tf.keras.layers.Dense(500, activation = "relu")(x)
    x = tf.keras.layers.Dense(250, activation= "relu")(x)
    x = tf.keras.layers.Dense(250, activation= "relu")(x)
    x = tf.keras.layers.Dense(100, activation= "relu")(x)
    x = tf.keras.layers.Dense(100, activation= "relu")(x)
    
    
    x = tf.keras.layers.Dense(50, activation= "relu")(x)
    x = tf.keras.layers.Dense(1)(x)
            
            


model_tranfer_test.evaluate(scaled_nyc_X_test, scaled_nyc_y_test)<jupyter_output><empty_output><jupyter_text>Chicago<jupyter_code># -- Scale Data using Standard Scaler Function -- #

scaled_chicago_X_train, scaled_chicago_y_train, scaled_chicago_X_test, scaled_chicago_y_test = scale_data(chicago_X_train, chicago_y_train, chicago_X_test, chicago_y_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1268361581920904, 'fraction_numerical': 0.02610169491525424, 'mean_word_length': 5.481508604906628, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '36529', 'license': 'MIT', 'n_tokens_mistral': 6836, 'n_tokens_neox': 6347, 'n_words': 1316, 'path': 'models/src/modeling/build/Multiple-Models.ipynb', 'repo_name': 'vbgupta/DS440-Transfer-Learning-Address-Sustainability-Issues', 'size': '17702'}