<jupyter_start><jupyter_text>Data Ingestion and Transformation<jupyter_code>%matplotlib inline

import os
import json
import time

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# Load dataset
df = pd.read_csv('dataset-1min.csv')
# Drop duplicates
df = df.drop_duplicates()
df.head(3)
# Determine the shape of the data
print("{} instances with {} features\n".format(*df.shape))

# Determine the frequency of each class
print(df.groupby('occupancy_category')['occupancy_category'].count())
# Helper function to encode occupancy_category based on the number of people 
def occupancy(df):
  if df['occupancy_category'] == 'very-low':
    return '1'
  elif df['occupancy_category'] == 'low':
    return '2'
  elif df['occupancy_category'] == 'fair':
    return '3'
  else:
    return '4'

df['occupancy_code'] = df.apply(occupancy, axis=1)
df.head(3)
# Read the data into a DataFrame
features = [
    'temperature',
    'humidity',
    'co2',
    'light',
    'noise',
    'bluetooth_devices',
    'occupancy_code'
]
classes = [
    'very-low',
    'low',
    'fair',
    'high'
] 

df = df[features]
df.shape
# Extract the target from the data
data   = df.ix[:, 0:-1]
target = df.ix[:, -1]

print(data.shape)
print(target.shape)
# Split into test and train data
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(data, target)
# Standarize data
from sklearn.preprocessing import StandardScaler
standard_scaler = StandardScaler()
X_train = standard_scaler.fit_transform(X_train)
X_test = standard_scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>Classification Models Evaluation<jupyter_code>from sklearn import metrics
from sklearn.cross_validation import KFold
from yellowbrick.classifier import ClassificationReport, ROCAUC, ClassBalance

from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
def fit_and_evaluate(dataset, model, label, **kwargs):
    """
    Because of the Scikit-Learn API, we can create a function to
    do all of the fit and evaluate work on our behalf!
    """
    start  = time.time() # Start the clock! 
    scores = {'precision':[], 'recall':[], 'accuracy':[], 'f1':[]}
    
    for train, test in KFold(data.shape[0], n_folds=12, shuffle=True):
        
        estimator = model(**kwargs)
        estimator.fit(X_train, y_train)
        
        expected  = y_test
        predicted = estimator.predict(X_test)
        
        # Append our scores to the tracker
        scores['precision'].append(metrics.precision_score(expected, predicted, average="weighted"))
        scores['recall'].append(metrics.recall_score(expected, predicted, average="weighted"))
        scores['accuracy'].append(metrics.accuracy_score(expected, predicted))
        scores['f1'].append(metrics.f1_score(expected, predicted, average="weighted"))

    # Report
    print("Build and Validation of {} took {:0.3f} seconds".format(label, time.time()-start))
    print("Validation scores are as follows:\n")
    print(pd.DataFrame(scores).mean())<jupyter_output><empty_output><jupyter_text>Support Vector Classfier<jupyter_code># Perform SVC Classification
svc = SVC()
fit_and_evaluate(df, SVC, "SVM Classifier")
visualizer = ClassificationReport(svc, classes=classes)

visualizer.fit(X_train, y_train)  
visualizer.score(X_test, y_test)  
g = visualizer.poof()<jupyter_output><empty_output><jupyter_text>K Nearest Neighbors Classifier<jupyter_code># Perform kNN Classification
knn = KNeighborsClassifier()
fit_and_evaluate(df, KNeighborsClassifier, "kNN Classifier", n_neighbors=12)
visualizer = ClassificationReport(knn, classes=classes)

visualizer.fit(X_train, y_train)  
visualizer.score(X_test, y_test)  
g = visualizer.poof()<jupyter_output><empty_output><jupyter_text>Random Forest Classfier<jupyter_code># Perform Random Forest Classification
rfc = RandomForestClassifier()
fit_and_evaluate(df, RandomForestClassifier, "Random Forest Classifier")
visualizer = ClassificationReport(rfc, classes=classes)

visualizer.fit(X_train, y_train)  
visualizer.score(X_test, y_test)  
g = visualizer.poof()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11546685673556664, 'fraction_numerical': 0.005226894749346638, 'mean_word_length': 5.209439528023599, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '463426', 'license': 'MIT', 'n_tokens_mistral': 1425, 'n_tokens_neox': 1373, 'n_words': 323, 'path': 'dataset_builder/Machine Learning Part II Classification.ipynb', 'repo_name': 'georgetown-analytics/crash-severity', 'size': '4209'}