<commit_before># -*- coding: utf-8 -*-

###############################################################################
# Information
###############################################################################
# Created by <NAME> 
# Input from <NAME>

# In partial fulfillment of the requirements for the Georgetown University Data Analytics Graduate Certificate Program

# useful links
# http://stackoverflow.com/questions/27504870/sklearn-kmeans-get-class-centroid-labels-and-reference-to-a-dataset

# May 1, 2015

# https://plus.google.com/+LinwoodCreekmoreIII/

###############################################################################
# Imports
###############################################################################


print(__doc__)

from time import time
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.decomposition import PCA
from sklearn import metrics
from sklearn.cluster import KMeans
from sklearn.datasets import load_digits
from sklearn.decomposition import PCA
from sklearn.preprocessing import scale
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import StandardScaler
import os


path = path = os.path.abspath(os.getcwd())

# Some colors for later
colors = np.array([x for x in 'bgrcmykbgrcmykbgrcmykbgrcmyk'])
colors = np.hstack([colors] * 20)

'''
df = pd.read_csv('./lin.csv')

digits = np.asfarray(df[['driver_id', 'trip_id', 'Average Velocity (mph)', 'Max Velocity', 'Velocity Stdev','Average Acceleration (mph per s)', 'Max Acceleration (mph per s)', ' Acceleration Stdev','Displacement','Total Distance Traveled','Max Direction Change per sec', ' Direction Stdev','Time (s)', 'Turns', 'Aggressive Turns', 'Stops', 'Large Deceleration Events', 'Deceleration Events', 'Max Deceleration Event']])
data = scale(digits.data)



#####
from sklearn.cluster import MiniBatchKMeans
import matplotlib.pyplot as plt

fig, axe = plt.subplots(figsize=(18, 4))
for idx, dataset in enumerate((circles, moons, blobs, noise)):
    X, y = dataset
    X = StandardScaler().fit_transform(X)
    
'''  


###
 #load data from a CSV to a dataframe
with open("./lin.csv") as in_data:
    skid_data = pd.DataFrame.from_csv(in_data, sep=',')

n_samples, n_features = skid_data.shape
print skid_data.shape

#skid_data=skid_data.fillna(value=-999)

#load all numeric data into an array. The offense column from the crime data
#is excluded
as_array = np.asfarray(skid_data[['Average Velocity (mph)','Max Velocity', 'Velocity Stdev','Average Acceleration (mph per s)', 'Max Acceleration (mph per s)', ' Acceleration Stdev','Displacement','Total Distance Traveled','Max Direction Change per sec', ' Direction Stdev','Time (s)', 'Turns', 'Aggressive Turns', 'Stops', 'Large Deceleration Events', 'Deceleration Events', 'Max Deceleration Event']])

#number of groups
n_clusters=5


#Correct missing data 
imputer = Imputer(missing_values="NaN", strategy="mean")
patched = imputer.fit_transform(as_array)

# Preprocessing tricks
patched = StandardScaler().fit_transform(patched)
#patched = scale(patched, axis=0, with_mean=True)




#cluster data 
cluster = KMeans(n_clusters=n_clusters)
cluster.fit(patched)

# assigned grouped labels to the Skid data
labels = cluster.labels_
skid_data["labels"]=labels
'''
  # Fit the model with our algorithm
cluster = MiniBatchKMeans(n_clusters=3)
cluster.fit(patched)
'''
reduced_data = PCA(n_components=2).fit_transform(patched)
print reduced_data

kmeans = KMeans(init='k-means++', n_clusters=n_clusters, n_init=20)
kmeans.fit(reduced_data)

# Make Predictions
predictions = cluster.predict(patched)

# array of indexes corresponding to classes around centroids, in the order of your dataset
classified_data = kmeans.labels_

#copy dataframe (may be memory intensive but just for illustration)
skid_data = skid_data.copy()
skid_data['Cluster Class'] = pd.Series(classified_data, index=skid_data.index)
print skid_data[['Average Velocity (mph)', 'Cluster Class']]
print list(skid_data.columns)




'''
#Ben Bengfort Visualization

# Find centers
centers = cluster.cluster_centers_
center_colors = colors[:len(centers)]
plt.scatter(centers[:, 0], centers[:, 1], s=100, c=center_colors)

#plt.subplot(1,4,idx+1)
plt.scatter(patched[:, 0], patched[:, 1], color=colors[predictions].tolist(), s=10)<commit_msg>Using cluster of all features and extracting class<commit_after># -*- coding: utf-8 -*-

###############################################################################
# Information
###############################################################################
# Created by <NAME> 
# Input from <NAME>

# In partial fulfillment of the requirements for the Georgetown University Data Analytics Graduate Certificate Program

# useful links
# http://stackoverflow.com/questions/27504870/sklearn-kmeans-get-class-centroid-labels-and-reference-to-a-dataset  -> returns the class of the cluster; can see where each driver's trip fell

# May 1, 2015

# https://plus.google.com/+LinwoodCreekmoreIII/

###############################################################################
# Imports
###############################################################################


print(__doc__)

from time import time
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.decomposition import PCA
from sklearn import metrics
from sklearn.cluster import KMeans
from sklearn.datasets import load_digits
from sklearn.decomposition import PCA
from sklearn.preprocessing import scale
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import StandardScaler
import os


path = path = os.path.abspath(os.getcwd())

# Some colors for later
colors = np.array([x for x in 'bgrcmykbgrcmykbgrcmykbgrcmyk'])
colors = np.hstack([colors] * 20)

'''
df = pd.read_csv('./lin.csv')

digits = np.asfarray(df[['driver_id', 'trip_id', 'Average Velocity (mph)', 'Max Velocity', 'Velocity Stdev','Average Acceleration (mph per s)', 'Max Acceleration (mph per s)', ' Acceleration Stdev','Displacement','Total Distance Traveled','Max Direction Change per sec', ' Direction Stdev','Time (s)', 'Turns', 'Aggressive Turns', 'Stops', 'Large Deceleration Events', 'Deceleration Events', 'Max Deceleration Event']])
data = scale(digits.data)



#####
from sklearn.cluster import MiniBatchKMeans
import matplotlib.pyplot as plt

fig, axe = plt.subplots(figsize=(18, 4))
for idx, dataset in enumerate((circles, moons, blobs, noise)):
    X, y = dataset
    X = StandardScaler().fit_transform(X)
    
'''  


###
 #load data from a CSV to a dataframe
with open("./lin.csv") as in_data:
    skid_data = pd.DataFrame.from_csv(in_data, sep=',')

n_samples, n_features = skid_data.shape
print skid_data.shape

#skid_data=skid_data.fillna(value=-999)

#load all numeric data into an array. The offense column from the crime data
#is excluded
as_array = np.asfarray(skid_data[['Average Velocity (mph)','Max Velocity', 'Velocity Stdev','Average Acceleration (mph per s)', 'Max Acceleration (mph per s)', ' Acceleration Stdev','Displacement','Total Distance Traveled','Max Direction Change per sec', ' Direction Stdev','Time (s)', 'Turns', 'Aggressive Turns', 'Stops', 'Large Deceleration Events', 'Deceleration Events', 'Max Deceleration Event']])

#number of groups
n_clusters=5


#Correct missing data 
imputer = Imputer(missing_values="NaN", strategy="mean")
patched = imputer.fit_transform(as_array)

# Preprocessing tricks
patched = StandardScaler().fit_transform(patched)
#patched = scale(patched, axis=0, with_mean=True)




#cluster data 
cluster = KMeans(n_clusters=n_clusters)
cluster.fit(patched)

# assigned grouped labels to the Skid data
labels = cluster.labels_
skid_data["labels"]=labels
'''
  # Fit the model with our algorithm
cluster = MiniBatchKMeans(n_clusters=3)
cluster.fit(patched)
'''
reduced_data = PCA(n_components=2).fit_transform(patched)
print reduced_data

kmeans = KMeans(init='k-means++', n_clusters=n_clusters, n_init=20)
kmeans.fit(reduced_data)

# Make Predictions
predictions = cluster.predict(patched)

# array of indexes corresponding to classes around centroids, in the order of your dataset
classified_data = kmeans.labels_

#copy dataframe (may be memory intensive but just for illustration)
skid_data = skid_data.copy()
skid_data['Cluster Class'] = pd.Series(classified_data, index=skid_data.index)

print skid_data

print list(skid_data.columns)
#skid_data[['Average Velocity (mph)', 'Cluster Class']].plt()






'''
#<NAME> Visualization

# Find centers
centers = cluster.cluster_centers_
center_colors = colors[:len(centers)]
plt.scatter(centers[:, 0], centers[:, 1], s=100, c=center_colors)

#plt.subplot(1,4,idx+1)
plt.scatter(patched[:, 0], patched[:, 1], color=colors[predictions].tolist(), s=10)
{'fraction_non_alphanumeric': 0.1864290610006854, 'fraction_numerical': 0.009138679460817912, 'mean_word_length': 6.43840271877655, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6092620', 'n_tokens_mistral': 2868, 'n_tokens_neox': 2654, 'n_words': 751}