<commit_before>"""
Creates a colorized kmeans cluster of crime in Washington, DC.
Attempted to add pop-ups but there are too many co-located
points. Data for kmeans is all numeric except for the input
crime labels. All data was normalized and coordinates are 
in WGS 84. Data will not load into ArcGIS online - the site
usually crashes. 
"""

import os
import time
import numpy as np
import pandas as pd
from conf import settings
from datetime import datetime
from bokeh.charts import Scatter
from sklearn.cluster import KMeans
from collections import OrderedDict
from bokeh.plotting import output_file
from sklearn.preprocessing import Imputer

#Create a timestamp for the output file
def time_stamp():
    """
    Creates mm-dd-yyyy timestamp
    """
    now = time.time()
    return datetime.fromtimestamp(now).strftime("%m%d%Y")
    
def create_ordered_dict(data_frame, group_by, x_field="X", y_field="Y"):
    """
    puts data into a dicitonary for Bokeh. 
    """
    pdict = OrderedDict()
    
    g = data_frame.groupby(group_by)
    
    for i in g.groups.keys():
        lat = getattr(g.get_group(i), y_field)
        lng = getattr(g.get_group(i), x_field)
        pdict[i] = zip(lat,lng)
    return pdict
    
def main():
    """
    This function will
        -load data from a csv
        -impute missing data with the column's mean
        -perform kmneans clustering
        -produce an html scatter plot
    """
    
    #load data from a CSV to a dataframe
    crime_data = pd.DataFrame.from_csv(settings["crime_data"], sep=',')
    
    #get all column headers
    headers = crime_data.columns
    
    #load all numeric data into an array. The offense column from the crime data
    #is excluded
    as_array = np.asfarray(crime_data[headers[1:]])
    
    #number of groups
    n_clusters=40
    
    #Correct missing data 
    imputer = Imputer(missing_values=-999, strategy="mean")
    patched = imputer.fit_transform(as_array)
    
    #cluster data 
    cluster = KMeans(n_clusters=n_clusters)
    cluster.fit(patched)
    
    #assigned grouped labels to the crime data
    labels = cluster.labels_
    crime_data["labels"]=labels
    
    pdict = create_ordered_dict(crime_data, "labels")
    
    #location of output graph
    file_name = os.path.join("..", 'tests', "kmeans_clusters_{0}.html".format(time_stamp()))
    output_file(file_name)
    
    #create out graph
    TOOLS="pan,wheel_zoom,box_zoom,reset"
    scatter = Scatter(pdict.values(), title="Crime Clusters", filename=file_name, tools=TOOLS)
    scatter.show()


if __name__ == "__main__":
    main()<commit_msg>set code to only ready X and Y cols<commit_after>"""
Creates a colorized kmeans cluster of crime in Washington, DC.
Attempted to add pop-ups but there are too many co-located
points. Data for kmeans is all numeric except for the input
crime labels. All data was normalized and coordinates are 
in WGS 84. Data will not load into ArcGIS online - the site
usually crashes. 
"""

import os
import time
import numpy as np
import pandas as pd
from conf import settings
from datetime import datetime
from bokeh.charts import Scatter
from sklearn.cluster import KMeans
from collections import OrderedDict
from bokeh.plotting import output_file
from sklearn.preprocessing import Imputer

#Create a timestamp for the output file
def time_stamp():
    """
    Creates mm-dd-yyyy timestamp
    """
    now = time.time()
    return datetime.fromtimestamp(now).strftime("%m%d%Y")
    
def create_ordered_dict(data_frame, group_by, x_field="X", y_field="Y"):
    """
    puts data into a dicitonary for Bokeh. 
    """
    pdict = OrderedDict()
    
    g = data_frame.groupby(group_by)
    
    for i in g.groups.keys():
        lat = getattr(g.get_group(i), y_field)
        lng = getattr(g.get_group(i), x_field)
        pdict[i] = zip(lat,lng)
    return pdict
    
def main():
    """
    This function will
        -load data from a csv
        -impute missing data with the column's mean
        -perform kmneans clustering
        -produce an html scatter plot
    """
    
    #load data from a CSV to a dataframe
    with open(settings["crime_data"]) as in_data:
        crime_data = pd.DataFrame.from_csv(in_data, sep=',')
    
    crime_data.fillna(value=-999)
    
    #load all numeric data into an array. The offense column from the crime data
    #is excluded
    as_array = np.asfarray(crime_data[["X","Y"]])
    
    #number of groups
    n_clusters=40
    
    #Correct missing data 
    imputer = Imputer(missing_values=-999, strategy="mean")
    patched = imputer.fit_transform(as_array)
    
    #cluster data 
    cluster = KMeans(n_clusters=n_clusters)
    cluster.fit(patched)
    
    #assigned grouped labels to the crime data
    labels = cluster.labels_
    crime_data["labels"]=labels
    
    pdict = create_ordered_dict(crime_data, "labels")
    
    #location of output graph
    file_name = os.path.join("..", 'tests', "kmeans_clusters_{0}.html".format(time_stamp()))
    output_file(file_name)
    
    #create out graph
    TOOLS="pan,wheel_zoom,box_zoom,reset"
    scatter = Scatter(pdict.values(), title="Crime Clusters", filename=file_name, tools=TOOLS)
    scatter.show()


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.09290347195726822, 'fraction_numerical': 0.0038153376573826785, 'mean_word_length': 3.1479430379746836, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1966911', 'n_tokens_mistral': 1714, 'n_tokens_neox': 1649, 'n_words': 516}