<commit_before>#
#      Author: <NAME>
#
#     Created: Mar 17, 2015
#
# Description: Answers to Module 4 Homework
#
#
# https://github.com/dwdii/IS602-AdvProgTech/blob/master/Lesson9/hw9_dittenhafer.py
#
__author__ = '<NAME>'

import csv
from decimal import *
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

def top10PlacesToSwim(dSiteData, bBest, figId):

    dSiteData.set_index(keys=["Site"])
    dSiteData.sort(columns=["EnteroCountInt64"], ascending=bBest, inplace=True)
    top10PlacesToSwim = dSiteData.head(10)
    if bBest:
        print("Top 10 Best Places to Swim")
    else:
        print("Top 10 Worst Places to Swim")
    print("=====================")
    print(top10PlacesToSwim[["Site", "EnteroCountInt64"]])

    fig2 = plt.figure(figId)

    plt1 = fig2.add_subplot(111)
    fig2.subplots_adjust(bottom=.3)
    plt.plot(top10PlacesToSwim["EnteroCountInt64"])
    plt1.set_xticklabels(top10PlacesToSwim["Site"].values, rotation='25')

    plt.show()

def cleanEnteroCount(x):
    if x[0] == "<":
        return int(eval(x[1:] + " - 1"))
    elif x[0] == ">" :
        return int(eval(x[1:] + " + 1"))
    else:
        return int(x)

def main():
    """Our main function."""
    sns.set_palette("deep", desat=.6)
    sns.set_context(rc={"figure.figsize": (8, 4)})

    # file = "https://github.com/jlaurito/CUNY_IS608/raw/master/lecture4/data/riverkeeper_data_2013.csv"
    file = "C:\Users\Dan\SkyDrive\GradSchool\IS608-VizAnalytics\data\\riverkeeper_data_2013.txt"

    # Load the data
    data = pd.read_csv(file, parse_dates=[2], infer_datetime_format=True)

    # Cleanup
    #
    # Convert to datetime data type
    data["Date"] = pd.to_datetime(data["Date"])
    # Convert to int64 (removes the greater/less than and adds/subtracts 1 so integers are reasonably sortable.
    data["EnteroCountInt64"] = data.EnteroCount.apply(cleanEnteroCount)
    print(data.dtypes)
    print(data.head(10))

    #data.sort(columns=["Date", "Site"], ascending=False, inplace=True)
    #print(data)

    dBySiteMaxDate = data.groupby(by=["Site"])["Date"].max()
    dBySiteMaxDate = dBySiteMaxDate.reset_index()
    #dBySiteMaxDate = dBySiteMaxDate.set_index(keys=["Site", "Date"])
    #dBySiteMaxDate.name = "Date"
    print(dBySiteMaxDate)

    # Join the site most recent date to the raw data to get
    # the most recent reading for each site.
    dSiteMaxDateEntero = pd.merge(data, dBySiteMaxDate,  how="inner")
    #print(dSiteMaxDateEntero["EnteroCountInt64"])
    # Export so I can double check via excel
    dSiteMaxDateEntero.to_csv("dSiteMaxDateEntero.csv")

    # 1a. List & graph the (10) best places to swim in the data set.
    top10PlacesToSwim(dSiteMaxDateEntero, True, 1)

    # 1b. List & graph the (10) WORST places to swim in the data set.
    top10PlacesToSwim(dSiteMaxDateEntero, False, 2)


# This is the main of the program.
if __name__ == "__main__":
     main()
<commit_msg>cleaning<commit_after>#
#      Author: <NAME>ittenhafer
#
#     Created: Mar 17, 2015
#
# Description: Answers to Module 4 Homework
#
#
# https://github.com/dwdii/IS602-AdvProgTech/blob/master/Lesson9/hw9_dittenhafer.py
#
__author__ = '<NAME>'

import csv
from decimal import *
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

def top10PlacesToSwim(dSiteData, bBest, figId):

    dSiteData.set_index(keys=["Site"])
    dSiteData.sort(columns=["EnteroCountInt64"], ascending=bBest, inplace=True)
    top10PlacesToSwim = dSiteData.head(10)
    if bBest:
        print("Top 10 Best Places to Swim")
    else:
        print("Top 10 Worst Places to Swim")
    print("=====================")
    print(top10PlacesToSwim[["Site", "EnteroCountInt64"]])

    fig2 = plt.figure(figId)

    plt1 = fig2.add_subplot(111)
    fig2.subplots_adjust(bottom=.3)
    plt.plot(top10PlacesToSwim["EnteroCountInt64"])
    plt1.set_xticklabels(top10PlacesToSwim["Site"].values, rotation='25')

    plt.show()

def cleanEnteroCount(x):
    if x[0] == "<":
        return int(eval(x[1:] + " - 1"))
    elif x[0] == ">" :
        return int(eval(x[1:] + " + 1"))
    else:
        return int(x)

def main():
    """Our main function."""
    sns.set_palette("deep", desat=.6)
    sns.set_context(rc={"figure.figsize": (8, 4)})

    # file = "https://github.com/jlaurito/CUNY_IS608/raw/master/lecture4/data/riverkeeper_data_2013.csv"
    file = "C:\Users\Dan\SkyDrive\GradSchool\IS608-VizAnalytics\data\\riverkeeper_data_2013.txt"

    # Load the data
    data = pd.read_csv(file, parse_dates=[2], infer_datetime_format=True)

    # Cleanup
    #
    # Convert to datetime data type
    data["Date"] = pd.to_datetime(data["Date"])

    # Convert to int64 (removes the greater/less than and adds/subtracts 1 so integers are reasonably sortable.
    data["EnteroCountInt64"] = data.EnteroCount.apply(cleanEnteroCount)
    #print(data.dtypes)

    #data.sort(columns=["Date", "Site"], ascending=False, inplace=True)
    #print(data.head(10))

    dBySiteMaxDate = data.groupby(by=["Site"])["Date"].max()
    dBySiteMaxDate = dBySiteMaxDate.reset_index()
    #dBySiteMaxDate = dBySiteMaxDate.set_index(keys=["Site", "Date"])

    print(dBySiteMaxDate)

    # Join the site most recent date to the raw data to get
    # the most recent reading for each site.
    dSiteMaxDateEntero = pd.merge(data, dBySiteMaxDate,  how="inner")

    # Export so I can double check via excel
    dSiteMaxDateEntero.to_csv("dSiteMaxDateEntero.csv")

    # 1a. List & graph the (10) best places to swim in the data set.
    top10PlacesToSwim(dSiteMaxDateEntero, True, 1)

    # 1b. List & graph the (10) WORST places to swim in the data set.
    top10PlacesToSwim(dSiteMaxDateEntero, False, 2)

    # 2a. Which sites have been tested most regularly?



# This is the main of the program.
if __name__ == "__main__":
     main()

{'fraction_non_alphanumeric': 0.14043500599417708, 'fraction_numerical': 0.03099845864017811, 'mean_word_length': 3.755700325732899, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3733085', 'n_tokens_mistral': 2247, 'n_tokens_neox': 2075, 'n_words': 452}