<jupyter_start><jupyter_text>NYPD COMPLAINTS DATA VISUALIZATION FINAL PROJECT INFORMATION VISUALIZATION CS-GY 6313 By <NAME> (<EMAIL>) This dataset includes all valid felony, misdemeanor, and violation crimes reported to the New York City Police Department (NYPD) https://data.cityofnewyork.us/Public-Safety/NYPD-Complaint-Data-Historic/qgea-i56i<jupyter_code>import pandas as pd
import ast
from sodapy import Socrata
import os
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
import geopandas as gpd
from keplergl import KeplerGl
from sklearn import metrics
from sklearn import preprocessing
from sklearn.preprocessing import LabelEncoder
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_recall_fscore_support
from sklearn.preprocessing import OrdinalEncoder
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
%matplotlib inline
sns.set_style("whitegrid")
client = Socrata("data.cityofnewyork.us", None)
results = client.get("qgea-i56i",order = "cmplnt_fr_dt DESC",limit = 1000000)
results_df = pd.DataFrame.from_records(results)
df = pd.read_csv('data.csv')
df2 = df
results_df.info()
df.info()
results_df.drop(['cmplnt_to_tm','rpt_dt','pd_cd','pd_desc','juris_desc','jurisdiction_code','housing_psa','x_coord_cd','y_coord_cd','lat_lon','patrol_boro','parks_nm','hadevelopt','transit_district','station_name'],axis = 1,inplace = True)
df.drop(['CMPLNT_TO_TM','RPT_DT','PD_CD','PD_DESC','JURIS_DESC','JURISDICTION_CODE','HOUSING_PSA','X_COORD_CD','Y_COORD_CD','Lat_Lon','PATROL_BORO','PARKS_NM','HADEVELOPT','TRANSIT_DISTRICT','STATION_NAME'],axis = 1,inplace = True)

results_df.dropna(subset=['cmplnt_fr_tm'], inplace=True)
results_df['date'] = pd.to_datetime(results_df['cmplnt_fr_dt'],errors = 'coerce')
df['date'] = pd.to_datetime(df['CMPLNT_FR_DT'],errors = 'coerce')
results_df['month'] = results_df['date'].dt.month
results_df['year'] =results_df['date'].dt.year
results_df['day'] = results_df['date'].dt.day

df['month'] = df['date'].dt.month
df['year'] =df['date'].dt.year
df['day'] = df['date'].dt.day
df['SUSP_AGE_GROUP'].value_counts()
values = ['-958', '-43','-974', '-972', '972', '951', '-942']
val = [np.nan, '928','952', '1018', '920', '2018', '938', '-2', '954', '1967']
df = df[df['VIC_AGE_GROUP'].isin(values) == False]
df = df[df['SUSP_AGE_GROUP'].isin(val) == False]
fig, ax = plt.subplots(figsize=(6, 5), dpi=100)
sns.countplot(y='month', data=results_df,palette="Blues_d",ax=ax)
plt.title('Count of all the Crimes month wise')
plt.show()
fig, ax = plt.subplots(figsize=(6, 5), dpi=100)
results_df.ofns_desc.value_counts().iloc[:10].sort_values().plot(kind="barh", color = '#347002',title = "Types of Crimes",ax=ax)
plt.show()
boro_df = results_df.groupby(['cmplnt_fr_dt']).agg({'cmplnt_fr_dt':'count'})
boro_df = results_df.groupby(['boro_nm']).agg({'cmplnt_num':'count'})
crimes_per_boro = boro_df.sort_values('cmplnt_num', ascending=False)
crimes_per_boro 
cpb_prop = crimes_per_boro.assign(percentage = crimes_per_boro.cmplnt_num / sum(crimes_per_boro.cmplnt_num))
cpb_prop = cpb_prop.reset_index()
x = cpb_prop['boro_nm']
y = cpb_prop['percentage']
colors = ['#2299aa', '#40b0bf', '#65c6d4', '#98dee8', '#c7f2f9']
fig, ax= plt.subplots(1,2,figsize=(15,9))
plt.subplot(1, 2, 1)
plot = results_df['boro_nm'].value_counts().sort_values().plot(kind="bar", color = list(reversed(colors)), title = 'Total of Crime Events by Borough')
plt.subplot(1, 2, 2)
plt.pie(y,labels=x,colors=colors,autopct = '%0.2f%%')
plt.title('Percentage of Crime Events by Borough')
plt.show()
fig, ax = plt.subplots(figsize=(6, 5), dpi=100)
sns.countplot(x='BORO_NM', hue="LAW_CAT_CD", data=df,ax=ax,color='#ff5110')
plt.title('Type of Offense by Borough')
plt.show()
corr = df.apply(lambda x : pd.factorize(x)[0]).corr()
plt.figure(figsize = (30,30))
sns_plot = sns.heatmap(corr, cmap = "seismic", linewidth = 2, linecolor = "white", annot=True,vmin=-1, vmax=1, center=0, square=True)
plt.title('Correlation matrix between All the Labels')
plt.savefig("corr.png")
fig, ax= plt.subplots(2,2,figsize=(25,20))
sns.countplot(x='VIC_SEX', hue="LAW_CAT_CD", data=df,ax=ax[0,0],color='#000043')
ax[0,0].set_title("Type of Offense Classified by Victim's Sex")
sns.countplot(x='VIC_AGE_GROUP', hue="LAW_CAT_CD", data=df,ax=ax[1,0],color='#005643')
ax[1,0].set_title("Type of Offense Classified by Victim's Age Group")
sns.countplot(x='VIC_RACE', hue="LAW_CAT_CD", data=df,ax=ax[0,1],color='#214301')
ax[0,1].set_title("Type of Offense Classified by Victim's Race")
cross_table = df.groupby(['SUSP_AGE_GROUP', 'VIC_AGE_GROUP'])['LAW_CAT_CD'].count()
cross_table = cross_table.divide(cross_table.sum(), axis=0).unstack(level=0)
sns.heatmap(cross_table, cmap="Greens",square=True,vmin=0, vmax=0.30,annot=True,fmt='.3f',ax=ax[1,1])
ax[1,1].set_title('Correlation matrix between Suspect and Victim Age Group for the type of Offense')
plt.show()
fig, ax= plt.subplots(2,2,figsize=(25,20))
sns.countplot(x='SUSP_SEX', hue="LAW_CAT_CD", data=df,ax=ax[0,0],color='#510343')
ax[0,0].set_title("Type of Offense Classified by Suspect's Sex")
sns.countplot(x='SUSP_AGE_GROUP', hue="LAW_CAT_CD", data=df,ax=ax[0,1],color='#ff0000')
ax[0,1].set_title("Type of Offense Classified by Suspect's Age Group")
sns.countplot(x='SUSP_RACE', hue="LAW_CAT_CD", data=df,ax=ax[1,0],color='#ff00ff')
ax[1,0].set_title("Type of Offense Classified by Suspect's Race")
cross_table = df.groupby(['SUSP_SEX', 'VIC_SEX'])['LAW_CAT_CD'].count()
cross_table = cross_table.divide(cross_table.sum(), axis=0).unstack(level=0)
sns.heatmap(cross_table, cmap="Reds",square=True,vmin=0, vmax=0.30,annot=True,fmt='.3f',ax=ax[1,1])
ax[1,1].set_title('Correlation matrix between Suspect and Victim Sex for the type of Offense')
plt.show()
corr = results_df.groupby(['susp_race', 'vic_race'])['ofns_desc'].count()
corr = corr.divide(corr.sum(), axis=0).unstack(level=0)
fig, ax = plt.subplots(1, 1, figsize=(10, 10))
sns.heatmap(corr, cmap="icefire", ax=ax,square=True,vmin=0, vmax=0.30,annot=True,fmt='.3f',)
plt.title('Correlation matrix between Suspect and Victim Race')
plt.show()
df3 = df.dropna()
df3['date'] = df3['date'].astype(str)
f = open("config_color.txt", "r")
config = ast.literal_eval(f.read())
map1 = KeplerGl(data={'NYC_CRIMES': df3}, height=600, width=800)
map1.config = config
map1
map1.save_to_html(file_name = 'testtype.html')
%%html
<iframe src="testtype.html" width="1000" height="1000"></iframe>
f = open("config_hex.txt", "r")
config = ast.literal_eval(f.read())
map_Hexogon = KeplerGl(data={'NYC_CRIMES': df3}, height=600, width=800)
map_Hexogon.config = config
map_Hexogon
map_Hexogon.save_to_html(file_name = 'hexamap_001.html')
%%html
<iframe src="hexamap_001.html" width="1000" height="1000"></iframe>
def extract(dataframe):
    date= results_df['date']
    month=results_df['month']
    longitude=results_df["longitude"]
    latitude=results_df["latitude"]
    boro= results_df['boro_nm']
    loc=results_df['prem_typ_desc']
    cat = results_df['law_cat_cd']
    return pd.concat([date, month, boro, loc, latitude, longitude,cat], axis=1)
data = extract(results_df)
data
data = data.dropna()
X = data.drop(['law_cat_cd'],axis=1)
Y = data['law_cat_cd']
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=42)
print(X_train.shape,y_train.shape,X_test.shape,y_test.shape)
encoder = OrdinalEncoder()
encoder.fit(X_train)
X_train_encoded = encoder.transform(X_train)
encoder.fit(X_test)
X_test_encoded = encoder.transform(X_test)
X_test_encoded
print(X_train_encoded.shape,y_train.shape,X_test_encoded.shape,y_test.shape)
knn = KNeighborsClassifier(n_neighbors=7)
clf = knn.fit(X_train_encoded,y_train)
knn_p = clf.predict(X_test_encoded)
accuracy_knn = accuracy_score(y_test, knn_p)
print(accuracy_knn)
dt = DecisionTreeClassifier(random_state=0,max_depth = 7)
dt = dt.fit(X_train_encoded,y_train)
dtp = dt.predict(X_test_encoded)
accuracy_dt = accuracy_score(y_test, dtp)
print(accuracy_dt)
gnb = GaussianNB()
gnb = gnb.fit(X_train_encoded,y_train)
gnb_p = gnb.predict(X_test_encoded)
accuracy_nb = accuracy_score(y_test, gnb_p)
print(accuracy_nb)
rf = RandomForestClassifier(max_depth=7, random_state=0)
rf.fit(X_train_encoded, y_train)
rfp = rf.predict(X_test_encoded)
accuracy_rf = accuracy_score(y_test, rfp)
print(accuracy_rf)<jupyter_output>0.5287530631101113
{'fraction_non_alphanumeric': 0.17152217152217153, 'fraction_numerical': 0.03556803556803557, 'mean_word_length': 8.582959641255606, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '255619', 'license': 'MIT', 'n_tokens_mistral': 3653, 'n_tokens_neox': 3388, 'n_words': 569, 'path': 'nkk6190_Final_Project.ipynb', 'repo_name': 'nikhilkhaneja/NYPD-Complaints-Data-Visualization', 'size': '8570'}