<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/gdrive')
import numpy as np
import pandas as pd
import os
d_crime = pd.read_csv("/content/gdrive/My Drive/SF dataset/Police_Department_Incident_Reports__Historical_2003_to_May_2018.csv")
d_crime.columns
np.random.seed(100)
random_d_crime=d_crime.sample(2215024)
train_size=int(0.67 *2215024)
train_df=random_d_crime[:train_size]
test_df=random_d_crime[train_size:]
train_df['DateTime'] = train_df.Date + " " + train_df.Time
train_df['DateTime'] = pd.to_datetime(train_df['DateTime'], format='%m/%d/%Y %H:%M')

test_df['DateTime'] = test_df.Date + " " + test_df.Time
test_df['DateTime'] = pd.to_datetime(test_df['DateTime'], format='%m/%d/%Y %H:%M')
train_df.columns
test_df.columns
train_df.shape
target = train_df["Category"].unique()
print(target.shape)
target
# X = train_df.drop(train_df.columns[[1, 2, 5, 6]], axis = 1)
X = train_df[['DateTime', 'DayOfWeek', 'PdDistrict', 'X', 'Y']]
X.head()
y = train_df.iloc[:, 1]
y.head()
def preprocess_data(dataset):
    dataset['DateTime'] = pd.to_datetime(dataset['DateTime'])
    dataset['Month'] = dataset.DateTime.apply(lambda x: x.month)
    dataset['Day'] = dataset.DateTime.apply(lambda x: x.day)
    dataset['Hour'] = dataset.DateTime.apply(lambda x: x.hour)
    dataset['Minute'] = dataset.DateTime.apply(lambda x: x.minute)
    dataset = dataset.drop('DateTime', 1)
    
    dataset = pd.get_dummies(data=dataset, columns=['DayOfWeek', 'PdDistrict'])
    return dataset
X = preprocess_data(X)
X.head()
from sklearn.preprocessing import LabelEncoder
y = y.to_frame()
le = LabelEncoder()
y["Category"] = le.fit_transform(y["Category"])
print(y.head())
keys = le.classes_
values = le.transform(le.classes_)
dictionary = dict(zip(keys, values))
print(dictionary)
# Splitting the dataset into the Training set and Test set
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
# print(type(X_train), type(y_train))
print(X_train.head())
from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(max_depth = 10, n_estimators = 256)
rf.fit(X_train.values, y_train.values.ravel())
y_pred = rf.predict(X_test)

from sklearn.metrics import accuracy_score
accuracy = accuracy_score(y_test, y_pred)
print ("Accuracy for Random Forest: %.2f%%" % (accuracy * 100.0))
# Computer precision score across multiple classes
from sklearn.metrics import precision_score

precision = precision_score(y_test, y_pred, average='micro')
print("Precision for Random Forest: %.2f%%" % precision)
# Compute recall over multiple classes
from sklearn.metrics import recall_score

recall = recall_score(y_test, y_pred, average='micro')
print("Recall for Random Forest: %.2f%%" % recall)
# Compute f1 score across multiple classes
from sklearn.metrics import f1_score

f1 = f1_score(y_test, y_pred, average='micro')
print("F1 score for Random Forest: %.2f%%" % f1)
from sklearn.metrics import classification_report, confusion_matrix
import seaborn as sns  
cm = confusion_matrix(y_test, y_pred)
cm = pd.DataFrame(cm)
sns.heatmap(cm, cmap="YlGnBu", square=True)<jupyter_output><empty_output><jupyter_text>**Preprocess test_df dataframe**<jupyter_code>test_data = test_df[['DateTime', 'DayOfWeek', 'PdDistrict', 'Address', 'X', 'Y']]
test_data.head()
test_data = preprocess_data(test_data)
test_data.head()
test_data.head()
y_pred_proba = rf.predict_proba(test_data)
colmn = ["ARSON","ASSAULT","BAD CHECKS","BRIBERY","BURGLARY","DISORDERLY CONDUCT","DRIVING UNDER THE INFLUENCE","DRUG/NARCOTIC","DRUNKENNESS","EMBEZZLEMENT","EXTORTION","FAMILY OFFENSES","FORGERY/COUNTERFEITING","FRAUD","GAMBLING","KIDNAPPING","LARCENY/THEFT","LIQUOR LAWS","LOITERING","MISSING PERSON","NON-CRIMINAL","OTHER OFFENSES","PORNOGRAPHY/OBSCENE MAT","PROSTITUTION","RECOVERED VEHICLE","ROBBERY","RUNAWAY","SECONDARY CODES","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE","STOLEN PROPERTY","SUICIDE","SUSPICIOUS OCC","TREA","TRESPASS","VANDALISM","VEHICLE THEFT","WARRANTS","WEAPON LAWS"]
result = pd.DataFrame(y_pred_proba, columns=colmn)
result<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15118223071411513, 'fraction_numerical': 0.013613565798901361, 'mean_word_length': 7.617283950617284, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '143103', 'license': 'MIT', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1504, 'n_words': 290, 'path': 'SF_drug_Random_forest.ipynb', 'repo_name': 'kart2k15/San-Francisco-Opioid-Trends', 'size': '4187'}