<jupyter_start><jupyter_text>Make Conviction Clusters<jupyter_code>cr = pd.read_csv("./base_data/conviction-rates.csv")
pr = pd.read_csv('./base_data/prosecution_all_years.csv')
cr.head()
pr.head()
pr = pd.get_dummies(pr, 'Agency Name')
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans, DBSCAN
pr_ss = StandardScaler().fit_transform(pr)
inertia_list = []

for k in range(1, 51):
    kmeans = KMeans(n_clusters=k, random_state=12)
    kmeans.fit(pr_ss)
    inertia_list.append(kmeans.inertia_)
from sklearn.metrics import silhouette_score
silhouette_list = []

for k in range(2, 51):
    kmeans = KMeans(n_clusters=k, random_state=12)
    kmeans.fit(pr_ss)
    silhouette_list.append(silhouette_score(pr_ss, kmeans.labels_))
silhouette_list_db = []

for ep in np.arange(.1,3,.1):
    dbscan = DBSCAN(eps=ep, n_jobs=-1)
    dbscan.fit(pr_ss)
    
    if len(np.unique(dbscan.labels_)) > 1:
        silhouette_list_db.append(silhouette_score(pr_ss, dbscan.labels_))
plt.plot(inertia_list)

plt.plot(silhouette_list)

X = cr[['conviction-rate']]
X = np.round(X.astype('float32'),5)
X[X['conviction-rate'] == np.inf] = 1
from sklearn.cluster import KMeans
kmeans = KMeans(n_clusters=5)
kmeans.fit(X)
X['labels'] = kmeans.labels_
X.groupby('labels').agg([np.mean, np.max, np.min, np.size])
X['label'] = X['labels'].map({0:0, 1:2, 2:1, 4:3, 3:4})
X.groupby('label').agg([np.mean, np.max, np.min, np.size])
plt.scatter(X['label'], X['conviction-rate'])
df.head()
cr['label'] = X['label']
cr.head(1)
df.head(1)
df['label'] = np.NAN
df['reference'] = df['ClosedYear'].map(str) + df['County']
cr['reference'] = cr['Year'].map(str) + cr['Agency Name']
ref_dict = {cr['Co'][i] : cr['label'][i] for i in range(len(cr['reference']))}
df['label'] = df['reference'].map(ref_dict)
df = df.drop(df[df['ClosedYear'] < 2004].index)
df.shape
df.head()
df['label'].unique()
df.drop(columns=['reference'], inplace=True)
df.head()
df.info()
df.drop(df.iloc[:, 17:23], axis=1, inplace=True)
df['WeaponType'].fillna('None', inplace=True)
df.info()
df['Offensive_Act'].fillna('Other', inplace=True)
df['label'].fillna("None", inplace=True)
df.info()
df.to_csv("./cleaned_hatecrime.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2001799370220423, 'fraction_numerical': 0.018443544759334234, 'mean_word_length': 6.082802547770701, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '587399', 'license': 'CC0-1.0', 'n_tokens_mistral': 956, 'n_tokens_neox': 914, 'n_words': 144, 'path': 'scratch/CM_data_cleaning.ipynb', 'repo_name': 'pezpet/group-1-project-4', 'size': '2223'}