<jupyter_start><jupyter_text>Ingestion<jupyter_code>df = get_latest_concatenated()
display(df.head())
print(df.shape)
print(df.columns)<jupyter_output>/Users/stevenkim/src/github/nypdcd/.venv/lib/python3.8/site-packages/pandas/core/reshape/concat.py:271: DtypeWarning: Columns (18,20) have mixed types.Specify dtype option on import or set low_memory=False.
  op = _Concatenator(<jupyter_text>Sanity checks  Internal descriptions<jupyter_code>df.PD_DESC.value_counts().head(10)<jupyter_output><empty_output><jupyter_text>Transit crimes<jupyter_code>df.TRANSIT_DISTRICT.value_counts()  # this is something related to NYPD transit enforcement
df.TRANSIT_DISTRICT.count()
df.TRANSIT_DISTRICT.count() / df.shape[0]
df.STATION_NAME.value_counts()
df.STATION_NAME.count()
df.STATION_NAME.count() / df.shape[0]
(df.STATION_NAME.isnull() == df.TRANSIT_DISTRICT.isnull()).all()<jupyter_output><empty_output><jupyter_text>Jurisdictions<jupyter_code>df.JURIS_DESC.value_counts()
df.loc[(df.JURIS_DESC == "N.Y. TRANSIT POLICE") & df.TRANSIT_DISTRICT.isnull()]
df.loc[~(df.JURIS_DESC == "N.Y. TRANSIT POLICE") & ~df.STATION_NAME.isnull()]<jupyter_output><empty_output><jupyter_text>What happens in train stations?<jupyter_code>df.loc[:, "OFNS_DESC"].value_counts() / df.shape[0]
flt = ~df.STATION_NAME.isnull()
df.loc[flt, "OFNS_DESC"].value_counts() / flt.sum()<jupyter_output><empty_output><jupyter_text>Breakdown by race<jupyter_code>cross_table = (
    df
    .groupby(['SUSP_RACE', 'VIC_RACE'])
    ['OFNS_DESC']
    .count()
    .unstack(level=0)
)
cross_table = cross_table.divide(cross_table.sum(axis=1), axis=0)

fig, ax = plt.subplots(1, 1, figsize=(10, 10))
sns.heatmap(
    cross_table, 
    cmap='RdBu_r', 
    ax=ax,
    square=True,
    vmin=0, 
    vmax=0.666,
    annot=True,
    fmt='.3f',
)
anti_asian = (
    df
    .groupby((df.VIC_RACE == "ASIAN / PACIFIC ISLANDER"))
    ["OFNS_DESC"]
    .value_counts()
    .unstack(level=0)
)
anti_asian /= anti_asian.sum(axis=0)
anti_asian.columns = anti_asian.columns.map({True: "AAPI", False: "Other"})
anti_asian = anti_asian.assign(
    abs_gap=anti_asian['AAPI'] - anti_asian['Other'],
    rel_gap=anti_asian['AAPI'] / anti_asian['Other'],
)

anti_asian.sort_values('abs_gap', ascending=False).head(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16199649737302976, 'fraction_numerical': 0.014448336252189142, 'mean_word_length': 6.030769230769231, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '553426', 'license': 'MIT', 'n_tokens_mistral': 987, 'n_tokens_neox': 938, 'n_words': 112, 'path': 'notebooks/exploratory.ipynb', 'repo_name': 'mikss/nypdcd', 'size': '2284'}