<jupyter_start><jupyter_text>Import block group & join with crimes<jupyter_code>def process_blocks():
    df = pd.read_csv('blocks.csv')
    df['GEOID10'] = df['GEOID10'].astype(str)
    df["block_group"] = df["GEOID10"].apply(lambda block: block[:12])
    df["the_geom"] = df["the_geom"].apply(shapely.wkt.loads)
    gdf = gpd.GeoDataFrame(df).set_geometry("the_geom").drop(columns=df.columns.difference(["block_group", "the_geom"]))
    gdf = gpd.GeoDataFrame(gdf).set_geometry('the_geom')
    return gdf
blocks_df = process_blocks()
crime_data = crime_data[crime_data['Latitude'].notnull()]
crime_data['the_geom'] = crime_data.apply(lambda row: Point(float(row["Longitude"]), float(row["Latitude"])), axis=1)
crimes_gdf = gpd.GeoDataFrame(crime_data).set_geometry('the_geom')
joined = gpd.sjoin(crimes_gdf, blocks_df, how="left", op='intersects')
joined.head()
joined['Date'] = joined['Date'].str.split(' ',expand=True)[0]
joined['Date'] = pd.to_datetime(joined['Date'])
weather_data['Date'] = pd.to_datetime(weather_data['DATE'])
combined = pd.merge(joined, weather_data, on='Date', how='outer')
combined
dfs_to_concat = [combined]
for col in ['Primary Type']:
    dummy_df = pd.get_dummies(combined[col])
    dfs_to_concat.append(dummy_df)
dataframe = pd.concat(dfs_to_concat, axis=1)
dataframe.head()
def get_crime_dict(crime_list):
    rt = {}
    for idx, crime in enumerate(crime_list):
        rt[crime] = idx
    return rt

def limit_by_crime_type(crime_list, df):
    return df[ df["Primary Type"].isin(crime_list)]

def replace_crime_type_with_id(df, crime_dict):
    df["Target"] = [0 for i in range(df.shape[0])]
    for idx, row in df.iterrows():
        c = row["Primary Type"]
        c_id = crime_dict[c]
        df.at[idx, "Target"] = c_id        
    return df

def main(data):
    types_of_crime = ["THEFT", "HOMICIDE", "BATTERY"]
    crime_dict = get_crime_dict(types_of_crime)
    df = limit_by_crime_type(types_of_crime, data)
    replaced_df = replace_crime_type_with_id(df, crime_dict)
    xs_list = ["Target", "TAVG", "Date", "Primary Type", "block_group"]
    model_df = replaced_df[xs_list]
    return model_df
df = main(dataframe)
def limit_blocks_by_count(df):

    blocks = {}
    for idx, row in df.iterrows():
        b = row["block_group"]
        if b not in blocks:
            blocks[b] = 1
        else:
            blocks[b] += 1
            
    keepers = {}
    for k, v in blocks.items():
        if v > 10:
            keepers[k] = v
    
    return keepers

b = limit_blocks_by_count(df)
blocks = list(b.keys())

limited_by_blocks = df[ df["block_group"].isin(blocks)]
binary = pd.get_dummies(limited_by_blocks, columns=["block_group"])
limited_by_blocks.head()
binary.head()
binary.to_csv("sagemaker_data.csv", index=False)<jupyter_output><empty_output><jupyter_text>Data Cleaning & Feature Generation<jupyter_code>dataframe.columns<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1522860089377793, 'fraction_numerical': 0.005843932622894466, 'mean_word_length': 4.564053537284895, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '72103', 'license': 'MIT', 'n_tokens_mistral': 1106, 'n_tokens_neox': 1067, 'n_words': 189, 'path': 'predicting_crime.ipynb', 'repo_name': 'katykoenig/amazon-sagemaker-architecting-for-ml', 'size': '2909'}