<jupyter_start><jupyter_text>LA County Shapefile<jupyter_code># Graph neighbors of a region
def graph_neighbors(df, region):
    df['plot_col'] = 0
    df.plot_col = df.plot_col.mask(df['region'] == region, 1)
    temp_n = df[df.region == region].neighbors.values[0].split(',')
    temp_n = [n.strip() for n in temp_n]
    for n in temp_n:
        df.plot_col = df.plot_col.mask(df['region'] == n, 2)
    ax = df.plot(column='plot_col', figsize=(10,10))
    green_patch = mpatches.Patch(color='green', label='Covering Polygon')
    yellow_patch = mpatches.Patch(color='yellow', label='Bordering Regions')
    purple_patch = mpatches.Patch(color='purple', label='Inner LA County')
    plt.legend(handles=[green_patch, yellow_patch, purple_patch])
#     ax.annotate()
#     df.apply(lambda x: ax.annotate(s=x.region, xy=x.geometry.centroid.coords[0], ha='center'),axis=1);
    
# Graph region
def graph_region(df, region):
    df['plot_col'] = 0
    df.plot_col = df.plot_col.mask(df['region'] == region, 1)

    df.plot(column='plot_col', figsize=(10,10), alpha=0.5)
# Load the shape file from github
# !wget -Nq https://raw.githubusercontent.com/kaushiktandon/COVID-19-Vaccine-Allocation/master/shape/la_shape.zip
# !unzip -nq la_shape -d shape
# la_shapes = gpd.read_file("shape/la_shape.shp") # L.A. County neighborhoods shapefile
la_shapes = gpd.read_file("shape/la_shape.shp") # L.A. County neighborhoods shapefile
la_filtered = la_shapes.rename(columns={'name':'region'})
la_filtered["neighbors"] = None  # add NEIGHBORS column
la_filtered["plot_col"] = 0 # Column used to graph neighbors
la_filtered = la_filtered[['region','geometry','neighbors','plot_col']]
la_filtered.head()<jupyter_output><empty_output><jupyter_text>Get bordering regions<jupyter_code># Create a polygon that covers the entire la county region
cover = gpd.GeoSeries(Polygon([(-119.4,35), (-119.4,33.2), (-117,33.2), (-117,35)])).set_crs(epsg=4269)
# Remove Catalina Island since it is an island which has no neighbors
la_noisland = la_filtered[la_filtered['region']!='Unincorporated Catalina Island']
# Remove small holes within the la county unary union polygon
no_holes = MultiPolygon(Polygon(p.exterior) for p in la_noisland.geometry.unary_union)
no_holes_series = gpd.GeoSeries(no_holes).set_crs(epsg=4269)
# Subtract la county from the covering polygon
diff = gpd.GeoSeries.difference(cover, no_holes_series)
# We can find the cities that are touching 'diff' to locate border cities
# diff.plot(figsize=(5,5))
# Get neighbors of diff to obtain bordered regions
border_regions = la_filtered[~la_filtered.geometry.disjoint(diff[0])].region.tolist()
la_filtered.loc[len(la_filtered)] = ["COVER", diff[0], ", ".join(border_regions), 0]
graph_neighbors(la_filtered, 'COVER')
la_filtered = la_filtered[:-1]<jupyter_output><empty_output><jupyter_text>Get neighbors of each region<jupyter_code>for index, row in la_filtered.iterrows():   
    # Get neighboring cities
    neighbors = la_filtered[~la_filtered.geometry.disjoint(row.geometry)].region.tolist()
    # Remove own name from list
    if row['region'] in neighbors:
        neighbors.remove(row['region'])
    la_filtered.at[index, "neighbors"] = ", ".join(neighbors)
graph_neighbors(la_filtered, 'Winnetka')
la_filtered = la_filtered[['region','neighbors','geometry']]<jupyter_output><empty_output><jupyter_text>Remove bordering regions<jupyter_code>la_inner = la_filtered[~la_filtered['region'].isin(border_regions)]
la_inner.shape<jupyter_output><empty_output><jupyter_text>Smooth/process COVID data<jupyter_code>!wget --directory-prefix=../data/covid -Nq https://raw.githubusercontent.com/ANRGUSC/lacounty_covid19_data/master/data/Covid-19.csv
covid_filename = '../data/covid/Covid-19.csv'

covid_df = pd.read_csv(covid_filename)
covid_df = covid_df.rename(columns={'Region':'region', 'Number of cases':'cases'})
covid_df.head()
# Set COVID cases to not be less than previously recorded - it should be a non-decreasing function
for region in covid_df["region"].unique():
  df = covid_df[covid_df['region'] == region]
  prev_cases = 0
  for idx, row in df.iterrows():
    if prev_cases > row['cases']:
      covid_df.loc[idx, 'cases'] =  prev_cases
    else:
      prev_cases = row['cases']
for region in covid_df["region"].unique():
  df = covid_df[covid_df['region'] == region]
  prev_cases = 0
  for idx, row in df.iterrows():
    covid_df.loc[idx, 'active_cases'] =  row['cases'] - prev_cases
    prev_cases = row['cases']
covid_df[covid_df['region'] == 'Alhambra']<jupyter_output><empty_output><jupyter_text>Fix inconsistencies in region name mapping<jupyter_code>la_to_covid = {
    'Downtown': ['City', 'Pico', 'Wholesale District', 'Central', 'Little Tokyo', 'Temple-Beaudry'],
    'Silver Lake': ['Silverlake'],
    'Vernon': ['West Vernon', 'Vernon Central'],
    'Baldwin Hills/Crenshaw': ['Baldwin Hills'],
    'West Hollywood': ['Park LaBrea'],
    'Mid-Wilshire': ['Park LaBrea', 'Cloverdale/Cochran'],
    'Santa Clarity': ['Canyon Country'],
    'Echo Park': ['Angelino Heights'],
    'Avocado Heights': ['Bassett'],
    'Bel-Air': ['Bel Air'],
    'Arlington Heights': ['Country Club Park'],
    'Exposition Park': ['Exposition'],
    'Chinatown': ['Figueroa Park Square'],
    'Torrance': ['Gramercy Place'],
    'Harbor City': ['Harbor Pines'],
    'Mid-City': ['Lafayette Square', 'Mid-city', 'Reynier Village', 'Victoria Park', 'Wellington Square'],
    'Azusa': ['Lakeview Terrace'],
    'East Hollywood': ['Little Armenia'],
    'Playa Vista': ['Longwood'],
    'Brentwood': ['Mandeville Canyon'],
    'Playa del Rey': ['Playa Del Rey'],
    'Porter Ranch': ['Reseda Ranch'],
    'Carthay': ['South Carthay'],
    'Los Feliz': ['Thai Town'],
    'Toluca Lake': ['Toluca Terrace'],
    'El Sereno': ['University Hills'],
    'Angeles Crest': ['Angeles National Forest'],
    'Mount Washington': ['Mt. Washington']
}

for key, value in la_to_covid.items():
    covid_df.loc[covid_df['region'].isin(value), 'region'] = key<jupyter_output><empty_output><jupyter_text>Set low pop covid cases to 0<jupyter_code>covid_unique_regions = pd.DataFrame(data=pd.unique(covid_df.region), columns=['region'])
population_df = pd.read_csv('../data/social_explorer_processed_data.csv')[['name','Total Population:']]
population_df = population_df.rename(columns={'name':'region', 'Total Population:': 'population'})
# population_df.plot.hist(by='region', figsize=(10,10))<jupyter_output><empty_output><jupyter_text>Histogram of pop of regions w/ covid data<jupyter_code>covid_population = population_df[population_df['region'].isin(pd.unique(covid_df.region))]
covid_population[covid_population.population <= 40000].plot.hist(by='region', figsize=(5,5), bins=10)<jupyter_output><empty_output><jupyter_text>Histogram of pop of regions w/ missing covid data<jupyter_code>check_df = population_df.assign(InDf2=population_df.region.isin(covid_unique_regions.region))
miss_arr = pd.unique(check_df[check_df['InDf2']==False]['region'])
covid_no_data = population_df[population_df['region'].isin(miss_arr)]
covid_no_data[covid_no_data.population <= 100000].plot.hist(by='region', figsize=(5,5), bins=10)<jupyter_output><empty_output><jupyter_text>Smaller regions have more missing data in general, but there are also outliers Small regions w/ a lot of covid cases<jupyter_code>small_pop_region = population_df[population_df['population'] <= 5000]['region'].values
pd.unique(covid_df_month[covid_df_month.region.isin(small_pop_region)].region)
covid_df_month[covid_df_month.region.isin(small_pop_region)].groupby('region').sum()<jupyter_output><empty_output><jupyter_text>Big regions w/ no covid cases<jupyter_code>print(covid_no_data[covid_no_data.population >= 30000].region.values)<jupyter_output>['Castaic Canyons' 'Central-Alameda' 'Historic South-Central' 'Long Beach'
 'Sawtelle' 'Vermont-Slauson' 'Westmont']<jupyter_text>Remove regions w/ missing neighbors & their neighbors<jupyter_code>covid_unique_regions = pd.DataFrame(data=pd.unique(covid_df.region), columns=['region'])
check_df = la_inner.assign(InDf2=la_inner.region.isin(covid_unique_regions.region))
miss_arr = pd.unique(check_df[check_df['InDf2']==False]['region'])
miss_set = set(miss_arr)
for region in miss_arr:
    temp_n = la_inner[la_inner.region == region].neighbors.values[0].split(',')
    temp_n = set([n.strip() for n in temp_n])
    miss_set = miss_set | temp_n
miss_arr = list(miss_set)
la_inner = la_inner[~la_inner['region'].isin(miss_arr)]
la_inner.shape<jupyter_output><empty_output><jupyter_text>Aggregate Covid Data  Census data<jupyter_code>census_data = pd.read_csv('../data/social_explorer_processed_data_only_percents.csv')
census_data.shape
census_data.dropna().dtypes
# Get all regions that have data
census_unique_regions = pd.DataFrame(data=pd.unique(census_data.name),columns=['region'])
regions = pd.merge(covid_unique_regions, la_inner[['region']], on='region', how='inner')
regions = pd.merge(regions, census_unique_regions, on='region', how='inner')['region'].values
len(regions)<jupyter_output><empty_output><jupyter_text>Month aggregate<jupyter_code>covid_df_month = covid_df[['Time Stamp', 'region', 'active_cases']]
covid_df_month['Time Stamp'] = pd.to_datetime(covid_df['Time Stamp'])
covid_df_month = covid_df_month.set_index('Time Stamp')
covid_df_month = covid_df_month.groupby([pd.Grouper(freq='M'), 'region']).sum()
covid_df_month = covid_df_month.reset_index(level=['Time Stamp','region']).rename(columns={'Time Stamp': 'month'})
covid_df_month.month = covid_df_month.month.dt.month
covid_df_month
covid_df[covid_df['region'] == 'East Los Angeles']<jupyter_output><empty_output><jupyter_text>Data w/o census<jupyter_code>month_data_no_census = np.array([])
# List of months
time_frames = pd.unique(covid_df_month.month)
# List of month case sums
covid_month_gp = covid_df_month.groupby('month').sum().reset_index()

for i in range(len(time_frames)-2): ## Need to ignore last month since data for last month is not complete
    time_frame = time_frames[i]
    for region in regions:
        # Build current row of data
        row_data = np.array([time_frame])
        # Get cases for current region
        region_cases = covid_df_month.loc[(covid_df_month.month == time_frame) & (covid_df_month.region == region)]['active_cases'].values
        if len(region_cases) == 0:
            region_cases = 0
        else:
            region_cases = region_cases[0]
        row_data = np.append(row_data, [region_cases])
        
        # Get neighbors cases, fill missing neighbor data with avg of other neighbors
        neighbors = (la_inner[la_inner['region']==region]['neighbors']).values[0]
        neighbors = neighbors.split(',')
        neighbors = [neighbor.strip() for neighbor in neighbors]
        neighbors_cases = 0
        no_data_count = 0
        for neighbor in neighbors:
            neighbor_cases_arr= covid_df_month.loc[(covid_df_month.month == time_frame) & (covid_df_month.region == neighbor)]['active_cases'].values
            if len(neighbor_cases_arr) != 0:
                neighbors_cases += neighbor_cases_arr[0]
            else:
                no_data_count += 1
        if no_data_count == len(neighbors):
            neighbors_cases += region_cases * len(neighbors)
        else:
            neighbors_cases += (no_data_count/(len(neighbors)-no_data_count))*neighbors_cases ## Use avg to fill missing data and add to neighbors_cases
        row_data = np.append(row_data, [neighbors_cases])
        
        # Get cases of the entire LA region
        total_cases = covid_month_gp[covid_month_gp['month'] == time_frame]['active_cases'].values[0]
        row_data = np.append(row_data, [total_cases])
        
        # Get cases for next time_frame
        next_cases = covid_df_month.loc[(covid_df_month.month == time_frames[i+1]) & (covid_df_month.region == region)]['active_cases'].values
        # This data is not available yet
        if len(next_cases) == 0:
            next_cases = 0
        else:
            next_cases = next_cases[0]
        row_data = np.append(row_data, [next_cases])
        
        # Add data to numpy arr
        if len(month_data_no_census.shape) == 1:
            month_data_no_census = np.array([row_data])
        else:
            month_data_no_census = np.append(month_data_no_census, [row_data], axis=0)
month_data_no_census
month_data_no_census.shape
month_data_with_census = np.array([])
# List of months
time_frames = pd.unique(covid_df_month.month)
# List of month case sums
covid_month_gp = covid_df_month.groupby('month').sum().reset_index()

for i in range(len(time_frames)-2): ## Need to ignore last month since data for last month is not complete
    time_frame = time_frames[i]
    for region in regions:
        # Build current row of data
        row_data = np.array([time_frame])
        # Get cases for current region
        region_cases = covid_df_month.loc[(covid_df_month.month == time_frame) & (covid_df_month.region == region)]['active_cases'].values
        if len(region_cases) == 0:
            region_cases = 0
        else:
            region_cases = region_cases[0]
        row_data = np.append(row_data, [region_cases])
        
        # Get neighbors cases, fill missing neighbor data with avg of other neighbors
        neighbors = (la_inner[la_inner['region']==region]['neighbors']).values[0]
        neighbors = neighbors.split(',')
        neighbors = [neighbor.strip() for neighbor in neighbors]
        neighbors_cases = 0
        no_data_count = 0
        for neighbor in neighbors:
            neighbor_cases_arr= covid_df_month.loc[(covid_df_month.month == time_frame) & (covid_df_month.region == neighbor)]['active_cases'].values
            if len(neighbor_cases_arr) != 0:
                neighbors_cases += neighbor_cases_arr[0]
            else:
                no_data_count += 1
        if no_data_count == len(neighbors):
            neighbors_cases += region_cases * len(neighbors)
        else:
            neighbors_cases += (no_data_count/(len(neighbors)-no_data_count))*neighbors_cases ## Use avg to fill missing data and add to neighbors_cases
        row_data = np.append(row_data, [neighbors_cases])
        
        # Get cases of the entire LA region
        total_cases = covid_month_gp[covid_month_gp['month'] == time_frame]['active_cases'].values[0]
        row_data = np.append(row_data, [total_cases])
        
        # Get census data
        census_row = census_data[census_data['name']==region].values[0][1:]/100
        row_data = np.append(row_data, census_row)
        
        # Get cases for next time_frame
        next_cases = covid_df_month.loc[(covid_df_month.month == time_frames[i+1]) & (covid_df_month.region == region)]['active_cases'].values
        # This data is not available yet
        if len(next_cases) == 0:
            next_cases = 0
        else:
            next_cases = next_cases[0]
        row_data = np.append(row_data, [next_cases])
        
        # Add data to numpy arr
        if len(month_data_with_census.shape) == 1:
            month_data_with_census = np.array([row_data])
        else:
            month_data_with_census = np.append(month_data_with_census, [row_data], axis=0)
month_data_with_census = month_data_with_census.astype(float)
month_data_with_census
month_data_with_census[:,~np.isnan(month_data_with_census).any(axis=0)].shape
month_data_with_census.dtype
np.savetxt("../data/covid/covid_month_with_census_filtered.csv", month_data_with_census, delimiter=",")
# np.save('../data/covid/covid_month_no_census_filtered', month_data_no_census)
np.save('../data/covid/covid_month_with_census_filtered', month_data_with_census)<jupyter_output><empty_output><jupyter_text>Week Aggregate<jupyter_code>covid_df_week = covid_df[['Time Stamp', 'Region', 'Number of cases']]
covid_df_week['Time Stamp'] = pd.to_datetime(covid_df['Time Stamp'])
covid_df_week = covid_df_week.set_index('Time Stamp')
covid_df_week = covid_df_week.groupby([pd.Grouper(freq='W'), 'Region']).sum()
covid_df_week = covid_df_week.reset_index(level=['Time Stamp','Region']).rename(columns={'Time Stamp': 'week', 'Region':'region', 'Number of cases':'cases'})
covid_df_week.week = covid_df_week.week.dt.week
covid_df_week
week_data_no_census = np.array([])
# List of months
time_frames = pd.unique(covid_df_week.week)
# List of month case sums
covid_week_gp = covid_df_week.groupby('week').sum().reset_index()

for i in range(len(time_frames)-2): ## Need to ignore last week since data for last week is not complete
    time_frame = time_frames[i]
    for region in regions:
        # Build current row of data
        row_data = np.array([time_frame])
        
        # Get cases for current region
        region_cases = covid_df_week.loc[(covid_df_week.week == time_frame) & (covid_df_week.region == region)]['cases'].values
        if len(region_cases) == 0:
            region_cases = 0
        else:
            region_cases = region_cases[0]
        row_data = np.append(row_data, [region_cases])
        
        # Get neighbors cases
        neighbors = (la_neighbors[la_neighbors['region']==region]['neighbors']).values[0]
        neighbors = neighbors.split(',')
        neighbors = [neighbor.strip() for neighbor in neighbors]
        neighbors_cases = 0
        no_data_count = 0
        for neighbor in neighbors:
            neighbor_cases_arr= covid_df_month.loc[(covid_df_week.week == time_frame) & (covid_df_week.region == neighbor)]['cases'].values
            if len(neighbor_cases_arr) != 0:
                neighbors_cases += neighbor_cases_arr[0]
            else:
                no_data_count += 1
        if no_data_count == len(neighbors):
            neighbors_cases += region_cases * len(neighbors) ## All neighbor data missing, fill with region itself
        else:
            neighbors_cases += (no_data_count/(len(neighbors)-no_data_count))*neighbors_cases ## Use avg to fill missing data and add to neighbors_cases
        row_data = np.append(row_data, [neighbors_cases])
        
        # Get cases of the entire LA region
        total_cases = covid_week_gp[covid_week_gp['week'] == time_frame]['cases'].values[0]
        row_data = np.append(row_data, [total_cases])
        
        # Get cases for next time_frame
        next_cases = covid_df_week.loc[(covid_df_week.week == time_frames[i+1]) & (covid_df_week.region == region)]['cases'].values
        # This data is not available yet
        if len(next_cases) == 0:
            next_cases = 0
        else:
            next_cases = next_cases[0]
        row_data = np.append(row_data, [next_cases])

        # Add data to numpy arr
        if len(week_data_no_census.shape) == 1:
            week_data_no_census = np.array([row_data])
        else:
            week_data_no_census = np.append(week_data_no_census, [row_data], axis=0)
week_data_no_census
week_data_no_census.shape
week_data_with_census = np.array([])
# List of months
time_frames = pd.unique(covid_df_week.week)
# List of month case sums
covid_week_gp = covid_df_week.groupby('week').sum().reset_index()

for i in range(len(time_frames)-2): ## Need to ignore last week since data for last week is not complete
    time_frame = time_frames[i]
    for region in regions:
        # Build current row of data
        row_data = np.array([time_frame])
        
        # Get cases for current region
        region_cases = covid_df_week.loc[(covid_df_week.week == time_frame) & (covid_df_week.region == region)]['cases'].values
        if len(region_cases) == 0:
            region_cases = 0
        else:
            region_cases = region_cases[0]
        row_data = np.append(row_data, [region_cases])
        
        # Get neighbors cases
        neighbors = (la_neighbors[la_neighbors['region']==region]['neighbors']).values[0]
        neighbors = neighbors.split(',')
        neighbors = [neighbor.strip() for neighbor in neighbors]
        neighbors_cases = 0
        no_data_count = 0
        for neighbor in neighbors:
            neighbor_cases_arr= covid_df_month.loc[(covid_df_week.week == time_frame) & (covid_df_week.region == neighbor)]['cases'].values
            if len(neighbor_cases_arr) != 0:
                neighbors_cases += neighbor_cases_arr[0]
            else:
                no_data_count += 1
        if no_data_count == len(neighbors):
            neighbors_cases += region_cases * len(neighbors) ## All neighbor data missing, fill with region itself
        else:
            neighbors_cases += (no_data_count/(len(neighbors)-no_data_count))*neighbors_cases ## Use avg to fill missing data and add to neighbors_cases
        row_data = np.append(row_data, [neighbors_cases])
        
        # Get cases of the entire LA region
        total_cases = covid_week_gp[covid_week_gp['week'] == time_frame]['cases'].values[0]
        row_data = np.append(row_data, [total_cases])
        
        # Get census data
        census_row = census_data[census_data['name']==region].values[0][1:]
        row_data = np.append(row_data, census_row)
        
        # Get cases for next time_frame
        next_cases = covid_df_week.loc[(covid_df_week.week == time_frames[i+1]) & (covid_df_week.region == region)]['cases'].values
        # This data is not available yet
        if len(next_cases) == 0:
            next_cases = 0
        else:
            next_cases = next_cases[0]
        row_data = np.append(row_data, [next_cases])

        # Add data to numpy arr
        if len(week_data_with_census.shape) == 1:
            week_data_with_census = np.array([row_data])
        else:
            week_data_with_census = np.append(week_data_with_census, [row_data], axis=0)
week_data_with_census
week_data_with_census.shape
np.save('../data/covid/covid_week_no_census', week_data_no_census)
np.save('../data/covid/covid_week_with_census', week_data_with_census)<jupyter_output><empty_output><jupyter_text>3-month aggregate<jupyter_code>covid_df_3month = covid_df[['Time Stamp', 'Region', 'Number of cases']]
covid_df_3month['Time Stamp'] = pd.to_datetime(covid_df['Time Stamp'])
covid_df_3month = covid_df_3month.set_index('Time Stamp')
covid_df_3month = covid_df_3month.groupby([pd.Grouper(freq='3M'), 'Region']).sum()
covid_df_3month = covid_df_3month.reset_index(level=['Time Stamp','Region']).rename(columns={'Time Stamp': 'month', 'Region':'region', 'Number of cases':'cases'})
covid_df_3month.month = covid_df_3month.month.dt.month
covid_df_3month

three_month_data_no_census = np.array([])
# List of months
time_frames = pd.unique(covid_df_3month.month)
# List of month case sums
covid_3month_gp = covid_df_3month.groupby('month').sum().reset_index()

for i in range(len(time_frames)-2): ## Need to ignore last week since data for last week is not complete
    time_frame = time_frames[i]
    for region in regions:
        # Build current row of data
        row_data = np.array([time_frame])
        
        # Get cases for current region
        region_cases = covid_df_3month.loc[(covid_df_3month.month == time_frame) & (covid_df_3month.region == region)]['cases'].values
        if len(region_cases) == 0:
            region_cases = 0
        else:
            region_cases = region_cases[0]
        row_data = np.append(row_data, [region_cases])
        
        # Get neighbors cases
        neighbors = (la_neighbors[la_neighbors['region']==region]['neighbors']).values[0]
        neighbors = neighbors.split(',')
        neighbors = [neighbor.strip() for neighbor in neighbors]
        neighbors_cases = 0
        no_data_count = 0
        for neighbor in neighbors:
            neighbor_cases_arr= covid_df_3month.loc[(covid_df_3month.month == time_frame) & (covid_df_3month.region == neighbor)]['cases'].values
            if len(neighbor_cases_arr) != 0:
                neighbors_cases += neighbor_cases_arr[0]
            else:
                no_data_count += 1
        if no_data_count == len(neighbors):
            neighbors_cases += region_cases * len(neighbors)
        else:
            neighbors_cases += (no_data_count/(len(neighbors)-no_data_count))*neighbors_cases ## Use avg to fill missing data and add to neighbors_cases
        row_data = np.append(row_data, [neighbors_cases])
        
        # Get cases of the entire LA region
        total_cases = covid_3month_gp[covid_3month_gp['month'] == time_frame]['cases'].values[0]
        row_data = np.append(row_data, [total_cases])
        
        # Get cases for next time_frame
        next_cases = covid_df_3month.loc[(covid_df_3month.month == time_frames[i+1]) & (covid_df_3month.region == region)]['cases'].values
        # This data is not available yet
        if len(next_cases) == 0:
            next_cases = 0
        else:
            next_cases = next_cases[0]
        row_data = np.append(row_data, [next_cases])

        # Add data to numpy arr
        if len(three_month_data_no_census.shape) == 1:
            three_month_data_no_census = np.array([row_data])
        else:
            three_month_data_no_census = np.append(three_month_data_no_census, [row_data], axis=0)
three_month_data_no_census
three_month_data_no_census.shape
three_month_data_with_census = np.array([])
# List of months
time_frames = pd.unique(covid_df_3month.month)
# List of month case sums
covid_3month_gp = covid_df_3month.groupby('month').sum().reset_index()

for i in range(len(time_frames)-2): ## Need to ignore last week since data for last week is not complete
    time_frame = time_frames[i]
    for region in regions:
        # Build current row of data
        row_data = np.array([time_frame])
        
        # Get cases for current region
        region_cases = covid_df_3month.loc[(covid_df_3month.month == time_frame) & (covid_df_3month.region == region)]['cases'].values
        if len(region_cases) == 0:
            region_cases = 0
        else:
            region_cases = region_cases[0]
        row_data = np.append(row_data, [region_cases])
        
        # Get neighbors cases
        neighbors = (la_neighbors[la_neighbors['region']==region]['neighbors']).values[0]
        neighbors = neighbors.split(',')
        neighbors = [neighbor.strip() for neighbor in neighbors]
        neighbors_cases = 0
        no_data_count = 0
        for neighbor in neighbors:
            neighbor_cases_arr= covid_df_3month.loc[(covid_df_3month.month == time_frame) & (covid_df_3month.region == neighbor)]['cases'].values
            if len(neighbor_cases_arr) != 0:
                neighbors_cases += neighbor_cases_arr[0]
            else:
                no_data_count += 1
        if no_data_count == len(neighbors):
            neighbors_cases += region_cases * len(neighbors)
        else:
            neighbors_cases += (no_data_count/(len(neighbors)-no_data_count))*neighbors_cases ## Use avg to fill missing data and add to neighbors_cases
        row_data = np.append(row_data, [neighbors_cases])
        
        # Get cases of the entire LA region
        total_cases = covid_3month_gp[covid_3month_gp['month'] == time_frame]['cases'].values[0]
        row_data = np.append(row_data, [total_cases])
        
        # Get census data
        census_row = census_data[census_data['name']==region].values[0][1:]
        row_data = np.append(row_data, census_row)
        
        # Get cases for next time_frame
        next_cases = covid_df_3month.loc[(covid_df_3month.month == time_frames[i+1]) & (covid_df_3month.region == region)]['cases'].values
        # This data is not available yet
        if len(next_cases) == 0:
            next_cases = 0
        else:
            next_cases = next_cases[0]
        row_data = np.append(row_data, [next_cases])

        # Add data to numpy arr
        if len(three_month_data_with_census.shape) == 1:
            three_month_data_with_census = np.array([row_data])
        else:
            three_month_data_with_census = np.append(three_month_data_with_census, [row_data], axis=0)
three_month_data_with_census
three_month_data_with_census.shape
np.save('../data/covid/covid_3month_no_census', three_month_data_no_census)
np.save('../data/covid/covid_3month_with_census', three_month_data_with_census)<jupyter_output><empty_output><jupyter_text>Daily aggregate<jupyter_code>covid_df_day = covid_df[['Time Stamp', 'Region', 'Number of cases']]
covid_df_day['Time Stamp'] = pd.to_datetime(covid_df['Time Stamp'])
covid_df_day = covid_df_day.set_index('Time Stamp')
covid_df_day = covid_df_day.groupby([pd.Grouper(freq='D'), 'Region']).sum()
covid_df_day = covid_df_day.reset_index(level=['Time Stamp','Region']).rename(columns={'Time Stamp': 'day', 'Region':'region', 'Number of cases':'cases'})
covid_df_day.day = covid_df_day.day.dt.dayofyear
covid_df_day

day_data_no_census = np.array([])
# List of months
time_frames = pd.unique(covid_df_day.day)
# List of month case sums
covid_day_gp = covid_df_day.groupby('day').sum().reset_index()

for i in range(len(time_frames)-2): ## Need to ignore last week since data for last week is not complete
    time_frame = time_frames[i]
    for region in regions:
        # Build current row of data
        row_data = np.array([time_frame])
        
        # Get cases for current region
        region_cases = covid_df_day.loc[(covid_df_day.day == time_frame) & (covid_df_day.region == region)]['cases'].values
        if len(region_cases) == 0:
            region_cases = 0
        else:
            region_cases = region_cases[0]
        row_data = np.append(row_data, [region_cases])
        
        # Get neighbors cases
        neighbors = (la_neighbors[la_neighbors['region']==region]['neighbors']).values[0]
        neighbors = neighbors.split(',')
        neighbors = [neighbor.strip() for neighbor in neighbors]
        neighbors_cases = 0
        no_data_count = 0
        for neighbor in neighbors:
            neighbor_cases_arr= covid_df_day.loc[(covid_df_day.day == time_frame) & (covid_df_day.region == neighbor)]['cases'].values
            if len(neighbor_cases_arr) != 0:
                neighbors_cases += neighbor_cases_arr[0]
            else:
                no_data_count += 1
        if no_data_count == len(neighbors):
            neighbors_cases += region_cases * len(neighbors)
        else:
            neighbors_cases += (no_data_count/(len(neighbors)-no_data_count))*neighbors_cases ## Use avg to fill missing data and add to neighbors_cases
        row_data = np.append(row_data, [neighbors_cases])
        
        # Get cases of the entire LA region
        total_cases = covid_day_gp[covid_day_gp['day'] == time_frame]['cases'].values[0]
        row_data = np.append(row_data, [total_cases])
        
        # Get cases for next time_frame
        next_cases = covid_df_day.loc[(covid_df_day.day == time_frames[i+1]) & (covid_df_day.region == region)]['cases'].values
        # This data is not available yet
        if len(next_cases) == 0:
            next_cases = 0
        else:
            next_cases = next_cases[0]
        row_data = np.append(row_data, [next_cases])
        
        # Add data to numpy arr
        if len(day_data_no_census.shape) == 1:
            day_data_no_census = np.array([row_data])
        else:
            day_data_no_census = np.append(day_data_no_census, [row_data], axis=0)
day_data_no_census
day_data_no_census.shape
day_data_with_census = np.array([])
# List of months
time_frames = pd.unique(covid_df_day.day)
# List of month case sums
covid_day_gp = covid_df_day.groupby('day').sum().reset_index()

for i in range(len(time_frames)-2): ## Need to ignore last week since data for last week is not complete
    time_frame = time_frames[i]
    for region in regions:
        # Build current row of data
        row_data = np.array([time_frame])
        
        # Get cases for current region
        region_cases = covid_df_day.loc[(covid_df_day.day == time_frame) & (covid_df_day.region == region)]['cases'].values
        if len(region_cases) == 0:
            region_cases = 0
        else:
            region_cases = region_cases[0]
        row_data = np.append(row_data, [region_cases])
        
        # Get neighbors cases
        neighbors = (la_neighbors[la_neighbors['region']==region]['neighbors']).values[0]
        neighbors = neighbors.split(',')
        neighbors = [neighbor.strip() for neighbor in neighbors]
        neighbors_cases = 0
        no_data_count = 0
        for neighbor in neighbors:
            neighbor_cases_arr= covid_df_day.loc[(covid_df_day.day == time_frame) & (covid_df_day.region == neighbor)]['cases'].values
            if len(neighbor_cases_arr) != 0:
                neighbors_cases += neighbor_cases_arr[0]
            else:
                no_data_count += 1
        if no_data_count == len(neighbors):
            neighbors_cases += region_cases * len(neighbors)
        else:
            neighbors_cases += (no_data_count/(len(neighbors)-no_data_count))*neighbors_cases ## Use avg to fill missing data and add to neighbors_cases
        row_data = np.append(row_data, [neighbors_cases])
        
        # Get cases of the entire LA region
        total_cases = covid_day_gp[covid_day_gp['day'] == time_frame]['cases'].values[0]
        row_data = np.append(row_data, [total_cases])
        
        # Get census data
        census_row = census_data[census_data['name']==region].values[0][1:]
        row_data = np.append(row_data, census_row)
        
        # Get cases for next time_frame
        next_cases = covid_df_day.loc[(covid_df_day.day == time_frames[i+1]) & (covid_df_day.region == region)]['cases'].values
        # This data is not available yet
        if len(next_cases) == 0:
            next_cases = 0
        else:
            next_cases = next_cases[0]
        row_data = np.append(row_data, [next_cases])
        
        # Add data to numpy arr
        if len(day_data_with_census.shape) == 1:
            day_data_with_census = np.array([row_data])
        else:
            day_data_with_census = np.append(day_data_with_census, [row_data], axis=0)
day_data_with_census
day_data_with_census.shape
np.save('../data/covid/covid_day_no_census', day_data_no_census)
np.save('../data/covid/covid_day_with_census', day_data_with_census)
# covid_df_week.to_csv('../data/covid/covid_week.csv')
covid_df_month.to_csv('../data/covid/covid_month.csv')
# covid_df_3month.to_csv('../data/covid/covid_3months.csv')<jupyter_output><empty_output><jupyter_text>Scratch work (IGNORE)<jupyter_code>la_neighbors[la_neighbors['region']=='Baldwin Hills/Crenshaw']
covid_df[covid_df['Region']=='Baldwin Hills']
region_map = {
    'Baldwin Hills/Crenshaw':'Baldwin Hills',
    'Unincorporated Santa Monica Mountains':'Santa Monica Mountains',
    'Mid-Wilshire':'Wilshire Center',
    'Exposition Park':'Exposition',
    
}
from shapely.geometry import Point, Polygon
def find_la_region(covid_region):
    polys = la_neighbors['geometry']
    lat = covid_df[covid_df['Region']==covid_region].iloc[0,:][2]
    lng = covid_df[covid_df['Region']==covid_region].iloc[0,:][3]
#     print(lat, lng)
    return (la_neighbors[polys.contains(Point(lng, lat))], lat, lng)
find_la_region('Santa Monica Mountains')
find_la_region('Pico')
find_la_region('Exposition').shape
graph_region(la_neighbors, 'Downtown')
cleaned_region_map = {
    'Downtown': ['City', 'Melrose', 'Temple', 'Pico']
}
wrong_map = ['Santa Monica Mountains', 'Melrose', '']
from collections import defaultdict

la_to_covid_map = defaultdict(list)

for covid_c in missing_arr:
    data = find_la_region(covid_c)
    if data[0].shape[0] == 0:
        continue
    la_to_covid_map[data[0].iloc[0,0]].append({covid_c:(data[1], data[2])})
la_to_covid_map<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12364972924440953, 'fraction_numerical': 0.008697847984063298, 'mean_word_length': 3.70894437838552, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 78, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '619737', 'license': 'MIT', 'n_tokens_mistral': 12327, 'n_tokens_neox': 12022, 'n_words': 2305, 'path': 'code/covid_cases_data_processing.ipynb', 'repo_name': 'kaushiktandon/COVID-19-Vaccine-Allocation', 'size': '35641'}