<jupyter_start><jupyter_text>Illinois Weather<jupyter_code>import pandas as pd
import geopandas as gpd
pd.read_csv("../data/processed/processed_rend_lake_data.csv")
# Illinois county
evi_stacked = pd.read_csv("../data/raw/EVI_stacked.csv")

geo_dataframe = gpd.read_file("../data/raw/illinois-counties.geojson")
geo_dataframe["geometry"].iloc[0].centroid.y
type(geo_dataframe["geometry"])
# Get centroid
def getXY(pt):
    """
    Function returns x and y coordinates of geo pandas centroid
    @param pt:
    @return:
    """
    return pt.x, pt.y
centroidseries = geo_dataframe['geometry'].centroid
x,y = [list(t) for t in zip(*map(getXY, centroidseries))]
names = list(geo_dataframe["name"])
# County centroid dataframe
county_centroid_dataframe = pd.DataFrame()
county_centroid_dataframe["county"] = names
county_centroid_dataframe["long"] = x
county_centroid_dataframe["lat"] = y
county_centroid_dataframe
# Zones
zone1_min_lat = evi_stacked[evi_stacked["county"] == "LIVINGSTON"]["lat"].max()
zone3_max_lat = evi_stacked[evi_stacked["county"] == "CRAWFORD"]["lat"].max()
zone3_max_lat
zone_1_counties = list(county_centroid_dataframe[county_centroid_dataframe["lat"] >
                                         zone1_min_lat]["county"])
zone_3_counties = list(county_centroid_dataframe[county_centroid_dataframe["lat"] <
                                         zone3_max_lat]["county"])
current_county_zone = zone_1_counties + zone_3_counties

zone_2_counties = list(county_centroid_dataframe[~county_centroid_dataframe["county"].isin
(current_county_zone)
]["county"])
if len(x) > len(set(x)):
   pass #
x = current_county_zone + zone_3_counties + zone_1_counties
x<jupyter_output><empty_output><jupyter_text>Grouping Data by Zone<jupyter_code>era5 = pd.read_csv("../data/raw/era5.csv")
era5["time"] = pd.to_datetime(era5["time"])
era5["year"] = era5["time"].dt.year
era5["month"] = era5["time"].dt.month
era5["day"] = era5["time"].dt.day
era5 = era5[era5["year"] >= 2001]
era5 = era5[(era5["month"] >= 4) & (era5["month"] <= 10)]
era5
county_name_list = list()
min_t2m = list()
max_t2m = list()
mean_t2m = list()
era5.groupby("county").mean()

a = era5[era5["county"] == "BOONE"]
t2m_df_year = pd.DataFrame()
for year in sorted(list(a["year"].unique())):
    b = a[a["year"] == year]

    month_list = list(b["month"].unique())
    monthly_mean_t2m_list = list(b.groupby("month").mean()["t2m"])
    monthly_max_t2m_list = list(b.groupby("month").max()["t2m"])
    monthly_min_t2m_list = list(b.groupby("month").min()["t2m"])

    t2m_df = pd.DataFrame()
    t2m_df["month"] = month_list
    t2m_df["mean"] = monthly_mean_t2m_list
    t2m_df["max"] = monthly_max_t2m_list
    t2m_df["min"] = monthly_min_t2m_list
    t2m_df["year"] = year
    t2m_df_year = t2m_df_year.append(t2m_df)
t2m_df_year
filtered_year_t2m = t2m_df_year[t2m_df_year["year"] == 2001]

columns = t2m_df_year.columns[1:-1]
columns
month_number_columns = ["month_" + str(x) for x in list(filtered_year_t2m["month"])]
month_number_mean_columns = [x + "_mean" for x in month_number_columns]
month_number_max_columns = [x + "_max" for x in month_number_columns]
month_number_min_columns = [x + "_min" for x in month_number_columns]
columns_to_filter = ["mean", "max", "min"]

concatenated_dataframe = pd.DataFrame()
for column in columns_to_filter:
    temporary_dataframe = pd.DataFrame(filtered_year_t2m[column]).T
    if column == "mean":
        temporary_dataframe.columns = [month_number_mean_columns]
    elif column == "max":
        temporary_dataframe.columns = [month_number_max_columns]
    else:
        temporary_dataframe.columns = [month_number_min_columns]
    temporary_dataframe.reset_index(inplace=True, drop=True)
    concatenated_dataframe = pd.concat([concatenated_dataframe, temporary_dataframe], axis=1)
concatenated_dataframe.shape
concatenated_dataframe
APRIL_TO_OCTOBER = [4, 5, 6, 7, 8, 9, 10]
MONTH_NUMBER_COLUMNS = ["month_" + str(x) for x in APRIL_TO_OCTOBER]
MONTH_NUMBER_MEAN_COLUMNS = [x + "_mean" for x in MONTH_NUMBER_COLUMNS]
MONTH_NUMBER_MAX_COLUMNS = [x + "_max" for x in MONTH_NUMBER_COLUMNS]
MONTH_NUMBER_MIN_COLUMNS = [x + "_min" for x in MONTH_NUMBER_COLUMNS]
COLUMNS_TO_FILTER = ["mean", "max", "min"]

def make_1_by_21_dataframe(dataframe):
    """

    @param dataframe:
    @return:
    """
    concatenated_dataframe = pd.DataFrame()
    for column in COLUMNS_TO_FILTER:
        temporary_dataframe = pd.DataFrame(dataframe[column]).T
        if column == "mean":
            temporary_dataframe.columns = [MONTH_NUMBER_MEAN_COLUMNS]
        elif column == "max":
            temporary_dataframe.columns = [MONTH_NUMBER_MAX_COLUMNS]
        else:
            temporary_dataframe.columns = [MONTH_NUMBER_MIN_COLUMNS]
        temporary_dataframe.reset_index(inplace=True, drop=True)
        concatenated_dataframe = pd.concat([concatenated_dataframe, temporary_dataframe], axis=1)
    return concatenated_dataframe

import numpy as np
APRIL_TO_OCTOBER = [4, 5, 6, 7, 8, 9, 10]
MONTH_NUMBER_COLUMNS = ["month_" + str(x) for x in APRIL_TO_OCTOBER]
MONTH_NUMBER_MEAN_COLUMNS = [x + "_mean" for x in MONTH_NUMBER_COLUMNS]
MONTH_NUMBER_MAX_COLUMNS = [x + "_max" for x in MONTH_NUMBER_COLUMNS]
MONTH_NUMBER_MIN_COLUMNS = [x + "_min" for x in MONTH_NUMBER_COLUMNS]
COLUMNS_TO_FILTER = ["mean", "max", "min"]
LIST_2001_TO_2019 = [2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012,
                     2013, 2014, 2015, 2016, 2017, 2018, 2019]

def make_1_by_21_dataframe(dataframe):
    """

    @param dataframe:
    @return:
    """
    concatenated_dataframe = pd.DataFrame()
    for column in COLUMNS_TO_FILTER:
        temporary_dataframe = pd.DataFrame(dataframe[column]).T
        if column == "mean":
            temporary_dataframe.columns = [MONTH_NUMBER_MEAN_COLUMNS]
        elif column == "max":
            temporary_dataframe.columns = [MONTH_NUMBER_MAX_COLUMNS]
        else:
            temporary_dataframe.columns = [MONTH_NUMBER_MIN_COLUMNS]
        temporary_dataframe.reset_index(inplace=True, drop=True)
        concatenated_dataframe = pd.concat([concatenated_dataframe, temporary_dataframe], axis=1)
    return concatenated_dataframe

def create_monthly_metric_list(dataframe, column):
    """

    @param dataframe:
    @return:
    """
    monthly_mean_metric_list = list(dataframe.groupby("month").mean()[column])
    monthly_max_metric_list = list(dataframe.groupby("month").max()[column])
    monthly_min_metric_list = list(dataframe.groupby("month").min()[column])
    return monthly_mean_metric_list, monthly_max_metric_list, monthly_min_metric_list

def create_monthly_metric_dataframe(month_list, monthly_mean_metric_list,
                                    monthly_max_metric_list, monthly_min_metric_list, year):
    """

    @param month_list:
    @param monthly_mean_metric_list:
    @param monthly_max_metric_list:
    @param monthly_min_metric_list:
    @return:
    """
    monthly_metric_dataframe = pd.DataFrame()
    monthly_metric_dataframe["month"] = month_list
    try:
        monthly_metric_dataframe["mean"] = monthly_mean_metric_list
        monthly_metric_dataframe["max"] = monthly_max_metric_list
        monthly_metric_dataframe["min"] = monthly_min_metric_list
    except:
        monthly_metric_dataframe["mean"] = np.nan
        monthly_metric_dataframe["max"] = np.nan
        monthly_metric_dataframe["min"] = np.nan
    monthly_metric_dataframe["year"] = year
    return monthly_metric_dataframe

def get_county_metric_history(dataframe, column):
    """

    @param dataframe: monthly_metric_dataframe
    @return:
    """
    county_metric_history_dataframe = pd.DataFrame()
    for year in LIST_2001_TO_2019:
        try:
            temporary_dataframe = dataframe[dataframe["year"] == year]
            monthly_mean_metric_list, monthly_max_metric_list, monthly_min_metric_list = \
                create_monthly_metric_list(temporary_dataframe, column)
            monthly_metric_dataframe = create_monthly_metric_dataframe(APRIL_TO_OCTOBER,
                                                                       monthly_mean_metric_list,
                                                                       monthly_max_metric_list,
                                                                       monthly_min_metric_list,
                                                                       year)
        except:
            pass
        county_metric_history_dataframe = county_metric_history_dataframe.append\
                (monthly_metric_dataframe)
    return county_metric_history_dataframe

def create_yearly_1_by_21_dataframe(dataframe):
    """

    @param dataframe: county_metric_history_dataframe
    @return:
    """
    yearly_1_by_21_dataframe = pd.DataFrame()
    for year in LIST_2001_TO_2019:
        temporary_dataframe = dataframe[dataframe["year"] == year]
        concatenated_dataframe = make_1_by_21_dataframe(temporary_dataframe)
        concatenated_dataframe["year"] = year
        yearly_1_by_21_dataframe = yearly_1_by_21_dataframe.append(concatenated_dataframe)
    return yearly_1_by_21_dataframe

def create_county_yearly_1_by_21_dataframe(dataframe, column, metric):
    """

    @param dataframe:
    @param column: column for for-loop
    @return:
    """
    county_yearly_1_by_21_dataframe = pd.DataFrame()
    for n in list(dataframe[column].unique()):
        temporary_dataframe = dataframe[dataframe[column] == n]
        county_metric_history_dataframe = get_county_metric_history(temporary_dataframe, metric)
        yearly_1_by_21_dataframe = create_yearly_1_by_21_dataframe(county_metric_history_dataframe)
        yearly_1_by_21_dataframe[column] = n
        county_yearly_1_by_21_dataframe = county_yearly_1_by_21_dataframe.append\
            (yearly_1_by_21_dataframe)
    return county_yearly_1_by_21_dataframe
country_yearly_evi_dataframe = create_county_yearly_1_by_21_dataframe(era5, "county", "t2m")
country_yearly_evi_dataframe
country_yearly_evi_dataframe.columns=country_yearly_evi_dataframe.columns.get_level_values(0).map\
    (lambda x : "t2m_" + x if x !='year' and x!='county' else x)
country_yearly_evi_dataframe
country_yearly_evi_dataframe.to_csv("../data/interim/country_yearly_t2m_1_by_n.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10821040372670808, 'fraction_numerical': 0.024747670807453416, 'mean_word_length': 4.029282576866764, 'pattern_counts': {'":': 6, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '604774', 'license': 'MIT', 'n_tokens_mistral': 3721, 'n_tokens_neox': 3590, 'n_words': 558, 'path': 'notebooks/JC_03_illinois_weather.ipynb', 'repo_name': 'AlexBernaus/aihack2021', 'size': '10304'}