<jupyter_start><jupyter_text>Load att df<jupyter_code>playa_att_csv_path = '../data/playa_nogeometry_whucs.csv'
playa_att_df = pd.read_csv(playa_att_csv_path).set_index('id')
# Convert HUC and author to int codes: (using huc8 instead of huc12)
playa_att_df['huc8'] = playa_att_df['huc12'].astype(str).str[0:8]
playa_att_df['huc8'] = playa_att_df['huc8'].astype('category').cat.codes

# Fix a couple bad values in author
playa_att_df.loc[71851, 'author'] = 'University of Kansas'
playa_att_df.loc[71852, 'author'] = 'PLJV'

# Calc codes for author
playa_att_df['author'] = playa_att_df['author'].astype('category').cat.codes
# Find default values for filling in NAs later, using median because both datasets are a bit skewed
print('mean sthick: ',np.mean(playa_att_df.loc[playa_att_df['sthick2013']>-9999, 'sthick2013']))
print('median sthick: ',np.median(playa_att_df.loc[playa_att_df['sthick2013']>-9999, 'sthick2013']))

print('mean devyr: ',np.mean(playa_att_df.loc[playa_att_df['winddevyr']<9999, 'winddevyr']))
print('median devyr: ',np.median(playa_att_df.loc[playa_att_df['winddevyr']<9999, 'winddevyr']))<jupyter_output><empty_output><jupyter_text>Preprocess data to split up by state and county<jupyter_code>inun_csv_path = '../data/jrc-water_1984-2019.csv'
inun_df = pd.read_csv(inun_csv_path).set_index('id').drop('.geo', axis=1).rename(columns={'mean':'inundation'})
def save_state_county_csvs_inun(state, inun_df, playa_att_df):
    # Join state and county
    state_df = inun_df.join(playa_att_df.loc[playa_att_df['state']==state,
                                             ['state', 'countyfips', 'acres','huc8', 'winddevyr',
                                              'author','meanwetfrq','cluster','healthy','farmed','hydromod',
                                              'fttoroad', 'sthick2013']
                                            ]
                            ,how='inner')
    
    # Split up weird index into year + months
    split_index = state_df['system:index'].str.split('_', expand=True)
    state_df = state_df.assign(year=split_index[0].astype('int16'),
                               month=split_index[1].astype('int16'),
                               area=state_df['acres']*state_df['inundation'])
    state_df.drop(['system:index', 'state'], axis=1, inplace=True)
    
    # Save state csv
    state_out_path = '../data/state_county_csvs/inun_frac_{}.csv'.format(state)
    state_df.drop('countyfips', axis=1).to_csv(state_out_path)
    
    # Save county csvs
    for county in state_df['countyfips'].unique():
        county_df = state_df.loc[state_df['countyfips']==county]
        county_out_path = '../data/state_county_csvs/counties/inun_frac_{}_{}.csv'.format(state, county)
        county_df.drop('countyfips', axis=1).to_csv(county_out_path)
# Save state and county csvs
for state in playa_att_df['state'].unique():
    save_state_county_csvs_inun(state, inun_df, playa_att_df)<jupyter_output><empty_output><jupyter_text>Preprocess and split weather data  Prism data (current version)<jupyter_code>weather_csv_path = '../data/prism.csv'
weather_df = pd.read_csv(weather_csv_path).set_index('id').drop('.geo', axis=1)
def save_state_county_csvs_weather(state, weather_df, playa_att_df):
    # Join state and county
    state_df = weather_df.join(playa_att_df.loc[playa_att_df['state']==state,
                                             ['state', 'countyfips']],how='inner')
    
    # Split up weird index into year + months, little different than inundation
    state_df = state_df.assign(year=state_df['system:index'].str.slice(0,4).astype('int16'),
                               month=state_df['system:index'].str.slice(4,6).astype('int16'))
    state_df.drop(['system:index', 'state'], axis=1, inplace=True)
    
    # Rename to simpler column names
    state_df = state_df.rename(columns={'tmean':'temp', 'ppt':'precip', 'vpdmin':'vpd'})
    
    # Save state csv
    state_out_path = '../data/state_county_csvs/weather_{}.csv'.format(state)
    state_df.drop('countyfips', axis=1).to_csv(state_out_path)
    
    # Save county csvs
    for county in state_df['countyfips'].unique():
        county_df = state_df.loc[state_df['countyfips']==county]
        county_out_path = '../data/state_county_csvs/counties/weather_{}_{}.csv'.format(state, county)
        county_df.drop('countyfips', axis=1).to_csv(county_out_path)
# Save state and county csvs
for state in playa_att_df['state'].unique():
    save_state_county_csvs_weather(state, weather_df, playa_att_df)<jupyter_output><empty_output><jupyter_text>Era Data (outdated, using prism now)<jupyter_code># weather_csv_path = '../data/era5_1984-2019.csv'
# weather_df = pd.read_csv(weather_csv_path).set_index('id').drop('.geo', axis=1)
# # Save state and county csvs
# for state in playa_att_df['state'].unique():
    
#     # Join state and county
#     state_df = weather_df.join(playa_att_df.loc[playa_att_df['state']==state,
#                                              ['state', 'countyfips']],how='inner')
    
#     # Split up weird index into year + months, little different than inundation
#     state_df = state_df.assign(year=state_df['system:index'].str.slice(0,4).astype('int16'),
#                                month=state_df['system:index'].str.slice(4,6).astype('int16'))
#     state_df.drop(['system:index', 'state'], axis=1, inplace=True)
    
#     # Rename to simpler column names
#     state_df = state_df.rename(columns={'mean_2m_air_temperature':'temp', 'total_precipitation':'precip'})
    
#     # Save state csv
#     state_out_path = '../data/state_county_csvs/weather_{}.csv'.format(state)
#     state_df.drop('countyfips', axis=1).to_csv(state_out_path)
    
#     # Save county csvs
#     for county in state_df['countyfips'].unique():
#         county_df = state_df.loc[state_df['countyfips']==county]
#         county_out_path = '../data/state_county_csvs/counties/weather_{}_{}.csv'.format(state, county)
#         county_df.drop('countyfips', axis=1).to_csv(county_out_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15028043549983502, 'fraction_numerical': 0.017980864401187728, 'mean_word_length': 3.8426517571884986, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '293872', 'license': 'MIT', 'n_tokens_mistral': 2213, 'n_tokens_neox': 2128, 'n_words': 315, 'path': 'playa-inundation/preprocessing/preprocess_split_data.ipynb', 'repo_name': 'kysolvik/playa-inundation', 'size': '6062'}