<jupyter_start><jupyter_text>Read GHCN precipitation data<jupyter_code>import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>Read station inventory<jupyter_code>ghcn_stations = pd.read_table("../input_data/GHCN-Daily/ghcnd-inventory.txt", sep='\s+',
                              names=['ID', 'LATITUDE', 'LONGITUDE', 'ELEMENT', 
                                'FIRST_YEAR', 'LAST_YEAR']) 
# select for precip stations
ghcn_stations = ghcn_stations.loc[ghcn_stations.ELEMENT == "PRCP"]

# select for midwest region
ghcn_stations = ghcn_stations.loc[(ghcn_stations.LATITUDE >= 37) & 
                        (ghcn_stations.LATITUDE <= 48) & 
                        (ghcn_stations.LONGITUDE >= -104) & 
                        (ghcn_stations.LONGITUDE <= -86)]

# select for full time period
ghcn_stations = ghcn_stations.loc[(ghcn_stations.FIRST_YEAR <= 1981) & 
                       (ghcn_stations.LAST_YEAR >= 2019)]<jupyter_output><empty_output><jupyter_text>Read/combine precipitation data for midwest stations<jupyter_code>ghcn_dat = []

for year in range(1981, 2020):
    ## read annual data and subset
    t = pd.read_csv("../input_data/GHCN-Daily/years_csv/"+str(year)+".csv", 
                        names = ['ID', 'DATE', 'ELEMENT', 'DATA_VALUE', 'M_FLAG', 
                        'Q_FLAG', 'S_FLAG', 'OBS_TIME'], 
                        dtype = {'ID': 'string', 'DATE': 'int64', 'ELEMENT': 'string', 
                        'DATA_VALUE': 'float64', 'M_FLAG': 'string', 'Q_FLAG': 'string', 
                        'S_FLAG': 'string'})
    t = t.loc[t.ID.isin(ghcn_stations.ID) & 
                 (t.ELEMENT == "PRCP")]
        
    ghcn_dat.append(t)
    
ghcn_dat = pd.concat(ghcn_dat)
# remove records that were flagged for quality issue 
ghcn_dat = ghcn_dat[ghcn_dat.Q_FLAG.isna()] 
## Find stations with complete data 
missing_thr = 10 # max percentage of missing dates

record_count = ghcn_dat.groupby(ghcn_dat.ID).count().reset_index()
complete_stations = record_count.loc[(14244-record_count.DATE)/14244 <= missing_thr/100].ID
    
ghcn_dat = ghcn_dat[ghcn_dat.ID.isin(complete_stations)]
    
print(str(len(complete_stations)) + ' stations with < ' + str(missing_thr) + '% of days missing')
## save combined data
ghcn_dat = ghcn_dat.drop(columns = ["M_FLAG", "Q_FLAG", "S_FLAG", "OBS_TIME", "ELEMENT"]).join(
        ghcn_stations.drop(columns=["ELEMENT", "FIRST_YEAR", "LAST_YEAR"]).set_index("ID"), 
              on = "ID", how = "left")
    
ghcn_dat.to_csv("../processed_data/ghcn_midwest_data.csv", index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1436251920122888, 'fraction_numerical': 0.016897081413210446, 'mean_word_length': 3.0138674884437595, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '443610', 'license': 'MIT', 'n_tokens_mistral': 982, 'n_tokens_neox': 905, 'n_words': 155, 'path': 'notebooks/0c_read_ghcn.ipynb', 'repo_name': 'fdavenport/GRL2021', 'size': '2604'}