<jupyter_start><jupyter_text>Read and process 2010-2018 merged dataset<jupyter_code>df = pd.read_csv('district-wise-rainfall-data-for-india-2014.csv', sep=';')
df
df['YEAR'] = df['DATE'].str[:4]
df['MONTH'] = df['DATE'].str[5:7]
df.drop(columns=['DATE'])
df.head()
df['YEAR'] = df['YEAR'].astype(int)
df['MONTH'] = df['MONTH'].astype(int)
rainfall_df = df[(df['INDIAN_STATES_NAME'] == 'Karnataka') & (df['VARIABLE_NAME'] == 'Rainfall (mm.)')].sort_values('DATE')
departure_df = df[(df['INDIAN_STATES_NAME'] == 'Karnataka') & (df['VARIABLE_NAME'] == 'Departures of rainfall (%)')].sort_values('DATE')
rainfall_df.drop(columns=['DATE', 'FREQUENCY', 'INDIAN_STATES_NAME', 'VARIABLE_NAME', 'VARIABLE_NOTES'], inplace=True)
rainfall_df.rename(columns={'VALUE': 'RAINFALL', 'DISTRICTS_NAME': 'DISTRICTS'}, inplace=True)
rainfall_df.head()
departure_df.drop(columns=['DATE', 'FREQUENCY', 'INDIAN_STATES_NAME', 'VARIABLE_NAME', 'VARIABLE_NOTES'], inplace=True)
departure_df.rename(columns={'VALUE': 'DEPARTURE', 'DISTRICTS_NAME': 'DISTRICTS'}, inplace=True)
departure_df.head()
rainfall_departure_df = rainfall_df.merge(departure_df, on=['DISTRICTS', 'YEAR', 'MONTH'])
rainfall_departure_df
rainfall_departure_df['DISTRICTS'] = rainfall_departure_df['DISTRICTS'].str.upper()
rainfall_departure_df['DISTRICTS'].value_counts()
rainfall_departure_df['DISTRICTS'].replace(to_replace='MYSORE', value='MYSURU', inplace=True)
rainfall_departure_df['DISTRICTS'].replace(to_replace='CHIKMAGALUR', value='CHIKKAMAGALURU', inplace=True)<jupyter_output><empty_output><jupyter_text>Read and process 2019-2020 departure data<jupyter_code>departure_19_20 = pd.read_csv('Departure_19-20.csv')
departure_19_20
departure_19_20 = departure_19_20.melt(id_vars=['DISTRICTS', 'YEAR'], var_name='MONTH', value_name='DEPARTURE')
departure_19_20
month_df = pd.DataFrame.from_dict({1: 'JAN', 2: 'FEB', 3: 'MAR', 4: 'APR', 5: 'MAY', 6: 'JUN',
                                   7: 'JUL', 8: 'AUG', 9: 'SEP', 10: 'OCT', 11: 'NOV', 12: 'DEC'}, orient='index').reset_index()
month_df.columns = ['index', 'MONTH']
month_df.head()
departure_19_20 = departure_19_20.merge(month_df, on='MONTH')
departure_19_20.drop(columns=['MONTH'], inplace=True)
departure_19_20.rename(columns={'index': 'MONTH'}, inplace=True)
departure_19_20.head()<jupyter_output><empty_output><jupyter_text>Read and process 2019-2020 rainfall data<jupyter_code>rainfall_2019 = pd.read_csv('Rainfall-2019.csv')
rainfall_2020 = pd.read_csv('Rainfall-2020.csv')
rainfall_19_20 = pd.concat([rainfall_2019, rainfall_2020])
rainfall_19_20
rainfall_19_20 = rainfall_19_20.melt(id_vars=['DISTRICTS', 'YEAR'], var_name='MONTH', value_name='RAINFALL')
rainfall_19_20
rainfall_19_20 = rainfall_19_20.merge(month_df, on='MONTH')
rainfall_19_20.drop(columns=['MONTH'], inplace=True)
rainfall_19_20.rename(columns={'index': 'MONTH'}, inplace=True)
rainfall_19_20.head()<jupyter_output><empty_output><jupyter_text>Merge 2019-2020 rainfall and departure data<jupyter_code>rainfall_departure_19_20 = rainfall_19_20.merge(departure_19_20, on=['DISTRICTS', 'YEAR', 'MONTH'])
rainfall_departure_19_20<jupyter_output><empty_output><jupyter_text>Merge all rainfall and departure data<jupyter_code>rainfall_departure_df = pd.concat([rainfall_departure_df, rainfall_departure_19_20])
rainfall_departure_df
rainfall_departure_df['DISTRICTS'].value_counts()<jupyter_output><empty_output><jupyter_text>Add season<jupyter_code>month_season = {'SEASON': {'JAN': 'WINTER', 'FEB': 'WINTER', 'MAR': 'SUMMER', 'APR': 'SUMMER',
                           'MAY': 'SUMMER', 'JUN': 'SUMMER', 'JUL': 'MONSOON', 'AUG': 'MONSOON',
                           'SEP': 'MONSOON', 'OCT': 'WINTER', 'NOV': 'WINTER', 'DEC': 'WINTER'}}
season_df = pd.DataFrame(month_season).reset_index()
season_df.rename(columns={'index': 'MONTH_NAME'}, inplace=True)
season_df
season_df = month_df.merge(season_df, left_on=['MONTH'], right_on=['MONTH_NAME'])
season_df
season_df.drop(columns=['MONTH', 'MONTH_NAME'], inplace=True)
season_df.rename(columns={'index': 'MONTH'}, inplace=True)
season_df
rainfall_departure_df = rainfall_departure_df.merge(season_df, on='MONTH')<jupyter_output><empty_output><jupyter_text>Save dataset<jupyter_code>rainfall_departure_df.to_csv('../rainfall_departure_dataset.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17657949548715576, 'fraction_numerical': 0.042119879657486696, 'mean_word_length': 9.490291262135923, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '606007', 'license': 'MIT', 'n_tokens_mistral': 1902, 'n_tokens_neox': 1713, 'n_words': 219, 'path': 'python_scripts/dataset/Rainfall 2010-2020/data/create_rainfall_departure_dataset.ipynb', 'repo_name': 'santanu94/WRI_Wave2WebHack', 'size': '4321'}