<jupyter_start><jupyter_text>Weather Forecast - Model Preparation  Import Cleaned Data Import Libraries<jupyter_code>import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>Importing Atmospheric Pressure Data<jupyter_code>filename = 'CleanedData/AtmosphericPressure_pascal.csv'
atm_pressure_df = pd.read_csv(filename)
atm_pressure_df.head()<jupyter_output><empty_output><jupyter_text>Importing Temperature (Minimum and Maximum) Data<jupyter_code>filename = 'CleanedData/MinimumTemperature_degCelsius.csv'
min_temp_df = pd.read_csv(filename)
min_temp_df.head()
filename = 'CleanedData/MaximumTemperature_degCelsius.csv'
max_temp_df = pd.read_csv(filename)
max_temp_df.head()<jupyter_output><empty_output><jupyter_text>Importing Relative Humidity Data<jupyter_code>filename = 'CleanedData/RelativeHumidity_Percent.csv'
rel_humidity_df = pd.read_csv(filename)
rel_humidity_df.head()<jupyter_output><empty_output><jupyter_text>Importing Wind Speed Data<jupyter_code>filename = 'CleanedData/WindSpeed_Knot.csv'
wind_speed_df = pd.read_csv(filename)
wind_speed_df.head()<jupyter_output><empty_output><jupyter_text>Flattening Time Series Data<jupyter_code>atm_pressure_df.head(3)
list_date = []
list_data = []

for year in ['2018', '2019', '2020']:
    month_data, day_data = list(atm_pressure_df['Month']), list(atm_pressure_df['DAY'])
    atm_data = list(atm_pressure_df[year])
    
    month_convert = {'Jan':'01', 'Feb':'02', 'Mar':'03', 'Apr':'04', 'May':'05', 'Jun':'06',
                    'Jul':'07', 'Aug':'08', 'Sep':'09', 'Oct':'10', 'Nov':'11', 'Dec':'12'}
    
    for month, day, data in zip(month_data, day_data, atm_data):
        day = '0' + str(day) if len(str(day)) == 1 else str(day)
        month = month_convert[month]
        
        date = day +'-' + month + '-' + year
        
        if (year in ['2018', '2019']) & (month == '02') & (day == '29'):
            continue
        
        list_date.append(date)
        list_data.append(data)
        
date_series = pd.to_datetime(list_date, dayfirst=True)
data_series = pd.Series(list_data, name='Atmospheric Pressure')

result_df = pd.DataFrame( { 'Date': date_series, 'Atmospheric Pressure': data_series } )
result_df.set_index(['Date'], inplace=True)

result_df<jupyter_output><empty_output><jupyter_text>Creating Python Function to Handle Flattening<jupyter_code>def flatten_data_table(input_df, table_name):
    list_date = []
    list_data = []

    for year in ['2018', '2019', '2020']:
        month_data, day_data = list(input_df['Month']), list(input_df['DAY'])
        table_data = list(input_df[year])

        month_convert = {'Jan':'01', 'Feb':'02', 'Mar':'03', 'Apr':'04', 'May':'05', 'Jun':'06',
                        'Jul':'07', 'Aug':'08', 'Sep':'09', 'Oct':'10', 'Nov':'11', 'Dec':'12'}

        for month, day, data in zip(month_data, day_data, table_data):
            day = '0' + str(day) if len(str(day)) == 1 else str(day)
            month = month_convert[month]

            date = day +'-' + month + '-' + year

            if (year in ['2018', '2019']) & (month == '02') & (day == '29'):
                continue

            list_date.append(date)
            list_data.append(data)

    date_series = pd.to_datetime(list_date, dayfirst=True)
    data_series = pd.Series(list_data, name = table_name)

    result_df = pd.DataFrame( { 'Date': date_series, table_name: data_series } )
    result_df.set_index(['Date'], inplace=True)

    return result_df
atm_pressure_df = flatten_data_table(atm_pressure_df, 'Atmospheric Pressure')
atm_pressure_df
min_temp_df = flatten_data_table(min_temp_df, 'Minimum Temperature')
min_temp_df
max_temp_df = flatten_data_table(max_temp_df, 'Maximum Temperature')
max_temp_df
rel_humidity_df = flatten_data_table(rel_humidity_df, 'Relative Humidity')
rel_humidity_df
wind_speed_df = flatten_data_table(wind_speed_df, 'Wind Speed')
wind_speed_df<jupyter_output><empty_output><jupyter_text>Converting Data to a Single Data Frame<jupyter_code>data_df = pd.concat([atm_pressure_df, min_temp_df, max_temp_df, rel_humidity_df, wind_speed_df], axis=1)
data_df.head()<jupyter_output><empty_output><jupyter_text>Saving data to be used for modelling<jupyter_code>data_df.to_csv('modelling_data.csv')<jupyter_output><empty_output><jupyter_text>Inspecting Data<jupyter_code>data_df.describe()
data_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 1096 entries, 2018-01-01 to 2020-12-31
Data columns (total 5 columns):
 #   Column                Non-Null Count  Dtype  
---  ------                --------------  -----  
 0   Atmospheric Pressure  1096 non-null   float64
 1   Minimum Temperature   1096 non-null   float64
 2   Maximum Temperature   1096 non-null   float64
 3   Relative Humidity     1096 non-null   float64
 4   Wind Speed            1096 non-null   float64
dtypes: float64(5)
memory usage: 51.4 KB<jupyter_text>Visualization of Data for Correlation and Relationship Checking Correlation Between Variables Before Modelling<jupyter_code>data_corr_df = data_df.corr()
data_corr_df
import matplotlib.pyplot as plt
import seaborn as sb

plt.figure(figsize=(8, 5))

plt.title('Correlation Plot of Variables', fontsize=18)

sb.heatmap(data_corr_df, annot=True, fmt=".2f", cmap="Blues", vmin= -1, vmax=1, linewidth=0.3, cbar_kws={"shrink": .8})

plt.show()
from sklearn.preprocessing import normalize

normalized_df = normalize(data_df, axis=0)
normalized_df = pd.DataFrame(normalized_df, index=date_series, 
                             columns=["Atmospheric Pressure", "Minimum Temperature", "Maximum Temperature", 
                                      "Relative Humidity", "Wind Speed"])
normalized_df
plt.figure(figsize=(20,8))

plt.title("Time Plot Summary", fontsize=18)
plt.plot(normalized_df)

plt.legend(["Atmospheric Pressure", "Minimum Temperature", "Maximum Temperature", "Relative Humidity", "Wind Speed"])

plt.show<jupyter_output><empty_output><jupyter_text>Plot Smoothing Performing a data filter on the normalized data to smooth the plot for better visualization of data.The visualization would also provide a summary of the relationship between the variables to be used in the modelling.<jupyter_code>from scipy.signal import savgol_filter

filtered_df = normalized_df.apply(lambda x : savgol_filter(x, 53, 4))
# fft_df = np.fft.ifftn(normalized_df)

filtered_df = pd.DataFrame(filtered_df, index=date_series, 
                             columns=["Atmospheric Pressure", "Minimum Temperature", "Maximum Temperature", 
                                      "Relative Humidity", "Wind Speed"])

filtered_df.head()
plt.figure(figsize=(20,8))

plt.title("Filtered Time Plot Summary", fontsize=18)

plt.plot(filtered_df)
plt.plot(normalized_df, alpha=0.2)

plt.legend(["Atmospheric Pressure", "Minimum Temperature", "Maximum Temperature", "Relative Humidity", "Wind Speed"])

plt.show<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13096270598438856, 'fraction_numerical': 0.027464585140213934, 'mean_word_length': 4.552969502407705, 'pattern_counts': {'":': 1, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '376463', 'license': 'MIT', 'n_tokens_mistral': 2608, 'n_tokens_neox': 2318, 'n_words': 474, 'path': 'WeatherForecast-PreLSTM.ipynb', 'repo_name': 'olanrewajufarooq/WeatherForecast-LSTMModel', 'size': '6918'}