<jupyter_start><jupyter_text>We are going to transform weather and electric load data to timeseries format, where each row represents a datetime with 1h granularity.  Electric Load<jupyter_code>load_data=pd.read_excel('Total_Load_20180601_20210630.xlsx') # This is the file we downloaded from ENTSO-e transparency plattform

# https://transparency.entsoe.eu/load-domain/r2/totalLoadR2/show?name=&defaultValue=false&viewType=TABLE&areaType=CTY&atch=false&dateTime.dateTime=22.07.2021+00:00|CET|DAY&biddingZone.values=CTY|10YGR-HTSO-----Y!CTY|10YGR-HTSO-----Y&dateTime.timezone=CET_CEST&dateTime.timezone_input=CET+(UTC+1)+/+CEST+(UTC+2)
load_data.head(48)
load_data['Time']=load_data['Time'].apply(lambda x: x.split(' -')[0]) # keep the first part of the date before '-'
load_data=load_data.drop_duplicates(subset=['Time'],keep='first') # in case there are duplicate dates
load_data
def to_date(df_row):
    date_time_str = df_row
    date_time_obj = datetime.datetime.strptime(df_row, '%d.%m.%Y %H:%M') # convert string to datetime
    return(date_time_obj)


load_data['Time']=load_data['Time'].apply(to_date)
load_data=load_data.sort_values(by='Time')
load_data.set_index(pd.DatetimeIndex(load_data.Time),inplace=True)
# in case there is no 1-hour granularity, resample per hour and forward fill missing hours
load_data=load_data.resample('H').pad()
load_data.Time = load_data.index.values
load_data=load_data.ffill(axis = 0)
load_data=load_data.reset_index(drop=True)
load_data
load_data.to_excel('Actual_load.xlsx',header=True,index=False) # export to excel<jupyter_output><empty_output><jupyter_text>Weather Data<jupyter_code># Join Athens and Thessaloniki weather timeseries
weather_athens=pd.read_excel('weather_timeseries_Athens.xlsx')
weather_thess=pd.read_excel('weather_timeseries_Thessaloniki.xlsx')

weather_timeseries = pd.merge(weather_athens, 
                            weather_thess, 
                            on='Date',
                            how='inner')

weather_timeseries.rename(columns={'Temperature_x': 'Temperature_Athens', 'Wind_x': 'Wind_Athens',
                                  'Humidity_x': 'Humidity_Athens', 'Temperature_y': 'Temperature_Thessaloniki',
                                  'Wind_y': 'Wind_Thessaloniki', 'Humidity_y': 'Humidity_Thessaloniki'}, inplace=True)

weather_timeseries = weather_timeseries[["Date", "Temperature_Athens", "Wind_Athens", "Humidity_Athens",
                                        "Temperature_Thessaloniki", "Wind_Thessaloniki", "Humidity_Thessaloniki"]]
weather_timeseries
# Concat the weather data with the load data for all the years
load_data.columns=['Date','Load']
timeseries_data = pd.merge(load_data, 
               weather_timeseries, 
               on='Date',
               how='left')
timeseries_data
timeseries_data.to_excel('all_data.xlsx',header=True,index=False) # export to excel<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12169130285321417, 'fraction_numerical': 0.014437951185974561, 'mean_word_length': 4.808383233532934, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '353029', 'license': 'MIT', 'n_tokens_mistral': 1046, 'n_tokens_neox': 986, 'n_words': 157, 'path': 'Collect and Preprocess/combine_weather_load.ipynb', 'repo_name': 'ManosPra/ShortTermLoadForecasting-with-ANNs', 'size': '2909'}