<jupyter_start><jupyter_text>This Notebook contained the creation and Visulization of HDF5 File   Importing the Neccessary Libraries<jupyter_code>import numpy as np
import h5py
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns 
from datetime import datetime, timedelta, date
from numpy import median, mean, var, std, amax, amin
import datetime as dt
import matplotlib.dates as mdates

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Creating The HDF5 Groups and the datasets<jupyter_code>#Nigeria Stock Exchange
path_first_st_nigeria = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/stock_ng_first.csv'
read_csv_st_ng_fr = pd.read_csv(path_first_st_nigeria)
first_data_stock_nigeria = np.array(read_csv_st_ng_fr, dtype='a25')

path_second_st_nigeria = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/stock_ng_second.csv'
read_csv_st_ng_sd = pd.read_csv(path_second_st_nigeria)
second_data_stock_nigeria = np.array(read_csv_st_ng_sd, dtype='a25')

path_third_st_nigeria = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/stock_ng_third.csv'
read_csv_st_ng_th = pd.read_csv(path_third_st_nigeria)
third_data_stock_nigeria = np.array(read_csv_st_ng_th, dtype='a25')

path_fourth_st_nigeria = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/stock_ng_fourth.csv'
read_csv_st_ng_ft = pd.read_csv(path_third_st_nigeria)
fourth_data_stock_nigeria = np.array(read_csv_st_ng_ft, dtype='a25')


#Central Bank of Nigeria 
path_first_cbn_nigeria = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/cbn_rate_first.csv'
read_csv_cbn_ng_fr = pd.read_csv(path_first_cbn_nigeria)
first_data_rate_nigeria = np.array(read_csv_cbn_ng_fr, dtype='a25')

path_second_cbn_nigeria = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/cbn_rate_second.csv'
read_csv_cbn_ng_sd = pd.read_csv(path_second_cbn_nigeria)
second_data_rate_nigeria = np.array(read_csv_cbn_ng_sd, dtype='a25')

path_third_cbn_nigeria = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/cbn_rate_third.csv'
read_csv_rate_ng_th = pd.read_csv(path_third_cbn_nigeria)
third_data_rate_nigeria = np.array(read_csv_rate_ng_th, dtype='a25')

path_fourth_cbn_nigeria = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/cbn_rate_fourth.csv'
read_csv_rate_ng_ft = pd.read_csv(path_fourth_cbn_nigeria)
fourth_data_rate_nigeria = np.array(read_csv_rate_ng_ft, dtype='a25')

#UAE Stock

path_first_st_uae = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/stock_uae_first.csv'
read_csv_st_uae_fr = pd.read_csv(path_first_st_uae)
first_data_stock_uae = np.array(read_csv_st_uae_fr, dtype='a25')

path_second_st_uae = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/stock_uae_second.csv'
read_csv_st_uae_sd = pd.read_csv(path_second_st_uae)
second_data_stock_uae = np.array(read_csv_st_uae_sd, dtype='a25')

path_third_st_uae = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/stock_uae_third.csv'
read_csv_st_uae_th = pd.read_csv(path_third_st_uae)
third_data_stock_uae = np.array(read_csv_st_uae_th, dtype='a25')

path_fourth_st_uae = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/stock_uae_fourth.csv'
read_csv_st_uae_ft = pd.read_csv(path_fourth_st_uae)
fourth_data_stock_uae = np.array(read_csv_st_uae_ft, dtype='a25')

#UAE Rate
path_first_rate_uae = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/rate_uae_first.csv'
read_csv_rate_uae_fr = pd.read_csv(path_first_rate_uae)
first_data_rate_uae = np.array(read_csv_rate_uae_fr, dtype='a25')

path_second_rate_uae = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/rate_uae_second.csv'
read_csv_rate_uae_sd = pd.read_csv(path_second_rate_uae)
second_data_rate_uae = np.array(read_csv_rate_uae_sd, dtype='a25')

path_third_rate_uae = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/rate_uae_third.csv'
read_csv_rate_uae_th = pd.read_csv(path_third_rate_uae)
third_data_rate_uae = np.array(read_csv_rate_uae_th, dtype='a25')

path_fourth_rate_uae = '/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/spiders/rate_uae_fourth.csv'
read_csv_rate_uae_ft = pd.read_csv(path_fourth_rate_uae)
fourth_data_rate_uae = np.array(read_csv_rate_uae_ft, dtype='a25')

with h5py.File('/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/scrape_data.h5', 'w') as hdf:
    
    UAECENTRALBANK = hdf.create_group('UAECentralBank')
    UAECENTRALBANK.create_dataset('ft_data_rate_uae', data=first_data_rate_uae)
    UAECENTRALBANK.create_dataset('sd_data_rate_uae', data=second_data_rate_uae)
    UAECENTRALBANK.create_dataset('td_data_rate_uae', data=third_data_rate_uae)
    UAECENTRALBANK.create_dataset('fth_data_rate_uae', data=fourth_data_rate_uae)

    UAECENTRALBANK.attrs['title'] = "The dataset in this group contain the exchange rate of UAE Central Bank"

    # UAE Stock Exchange
    UAESTOCK = hdf.create_group('UAEStockExchange')
    UAESTOCK.create_dataset('ft_data_stock_uae', data=first_data_stock_uae)
    UAESTOCK.create_dataset('sd_data_stock_uae', data=second_data_stock_uae)
    UAESTOCK.create_dataset('td_data_stock_uae', data=third_data_stock_uae)
    UAESTOCK.create_dataset('fth_data_stock_uae', data=fourth_data_stock_uae)

    UAESTOCK.attrs['title'] = "The dataset in this group contain the stock exchange of UAE"

    # Nigeria Exchange Rate
    CBNRATES = hdf.create_group('NigeriaExchangeRate')
    CBNRATES.create_dataset('ft_data_rate_cbn', data=first_data_rate_nigeria)
    CBNRATES.create_dataset('sd_data_rate_cbn', data=second_data_rate_nigeria)
    CBNRATES.create_dataset('td_data_rate_cbn', data=third_data_rate_nigeria)
    CBNRATES.create_dataset('fth_data_rate_cbn', data=fourth_data_rate_nigeria)

    CBNRATES.attrs['title'] = "The dataset in this group contain Nigerian Exchange rate"
    # Nigeria Stock Exchange
    STOCKNIGERIA = hdf.create_group('NigeriaStockExchange')
    STOCKNIGERIA.create_dataset('ft_stock_data_ng', data=first_data_stock_nigeria)
    STOCKNIGERIA.create_dataset('sd_stock_data_ng', data=second_data_stock_nigeria)
    STOCKNIGERIA.create_dataset('td_stock_data_ng', data=third_data_stock_nigeria)
    STOCKNIGERIA.create_dataset('fth_stock_data_ng', data=fourth_data_stock_nigeria)

    STOCKNIGERIA.attrs['title'] = "The dataset in this group contain Nigerian stock Exchange data"<jupyter_output><empty_output><jupyter_text>Reading the HDF5 file<jupyter_code>with h5py.File('/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/scrape_data.h5', 'r') as hdf:
    ls = list(hdf.keys())
    print('List of Group in this file:', ls)
    data = hdf.get('NigeriaExchangeRate')
    data_items = list(data.items())
    print('Items in the Group',data_items)
    get_data = hdf.get('NigeriaExchangeRate/fth_data_rate_cbn')
    fourth_rate = np.array(get_data)
    print(fourth_rate)<jupyter_output>List of Group in this file: ['NigeriaExchangeRate', 'NigeriaStockExchange', 'UAECentralBank', 'UAEStockExchange']
Items in the Group [('ft_data_rate_cbn', <HDF5 dataset "ft_data_rate_cbn": shape (2000, 5), type "|S25">), ('fth_data_rate_cbn', <HDF5 dataset "fth_data_rate_cbn": shape (2000, 5), type "|S25">), ('sd_data_rate_cbn', <HDF5 dataset "sd_data_rate_cbn": shape (2000, 5), type "|S25">), ('td_data_rate_cbn', <HDF5 dataset "td_data_rate_cbn": shape (2000, 5), type "|S25">)]
[[b'1/27/2022' b'US DOLLAR' b'415.16' b'415.66' b'416.16']
 [b'1/27/2022' b'POUNDS STERLING' b'555.9408' b'556.6103' b'557.2799']
 [b'1/27/2022' b'EURO' b'463.4016' b'463.9597' b'464.5178']
 ...
 [b'5/25/2021' b'YUAN/RENMINBI' b'63.8865' b'63.965' b'64.0435']
 [b'5/25/2021' b'RIYAL' b'109.1304' b'109.2638' b'109.3971']
 [b'5/25/2021' b'SOUTH AFRICAN RAND' b'29.5957' b'29.6319' b'29.6681']]<jupyter_text>Changing the data type from bytes to string<jupyter_code>formated_data_cbn = fourth_rate.astype('U13')
formated_data_cbn<jupyter_output><empty_output><jupyter_text>Numpy arrays to Pandas DataFrame<jupyter_code>cbn_rate = pd.DataFrame(formated_data_cbn, columns=['date', 'currency', 'buying', 'centralrate', 'sellingrate'])
cbn_rate.head(10)<jupyter_output><empty_output><jupyter_text>Dollar  Silicing the dataset to get only US Dollar data<jupyter_code>clean_rate_dollar = cbn_rate.loc[cbn_rate['currency'].isin(['US DOLLAR'])]
clean_rate_new_dollar = clean_rate_dollar[['date', 'currency', 'sellingrate']]
dollar_rate = clean_rate_new_dollar.astype({  "date": np.datetime64,"currency": str,  "sellingrate": float})
dollar_cbn_clean = dollar_rate[:35]
dollar_cbn_clean<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>fig_dims = (16, 8)

fig, ax = plt.subplots(figsize=fig_dims)
dollar_rate = sns.lineplot(data=dollar_cbn_clean, x="date", y="sellingrate", hue="currency", ax=ax)
sns.despine()

ax.annotate('Naira appreciate with 2 Kobo',
            xy=(dollar_cbn_clean.date[355], dollar_cbn_clean.sellingrate[355]), xycoords='data',
            xytext=(0, 100), 
            textcoords='offset points', 
            arrowprops=dict(arrowstyle="->"), 
            ha='center')

ax.annotate('4.42 Naira lost in Value ',
            xy=(dollar_cbn_clean.date[0], dollar_cbn_clean.sellingrate[0]), xycoords='data',
            xytext=(30, -50), textcoords='offset points',
            arrowprops=dict(arrowstyle="->"), ha='center')
ax.annotate('Depreciate with almost 1.50 Naira ',
            xy=(dollar_cbn_clean.date[213], dollar_cbn_clean.sellingrate[213]), xycoords='data',
            xytext=(0, 100), textcoords='offset points',
            arrowprops=dict(arrowstyle="->"), ha='center')
ax.annotate('Naira Depreciate',
            xy=(dollar_cbn_clean.date[237], dollar_cbn_clean.sellingrate[237]), xycoords='data',
            xytext=(-50, 100), textcoords='offset points',
            arrowprops=dict(arrowstyle="->"), ha='center')

plt.title("Movement of Dollar against Naira ")
plt.xlabel("Dates", size=16)
plt.ylabel("Rates", size=16)<jupyter_output><empty_output><jupyter_text>UAE Dirham  Reading the HDF5 file<jupyter_code>with h5py.File('/Users/buhariabubakar/PycharmProjects/boka/web_mining/web_mining/scrape_data.h5', 'r') as hdf:
    ls = list(hdf.keys())
    print('List of Group in this file:', ls)
    data = hdf.get('UAECentralBank')
    data_items = list(data.items())
    print('Items in the Group',data_items)
    get_data_first = hdf.get('UAECentralBank/ft_data_rate_uae')
    uae_rate_first = np.array(get_data_first)
    get_data_second = hdf.get('UAECentralBank/sd_data_rate_uae')
    uae_rate_second = np.array(get_data_second)
    get_data_third = hdf.get('UAECentralBank/td_data_rate_uae')
    uae_rate_third = np.array(get_data_third)
    get_data_fourth = hdf.get('UAECentralBank/fth_data_rate_uae')
    uae_rate_fourth = np.array(get_data_fourth)
    print(uae_rate_first)<jupyter_output>List of Group in this file: ['NigeriaExchangeRate', 'NigeriaStockExchange', 'UAECentralBank', 'UAEStockExchange']
Items in the Group [('ft_data_rate_uae', <HDF5 dataset "ft_data_rate_uae": shape (76, 2), type "|S25">), ('fth_data_rate_uae', <HDF5 dataset "fth_data_rate_uae": shape (76, 2), type "|S25">), ('sd_data_rate_uae', <HDF5 dataset "sd_data_rate_uae": shape (76, 2), type "|S25">), ('td_data_rate_uae', <HDF5 dataset "td_data_rate_uae": shape (152, 2), type "|S25">)]
[[b'US Dollar' b'3.6725']
 [b'Argentine Peso' b'0.03511']
 [b'Australian Dollar' b'2.618725']
 [b'Bangladesh Taka' b'0.042733']
 [b'Bahrani Dinar' b'9.742155']
 [b'Brunei Dollar' b'2.730889']
 [b'Brazilian Real' b'0.667934']
 [b'Botswana Pula' b'0.317487']
 [b'Belarus Rouble' b'1.409248']
 [b'Canadian Dollar' b'2.904999']
 [b'Swiss Franc' b'3.997932']
 [b'Chilean Peso' b'0.004542']
 [b'Chinese Yuan - Offshore' b'0.579478']
 [b'Chinese Yuan' b'0.580229']
 [b'Colombian Peso' b'0.00092']
 [b'Czech Koruna' b'0.169022']
 [[...]<jupyter_text>Silicing the dataset to get only US Dollar, Nigerian Naira against Dirham  Changing the data type from bytes to string<jupyter_code>uae_1_data_cbn = uae_rate_first.astype('U13')
uae_2_data_cbn = uae_rate_second.astype('U13')
uae_3_data_cbn = uae_rate_third.astype('U13')
uae_4_data_cbn = uae_rate_fourth.astype('U13')<jupyter_output><empty_output><jupyter_text>Numpy arrays to Pandas DataFrame<jupyter_code>uae_data_f = pd.DataFrame(uae_1_data_cbn, columns=[ 'currency', 'rate'])
uae_data_s = pd.DataFrame(uae_2_data_cbn, columns=[ 'currency', 'rate'])
uae_data_t = pd.DataFrame(uae_3_data_cbn, columns=[ 'currency', 'rate'])
uae_data_ft = pd.DataFrame(uae_4_data_cbn, columns=[ 'currency', 'rate'])<jupyter_output><empty_output><jupyter_text>Date column is not provided in the datasets, need to be added for analysis<jupyter_code>uae_data_f['date'] = pd.date_range(start='1/25/2022', end='1/25/2022',  periods=len(uae_data_f))
uae_data_s['date'] = pd.date_range(start='1/26/2022', end='1/26/2022',  periods=len(uae_data_s))
uae_data_t['date'] = pd.date_range(start='1/27/2022', end='1/27/2022',  periods=len(uae_data_t))
uae_data_ft['date'] = pd.date_range(start='1/28/2022', end='1/28/2022',  periods=len(uae_data_ft))


uae_data_ft.head(5)<jupyter_output><empty_output><jupyter_text>Append all the datasets to have one merge datasets<jupyter_code>uae_cbn_clean = uae_data_f.append([uae_data_s, uae_data_t, uae_data_ft])
result_dtypes = uae_cbn_clean.astype({ "date": np.datetime64, "currency": str,  "rate": float})
result_naira_new = result_dtypes.loc[result_dtypes['currency'].isin(['Nigerian Nair', 'US Dollar'])]
dirham_naira_rate = pd.DataFrame(result_naira_new, columns=['date', 'currency', 'rate'])
dirham_naira_rate
dirham_naira_rate.to_csv('/Users/buhariabubakar/Documents/IU/Semester 4/uae_rate.csv')
read_dirham = pd.read_csv('/Users/buhariabubakar/Documents/IU/Semester 4/uae_rate.csv')
cleaned_dubai_rate = read_dirham[['date', 'currency', 'rate' ]]
cleaned_dubai_rate<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>fig_dims = (15, 6)
fig, ax = plt.subplots(figsize=fig_dims)
euro_rate = sns.lineplot(data=cleaned_dubai_rate, x="date", y="rate", hue="currency", ax=ax,)
sns.despine()
plt.title("Movement of Dirham against Naira, and USD ")
plt.xlabel("Date", size=18)
plt.ylabel("Rates", size=18)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1297556234379339, 'fraction_numerical': 0.035129130797000835, 'mean_word_length': 7.259747706422019, 'pattern_counts': {'":': 14, '<': 59, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '511354', 'license': 'MIT', 'n_tokens_mistral': 6377, 'n_tokens_neox': 5928, 'n_words': 799, 'path': 'HDF.ipynb', 'repo_name': 'buhari15/boka', 'size': '14404'}