<jupyter_start><jupyter_text>Reading Dataset<jupyter_code>CANADA_PORT_OF_ENTRY_FILE = "../data/canada_port_of_entry_data.csv"
# Read CSV files into DataFrame df

df_canada_entry = pd.read_csv(CANADA_PORT_OF_ENTRY_FILE, 
#                                         index_col = 0, 
                                        header=0)<jupyter_output><empty_output><jupyter_text>Cleanig Data<jupyter_code>df_canada_entry.drop(["DGUID",
                      "UOM",
                      "UOM_ID",
                      "SCALAR_FACTOR",
                      "SCALAR_ID",
                      "VECTOR",
                      "COORDINATE",
                      "STATUS",
                      "SYMBOL",
                      "TERMINATED",
                      "DECIMALS"], inplace = True, axis =1)
df_canada_entry = df_canada_entry[df_canada_entry["GEO"] != "Canada"]
df_canada_entry = df_canada_entry[df_canada_entry["Country of residence"].str.contains('total') == False]
df_canada_entry = df_canada_entry[df_canada_entry["Country of residence"].str.contains('Total') == False]
df_canada_entry = df_canada_entry[df_canada_entry["Country of residence"].str.contains('classified') == False]
df_canada_entry = df_canada_entry[df_canada_entry["Country of residence"].str.contains('Island') == False]
df_canada_entry = df_canada_entry[df_canada_entry["Country of residence"].str.contains('part') == False]
df_canada_entry = df_canada_entry[df_canada_entry["Country of residence"].str.contains('Territory') == False]
df_canada_entry[['YEAR', 'MONTH']] = df_canada_entry['REF_DATE'].str.split('-', 1, expand=True)

df_canada_entry['YEAR'] = df_canada_entry['YEAR'].astype(int)
df_canada_entry['MONTH'] = df_canada_entry['MONTH'].astype(int)
df_canada_entry.drop(["REF_DATE"], inplace = True, axis =1)
df_cleaned_groupby_entry = df_canada_entry.groupby(['YEAR','MONTH','Country of residence','GEO']).sum()
df_cleaned_groupby_entry<jupyter_output><empty_output><jupyter_text>Extracting DEMAND by Entry Data by Year and Month<jupyter_code># df_cleaned_groupby_entry = (df_cleaned_groupby_entry - df_cleaned_groupby_entry.min())/(df_cleaned_groupby_entry.max() - df_cleaned_groupby_entry.min())
df_year_month_entry_data = {}

START_YR = 2017
END_YR = 2021

for year in range(START_YR, END_YR + 1):
    months_data = {}
    for month in range(1, 12 + 1):
        if year == 2021 and month == 9:
            break
        month_df = (df_cleaned_groupby_entry.loc[(year,month)] - df_cleaned_groupby_entry.loc[(year)].min()) / (df_cleaned_groupby_entry.loc[(year)].max() - df_cleaned_groupby_entry.loc[(year)].min())
        months_data[month] = month_df.reset_index().pivot_table(index='Country of residence', 
                                                                  columns='GEO', 
                                                                  values='VALUE',
                                                                  aggfunc='sum', 
                                                                  fill_value=0)
        months_data[month] = months_data[month].sort_values(['Ontario'], ascending=False)
    df_year_month_entry_data[year] = months_data
    
YEAR = 2019
MONTH = 12

df_year_month_entry_data[YEAR][MONTH]#.style.background_gradient(cmap='Reds')<jupyter_output><empty_output><jupyter_text>Problity Data<jupyter_code>df_year_month_probablity_data = {}

START_YR = 2017
END_YR = 2021

for year in range(START_YR, END_YR + 1):
    months_data = {
        "proviences_sums" : [],
        "provience_df" : None,
        "countries_sum" : [],
        "country_df" : None
    }
    for month in range(1, 12 + 1):
        if year == 2021 and month == 9:
            break
        
        df_groupby = df_cleaned_groupby_entry.loc[(year,month)].groupby(['GEO'])
        df_sum = df_groupby.sum().sum()
        months_data["proviences_sums"].append(df_sum.VALUE)
        if(month == 1):
            months_data["provience_df"] = df_groupby.sum() / df_sum
        else:
            df = df_groupby.sum() / df_sum
            months_data["provience_df"] = months_data["provience_df"].join(df,
                                              how="inner",
                                              lsuffix="",
                                              rsuffix="_"+str(month))
        
        df_groupby = df_cleaned_groupby_entry.loc[(year,month)].groupby(['Country of residence'])
        df_sum = df_groupby.sum().sum()
        months_data["countries_sum"].append(df_sum.VALUE)
        if(month == 1):
            months_data["country_df"] = df_groupby.sum() / df_sum
        else:
            df = df_groupby.sum()/ df_sum
            months_data["country_df"] = months_data["country_df"].join(df,
                                              how="inner",
                                              lsuffix="",
                                              rsuffix="_"+str(month))
            
    months_data["provience_df"].rename(columns = {'VALUE':'1',
                                                  'VALUE_2':'2',
                                                  'VALUE_3':'3',
                                                  'VALUE_4':'4',
                                                  'VALUE_5':'5',
                                                  'VALUE_6':'6',
                                                  'VALUE_7':'7',
                                                  'VALUE_8':'8',
                                                  'VALUE_9':'9',
                                                  'VALUE_10':'10',
                                                  'VALUE_11':'11',
                                                  'VALUE_12':'12'}, inplace = True)
    months_data["provience_df"].sort_values(['1'], ascending=False, inplace = True)
    
    months_data["country_df"].rename(columns = {'VALUE':'1',
                                                  'VALUE_2':'2',
                                                  'VALUE_3':'3',
                                                  'VALUE_4':'4',
                                                  'VALUE_5':'5',
                                                  'VALUE_6':'6',
                                                  'VALUE_7':'7',
                                                  'VALUE_8':'8',
                                                  'VALUE_9':'9',
                                                  'VALUE_10':'10',
                                                  'VALUE_11':'11',
                                                  'VALUE_12':'12'}, inplace = True)
    months_data["country_df"].sort_values(['1'], ascending=False, inplace = True)
    
    df_year_month_probablity_data[year] = months_data
    
YEAR = 2020
df_year_month_probablity_data[YEAR]["provience_df"]#.style.background_gradient(cmap='Reds')
df_year_month_probablity_data[YEAR]["country_df"]#.style.background_gradient(cmap='Reds')<jupyter_output><empty_output><jupyter_text>Heat map for probablities<jupyter_code>sns.set(rc = {'figure.figsize':(15,1)})

def country_heatmap_probablities(year):
    sns.heatmap(df_year_month_probablity_data[year]["country_df"].loc[["China","United Kingdom","France"]],
            annot = True, 
            linewidths=1,
            cmap="Blues",
            vmin=0.0, 
            vmax=1.0)
    plt.show()
interact(country_heatmap_probablities, year = (START_YR,END_YR))
sns.set(rc = {'figure.figsize':(15,5)})

def provience_heatmap_probablities(year):
    sns.heatmap(df_year_month_probablity_data[year]["provience_df"],
            annot = True, 
            linewidths=1,
            cmap="Blues",
            vmin=0.0, 
            vmax=1.0)
    plt.show()
interact(provience_heatmap_probablities, year = (START_YR,END_YR))<jupyter_output><empty_output><jupyter_text>Heat map for deman year wise<jupyter_code>MONTH = 1
sns.set(rc = {'figure.figsize':(15,3)})

def country_heatmap_demand(year):
    sns.heatmap(df_year_month_entry_data[year][MONTH].loc[["China","United Kingdom","France","India","Australia"]],
            annot = True, 
            linewidths=1,
            cmap="Blues",
            vmin=0.0, 
            vmax=1.0)
    plt.show()
interact(country_heatmap_demand, year = (START_YR,END_YR))<jupyter_output><empty_output><jupyter_text>Save data in csv<jupyter_code>NORMAL_COUNTRY_PROBABLITY = '../data/cleaned/canada/canada_country_normal_monthly_probablity.csv'
NORMAL_PROVIENCE_PROBABLITY = '../data/cleaned/canada/canada_provience_normal_monthly_probablity.csv'
PANDAMIC_COUNTRY_PROBABLITY = '../data/cleaned/canada/canada_country_pandamic_monthly_probablity.csv'
PANDAMIC_PROVIENCE_PROBABLITY = '../data/cleaned/canada/canada_provience_pandamic_monthly_probablity.csv'
df_year_month_probablity_data[2019]["country_df"].to_csv(NORMAL_COUNTRY_PROBABLITY)
df_year_month_probablity_data[2019]["provience_df"].to_csv(NORMAL_PROVIENCE_PROBABLITY)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1220221451739179, 'fraction_numerical': 0.01644111396935466, 'mean_word_length': 1.9222222222222223, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 67, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '365101', 'license': 'MIT', 'n_tokens_mistral': 2814, 'n_tokens_neox': 2636, 'n_words': 323, 'path': 'notebooks/canada_port_of_entry_data.ipynb', 'repo_name': 'AbdulMutakabbir/airfare-multi-agent-system', 'size': '8941'}