<jupyter_start><jupyter_text>1 Preprocessing @author: <NAME> @email:  <EMAIL>This code aims to make brut data workable.   LIBRARIES<jupyter_code>import numpy as np                                      # library for matrix computation.
import pandas as pd                                     # library for data frame manipulation.
import matplotlib.pyplot as plt                         # library for graphic plotting.
from preprocessingFunctions import build_dict_currency  # python script containing the function which retrieves exchange rates.<jupyter_output><empty_output><jupyter_text>FUNCTIONS<jupyter_code># Collecting all of available genres in our data.
def list_str(row, L):
    '''
    <list_str> adds all elements of the <row> array to the list L.
  
    Parameters
    ----------
    row : array
        
    L : list
        List having all elements of row among its items.

    Returns
    -------
    None.
    '''    
    for k in row.split('|'):
        L.append(k)


# Getting the exchange rates associated with the different currencies present in our data.
''' Annual range of the study.'''
YEAR_START, YEAR_END = 2000, 2019
set_list = build_dict_currency(YEAR_START, YEAR_END)

def get_coeff(row, year):
    '''
    <get_coeff> retrieves the coefficient associated with a given exchange rate
    <row> at the year: <year>, we set the US dollar ($) as baseline.

    Parameters
    ----------
    row : str
        The non-numerical string supposed representing currency symbol.
    year : int
        The year at which we want knowing the exchange rate of the currency represented
        by <row>.

    Returns
    -------
    coeff : float
        Exchange rate of the <row> currency against the US dollar.
    '''
    coeff = 1                                                          # default coefficient that does not make change from US dollars.
    if (year>=YEAR_START and year<=YEAR_END) and row and row!='<a':
        coeff = set_list[str(year)][row]                               # accesses to the desired exchange rate.
    else:
        pass      
    return coeff<jupyter_output>2000: downloaded exchange rates.
2001: downloaded exchange rates.
2002: downloaded exchange rates.
2003: downloaded exchange rates.
2004: downloaded exchange rates.
2005: downloaded exchange rates.
2006: downloaded exchange rates.
2007: downloaded exchange rates.
2008: downloaded exchange rates.
2009: downloaded exchange rates.
2010: downloaded exchange rates.
2011: downloaded exchange rates.
2012: downloaded exchange rates.
2013: downloaded exchange rates.
2014: downloaded exchange rates.
2015: downloaded exchange rates.
2016: downloaded exchange rates.
2017: downloaded exchange rates.
2018: downloaded exchange rates.
2019: downloaded exchange rates.<jupyter_text>SCRIPT  1)  Data CleaningTHIS PART ALLOW US TO GATHER ALL THE DATA AND THEN CENTRALIZE COMPUTATIONS THEREWITH AVOIDING USELESS 'FOR' LOOP TO MAKE CALCULUS AT EACH YEAR.<jupyter_code>#------------------------------------------------------------------------------
'''
1)
THIS PART ALLOW US TO GATHER ALL THE DATA AND THEN CENTRALIZE COMPUTATIONS 
THEREWITH AVOIDING USELESS 'FOR' LOOP TO MAKE CALCULUS AT EACH YEAR.
'''

df0 = pd.read_csv('../2_data/1_original/1_imdbRetrieval.csv', sep=';', dtype=str)
df0.rename(columns={k:k.replace(' ', '_') for k in df0.columns}, inplace=True)

df1 = df0[~df0.Cumulative_Worldwide_Gross.isna() & ~df0.Budget.isna() &
          ~df0.genre.isna() & ~df0.year.isna() &
          ~df0.note.isna() & ~df0.nbre_note.isna() &
          ~df0.year.str.contains('\D', na=False)].copy()

df1.loc[:, 'year'] = df1.year.astype(int)
float_col_list = ['note', 'nbre_note', 'Budget', 'Opening_Weekend_USA',
       'Gross_USA', 'Cumulative_Worldwide_Gross']
df1.loc[:, float_col_list]= df1.loc[:, float_col_list].astype(float)

df1.loc[:, 'genre'] = df1.loc[:, 'genre'].apply(
    lambda row: '|'.join(sorted(row.split('|'))))
df1 = df1[(df1.year >= YEAR_START) & (df1.year<=YEAR_END)]
runtime_HM_serie = df1.runtime.replace({'(?<=\d)*h$': ':00', 'h\s*': ':',
                          '^(?!\dh)':'00:', 'min':''}, regex=True)
time_HM_serie = pd.to_datetime(runtime_HM_serie, format='%H:%M').dt
df1.loc[:, 'runtime_min'] = time_HM_serie.hour *60 + time_HM_serie.minute
#------------------------------------------------------------------------------            
'''           
3)
MAKING SOME OBSERVATIOPNS ON THE DATA QUALITY AND SOME INSIGHTS VIA GRAPHICS.
'''
# a. Data quality
v = df0.isna().any()                                                 # boolean serie having as index <df0> columns and False value if a given column contains NaN values, True otherwise.
print('Number of columns containing at least one N.A.N. value: ',
      v[v].shape[0])

s = df0.duplicated()                                                 # boolean serie having indexed by duplicated rows of <df0>.
index_dup = s[s].index.values
print('Number of duplicated rows:', index_dup.shape[0])

# b. Getting all the different existent genres.
genre_list = []
df1["genre"].apply(lambda x: list_str(x, genre_list))                # appends all 'genres' values in <df1> into the list <genre_list>
genre_list = sorted(genre_list)
dif_genre_list = sorted(list(set(genre_list)))                       # gets all different genres.


# c. Building of a dictionary having the 'genres' as keys and their 
# associated count as values, and sorted by count.
count_genres_dict = {k:genre_list.count(k) for k in dif_genre_list}
count_genres_dict = {
k: v for k,v in sorted(count_genres_dict.items(),
                       key= lambda item: item[1],
                       reverse=True)}
# Graph Count
genre_name_list = list(count_genres_dict.keys())
genre_count_list = list(count_genres_dict.values())
fig, axs = plt.subplots(figsize=(10, 5))
axs.bar(genre_name_list, genre_count_list)
plt.xticks(rotation=90)
plt.xlabel("Genres")
plt.ylabel("Count")
plt.title('Count of the available genres.')
plt.show()
#------------------------------------------------------------------------------            
'''   
4)
CONVERSION OF MONEY IN US DOLLARS ($).
'''

df1.loc[:, 'Budget_coef'] = df1[["Budget_curr", "year"]].apply(
    lambda x: get_coeff(x[0], x[1]), axis=1)

df1.loc[:, 'CWG_coef'] = df1[["Cumulative_Worldwide_Gross_curr", "year"]].apply(
    lambda x: get_coeff(x[0], x[1]), axis=1)


df1.loc[:, "Budget_std"] = df1["Budget_coef"] * df1["Budget"]
df1["CWG_std"] = df1["CWG_coef"] * df1["Cumulative_Worldwide_Gross"]
#------------------------------------------------------------------------------            
'''   
5)
STORING OF THE DATA
'''

lcol = ["title", 'genre', 'year', "country", "runtime_min", 'note', 'nbre_note',
        'Budget', 'Cumulative_Worldwide_Gross', 'Budget_curr',
        'Cumulative_Worldwide_Gross_curr', 'Budget_std', 'CWG_std', 'Directors',
        'Writers', 'Stars']
df2 = df1.loc[:,lcol].drop_duplicates()

df2.to_csv('../2_data/2_transformed/1_processedData.csv', sep=';', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1665716728412653, 'fraction_numerical': 0.02336848104873183, 'mean_word_length': 2.627390180878553, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '294909', 'license': 'MIT', 'n_tokens_mistral': 2326, 'n_tokens_neox': 2170, 'n_words': 620, 'path': '1_scripts/.ipynb_checkpoints/1_preprocessing.py-checkpoint.ipynb', 'repo_name': 'rgb250/MostValuableFilmGenre', 'size': '7040'}