<jupyter_start><jupyter_text>Reading data<jupyter_code>data = pd.read_csv("data/Input2_clientes_venta.csv", delimiter = ';')
data.head()
clientes = pd.read_csv("data/Input1_clientes_estructura.csv", delimiter = ";")
clientes.head()
test = pd.read_csv("data/Input3_clientes_test.csv", delimiter = ";")
test.head()

test.shape
print ('Size of data: {}'.format(data.shape))
print ('Number of entries: {}'.format(data.shape[0]))
print ('Number of columns: {}'.format(data.shape[1]))

print ('\nList of columns in dataset:')

for col in data.columns:
    print (col)
features_tab1 = [i for i in list(data) if i not in ['Marca2', 'Cupo2', 'CapacidadEnvase2']]
features_tab2 = [i for i in list(clientes)]<jupyter_output><empty_output><jupyter_text>Train and test datasets<jupyter_code>marca1_data = data.loc[(data['Marca2'] == 'Marca_20') & (data['Cupo2'] == 'Cupo_3') & (data['CapacidadEnvase2'] == 'CapacidadEnvase_9')]
marca2_data = data.loc[(data['Marca2'] == 'Marca_16') & (data['Cupo2'] == 'Cupo_2')  & (data['CapacidadEnvase2'] == 'CapacidadEnvase_10')]
marca3_data = data.loc[(data['Marca2'] == 'Marca_9') & (data['Cupo2'] == 'Cupo_3')  & (data['CapacidadEnvase2'] == 'CapacidadEnvase_12')]
marca4_data = data.loc[(data['Marca2'] == 'Marca_38') & (data['Cupo2'] == 'Cupo_2')  & (data['CapacidadEnvase2'] == 'CapacidadEnvase_10')]
marca5_data = data.loc[(data['Marca2'] == 'Marca_39') & (data['Cupo2'] == 'Cupo_2')  & (data['CapacidadEnvase2'] == 'CapacidadEnvase_10')]

print ('Number of total Marca_1: {}'.format(marca1_data.shape[0]))
print ('Number of total Marca_2: {}'.format(marca2_data.shape[0]))
print ('Number of total Marca_3: {}'.format(marca3_data.shape[0]))
print ('Number of total Marca_Inno1: {}'.format(marca4_data.shape[0]))
print ('Number of total Marca_Inno2: {}'.format(marca5_data.shape[0]))
print ('++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')
print('Number of total entries of interest:', marca1_data.shape[0] 
      + marca2_data.shape[0] + marca3_data.shape[0] + marca4_data.shape[0] + marca5_data.shape[0])
print('Number of total entries:', data.shape[0])<jupyter_output>Number of total Marca_1: 5347
Number of total Marca_2: 9676
Number of total Marca_3: 56898
Number of total Marca_Inno1: 9089
Number of total Marca_Inno2: 9089
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Number of total entries of interest: 90099
Number of total entries: 1406116<jupyter_text>The **entries of interest** are around ~6.5% of the dataset for training and testing the algorithm<jupyter_code>marca1_data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21727343144848954, 'fraction_numerical': 0.046475600309837335, 'mean_word_length': 7.496710526315789, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '587453', 'license': 'MIT', 'n_tokens_mistral': 1030, 'n_tokens_neox': 984, 'n_words': 195, 'path': 'notebooks/Processing_sample.ipynb', 'repo_name': 'LauraTrujilloT/Datacup2020_TheCurlyPals_COL', 'size': '2582'}