<jupyter_start><jupyter_text>Removing outliers<jupyter_code>numvars = ['competidores',
       'ingreso_mediana', 'ingreso_promedio', 'densidad_poblacional',
       'pct_0a5', 'pct_5a9', 'pct_10a14', 'pct_15a19', 'pct_20a24',
       'pct_25a29', 'pct_30a34', 'pct_35a39', 'pct_40a44', 'pct_45a49',
       'pct_50a54', 'pct_55a59', 'pct_60a64', 'pct_65a69', 'pct_70a74',
       'pct_75a79', 'pct_80a84', 'pct_85ainf', 'pct_bachelors',
       'pct_doctorados', 'pct_secundario', 'pct_master', 'pct_bicicleta',
       'pct_omnibus', 'pct_subtes', 'pct_taxi', 'pct_caminata',
       'mediana_valor_hogar'
]
for n in numvars:
    outliers_ext = np.nanpercentile(pos[n].values, [5, 95]) # calculamos extremos de outliers
    pos[n] = np.where(pos[n] > outliers_ext[1], outliers_ext[1], pos[n]) 
    pos[n] = np.where(pos[n] < outliers_ext[0], outliers_ext[0], pos[n])<jupyter_output><empty_output><jupyter_text>Normalize numeric values<jupyter_code># Create the Scaler object
scaler = preprocessing.StandardScaler()
# Fit your data on the scaler object
scaled_df = scaler.fit_transform(pos[numvars])
scaled_df = pd.DataFrame(scaled_df, columns=numvars, index=pos.index)
submission = submission[submission.columns[submission.columns.isin(numvars) != True]]
submission = pd.merge(submission, scaled_df, how='inner', left_index=True, right_index=True)
submission.to_csv('../data/processed/submission_norm.csv', index=False, sep=';')
with open('../data/processed/submission_norm.pkl', 'wb') as f:
    # Pickle the 'data' dictionary using the highest protocol available.
    pickle.dump(submission, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15379864113650402, 'fraction_numerical': 0.045089561457689935, 'mean_word_length': 6.2, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '95680', 'license': 'MIT', 'n_tokens_mistral': 710, 'n_tokens_neox': 649, 'n_words': 105, 'path': 'notebooks/normalize_submission_data.ipynb', 'repo_name': 'Guillin/alixpartners', 'size': '1619'}