<jupyter_start><jupyter_text>0.0 Imports<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np<jupyter_output><empty_output><jupyter_text>1.0 Abertura do arquivo<jupyter_code>df_enem = pd.read_csv('https://raw.githubusercontent.com/kleite/Imersao_dados/main/data/enem_2019_sample_43278')
df_enem.head()<jupyter_output><empty_output><jupyter_text>2.0 Descriçao dos dados  2.1. Tamanho dos dados<jupyter_code>print('Número de linhas: {}'.format(df_enem.shape[0]))
print('Número de colunas: {}'.format(df_enem.shape[1]))<jupyter_output>Número de linhas 21150
Número de colunas 136<jupyter_text>2.2 Tipos de dados<jupyter_code>num_attributes = df_enem.select_dtypes( include=['int64', 'float64'] )
cat_attributes = df_enem.select_dtypes( exclude=['int64','float64'] )

print('Número de variáveis do tipo numérico: {}'.format(num_attributes.columns.value_counts().sum()))
print('Número de tipo categorico: {}'.format(cat_attributes.columns.value_counts().sum()))<jupyter_output>Número de variáveis do tipo numérico: 95
Número de tipo categorico: 41<jupyter_text>2.3 Dados repetidos<jupyter_code>df_enem = df_enem.drop_duplicates()<jupyter_output><empty_output><jupyter_text>2.4 Dados em branco 2.4.1 Contando dados em branco<jupyter_code>df_enem = df_enem.isnull().sum()/df_enem.shape[0]
df_enem[(df_enem) != 0].sort_values(ascending = False)<jupyter_output><empty_output><jupyter_text>2.4.1 Preenchendo dados em branco 2.5 Análise descritiva<jupyter_code>skew = num_attributes.skew()
skew = pd.DataFrame(skew, columns = ['skew']).T

kurt = num_attributes.kurt()
kurt = pd.DataFrame(kurt, columns = ['kurt']).T
pd.concat([num_attributes.describe(), skew, kurt])<jupyter_output><empty_output><jupyter_text>3.0 Feature engineering<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13578889499725125, 'fraction_numerical': 0.031335898845519516, 'mean_word_length': 8.837837837837839, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '495269', 'license': 'MIT', 'n_tokens_mistral': 767, 'n_tokens_neox': 734, 'n_words': 113, 'path': 'Imersao_dados.ipynb', 'repo_name': 'kleite/Imersao_dados', 'size': '1819'}