<jupyter_start><jupyter_text>Exploração de dados das bases de matemática e português * Importando bibliotecas<jupyter_code>import pandas as pd
import numpy as snp
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib
import copy<jupyter_output><empty_output><jupyter_text>* Importando base de dados<jupyter_code>df_mat = pd.read_csv("student-mat.csv")
df_port = pd.read_excel("student-por.xlsx")<jupyter_output><empty_output><jupyter_text>* Para matemática<jupyter_code>df_mat.head()<jupyter_output><empty_output><jupyter_text>* Para português<jupyter_code>df_port.head()<jupyter_output><empty_output><jupyter_text>* Features das bases de dados<jupyter_code>df_mat.columns<jupyter_output><empty_output><jupyter_text>* Distribuição da base de matemática<jupyter_code>df_mat.describe().T<jupyter_output><empty_output><jupyter_text>* Distribuição da base de português<jupyter_code>df_port.describe().T<jupyter_output><empty_output><jupyter_text>* Filtrando alunos por escola<jupyter_code># Base de Matemática
df_mat_GP = df_mat.loc[(df_mat["school"]=="GP")]
df_mat_MS = df_mat.loc[(df_mat["school"]=="MS")]
# Base de português
df_port_GP = df_port.loc[(df_port["school"]=="GP")]
df_port_MS = df_port.loc[(df_port["school"]=="MS")]<jupyter_output><empty_output><jupyter_text>* Pode-se percerber um desbalanceamento dos dados quanto as amostras para cada escola<jupyter_code># Proporção de alunos por escola na base de matemática
print(df_mat_GP.shape)
print(df_mat_MS.shape)
# Proporção de alunos por escola na base de português
print(df_port_GP.shape)
print(df_port_MS.shape)<jupyter_output>(423, 33)
(226, 33)<jupyter_text>* Analizando correlação entre atributos<jupyter_code># Matemática
sns.heatmap(df_mat.corr())
# Português
sns.heatmap(df_port.corr())<jupyter_output><empty_output><jupyter_text>* Distribuição geral de alunos por sexo e idade<jupyter_code># Matemática
sns.kdeplot(df_mat.groupby('sex').get_group('M')['age'], shade = True,label = 'male')
sns.kdeplot(df_mat.groupby('sex').get_group('F')['age'], shade = True, label = 'female')
plt.xlabel('data range')
plt.ylabel('% data distribution')
# Português
sns.kdeplot(df_port.groupby('sex').get_group('M')['age'], shade = True,label = 'male')
sns.kdeplot(df_port.groupby('sex').get_group('F')['age'], shade = True, label = 'female')
plt.xlabel('data range')
plt.ylabel('% data distribution')<jupyter_output><empty_output><jupyter_text>* Calculándo média das notas<jupyter_code>df_mat['mean_grades'] = (df_mat.G1 + df_mat.G2 + df_mat.G3) / 3
df_port['mean_grades'] = (df_port.G1 + df_port.G2 + df_port.G3) / 3<jupyter_output><empty_output><jupyter_text>* Distribuição da média na base de matemática<jupyter_code>plt.hist(df_mat.mean_grades)<jupyter_output><empty_output><jupyter_text>* Distribuição de média na base de português<jupyter_code>plt.hist(df_port.mean_grades)<jupyter_output><empty_output><jupyter_text>Análise de proporção de atributos entre os sexos masculinos e feminos * Para matemática<jupyter_code>cols_obj = list(df_mat.dtypes[df_mat.dtypes == object].index[0:])
cols_obj.pop(1)
i=1
plt.figure(figsize=[15,20])
y, hue = 'proportion', 'sex'

for f in cols_obj:
    plt.subplot(8,2,i)
    df_mat[[f, 'sex']]\
       .value_counts(normalize=True)\
       .rename(y)\
       .reset_index()\
       .pipe((sns.barplot, "data"), x=f, y='proportion', hue='sex', alpha=0.8)
    plt.title(f'Proportion of {f}')
    i+=1
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>* Para português<jupyter_code>cols_obj = list(df_port.dtypes[df_port.dtypes == object].index[0:])
cols_obj.pop(1)
i=1
plt.figure(figsize=[15,20])
y, hue = 'proportion', 'sex'

for f in cols_obj:
    plt.subplot(8,2,i)
    df_port[[f, 'sex']]\
       .value_counts(normalize=True)\
       .rename(y)\
       .reset_index()\
       .pipe((sns.barplot, "data"), x=f, y='proportion', hue='sex', alpha=0.8)
    plt.title(f'Proportion of {f}')
    i+=1
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>BoxPlot * Para matemática<jupyter_code>plt.figure(figsize = (15,15))
for i, item in enumerate(['schoolsup', 'famsup', 'paid','activities',
       'nursery', 'higher', 'internet', 'sex']):
    plt.subplot(4,2,i+1)
    order_by = df_mat.groupby(item)['mean_grades'].median().sort_values(ascending = False).index
    sns.boxplot(x = df_mat[item], y = df_mat['mean_grades'],order = order_by)
    plt.xticks(rotation = 90)
    plt.title(item+' v/s mean_grades')<jupyter_output><empty_output><jupyter_text>* Português<jupyter_code>plt.figure(figsize = (15,15))
for i, item in enumerate(['schoolsup', 'famsup', 'paid','activities',
       'nursery', 'higher', 'internet', 'sex']):
    plt.subplot(4,2,i+1)
    order_by = df_port.groupby(item)['mean_grades'].median().sort_values(ascending = False).index
    sns.boxplot(x = df_port[item], y = df_port['mean_grades'],order = order_by)
    plt.xticks(rotation = 90)
    plt.title(item+' v/s mean_grades')<jupyter_output><empty_output><jupyter_text>Relação entre: tempo de estudo, sexo e notas * Matemática<jupyter_code>sns.histplot(data=df_mat, x='studytime', y='mean_grades', hue='sex')<jupyter_output><empty_output><jupyter_text>* Português<jupyter_code>sns.histplot(data=df_port, x='studytime', y='mean_grades', hue='sex')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1590521327014218, 'fraction_numerical': 0.011374407582938388, 'mean_word_length': 7.308661417322835, 'pattern_counts': {'":': 0, '<': 80, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '362918', 'license': 'MIT', 'n_tokens_mistral': 2205, 'n_tokens_neox': 2142, 'n_words': 328, 'path': 'notebooks/data-exploration.ipynb', 'repo_name': 'tips2/projeto-da-discplina-de-ia-ahr', 'size': '5275'}