<jupyter_start><jupyter_text>Analisando o gênero A analise de gênero do beneficiário é realizada com base no Censo 2010.<jupyter_code>def load_name_data():
    fobj = io.TextIOWrapper(gzip.open("C:/Users/Aninha/Documents/Area de Trabalho/UFAL-DS/data/raw/censo/nomes.csv.gz"), encoding="utf-8")
    csv_reader = csv.DictReader(fobj)
    data = {
        row["first_name"]: row["classification"]
        for row in csv_reader
    }
    fobj.close()
    return data

name_data = load_name_data()

print(f"Dicionário de nomes do Censo 2010 contendo {len(name_data)} nomes.")
def encode(name):
    ascii_name = normalize("NFKD", name).encode("ascii", errors="ignore").decode("ascii")
    return ascii_name.upper()

def classify_gender(name):
    encoded_name = encode(name)
    try:
        gender = name_data[encoded_name]
        return gender
    except:
        return "NaN"

print(classify_gender("Wagner"))
dataset["gender"] = dataset.name.str.split(" ").str[0]
dataset["gender"] = dataset.gender.map(classify_gender)
f, (ax_box, ax_hist) = plt.subplots(2, sharex=True, gridspec_kw={"height_ratios": (0.25, 1)}, figsize=(16,8))
mean = dataset.value.mean()
median = dataset.value.median()

_ = sns.boxplot(dataset.value, ax=ax_box,  color=color, showfliers=True)
_ = ax_box.axvline(mean, color='r', linestyle='--')
_ = ax_box.axvline(median, color='g', linestyle='-')
_ = ax_box.set(xlabel='')

_ = sns.distplot(dataset.value, ax=ax_hist, color=color, kde=False)
_ = ax_hist.axvline(mean, color='r', linestyle='--')
_ = ax_hist.axvline(median, color='g', linestyle='-')
_ = ax_hist.xaxis.set_major_locator(ticker.MultipleLocator(5000))

_ = plt.legend({'Mean':mean,'Median':median}, fontsize='large')<jupyter_output><empty_output><jupyter_text>Criação do csv com os dados preprocessados<jupyter_code>dataset.to_csv('C:/Users/Aninha/Documents/Area de Trabalho/UFAL-DS/data/raw/preprocessing/abono.csv', sep=',')
dataset = pd.read_csv('C:/Users/Aninha/Documents/Area de Trabalho/UFAL-DS/data/raw/preprocessing/abono.csv', sep=',')
dataset.head()
## Tirando os outliers
![title](https://www.gradusct.com.br/wp-content/uploads/2018/07/Boxplot-04.png)
values = dataset['value']

q1 = values.quantile(.25)
q3 = values.quantile(.75)
iiq = q3 - q1
minimum = q1 - 1.5*iiq
maximum = q3 + 1.5*iiq

selection = (values >= minimum) & (values <= maximum)
data_new = dataset[selection]

selection = (values <= minimum) | (values >= maximum)
data_outliers = dataset[selection]<jupyter_output><empty_output><jupyter_text>Análise dos dados (sem os Outiliers)<jupyter_code>profile = ProfileReport(data_new, title='Abono Permanencia',html={'style':{'full_width':True}})
profile.to_notebook_iframe()
gp = data_new.groupby('region')
levels = data_new.region.value_counts()

fig = plt.figure(figsize=(14,30))
fig.subplots_adjust(hspace=0.4, wspace=0.4)

total_plots = levels.shape[0]
i = 0
for level in levels.keys():
    ax = fig.add_subplot(total_plots, 2, i+1)
    ax.set_title(level)
    
    top = gp.get_group(level)["gender"].value_counts()[:3]
    _ = sns.barplot(x=top.keys(), y=top.values, color=color, ax=ax)
    
    i += 1
plt.figure(figsize=(15,8))
ax = sns.distplot(data_new.value, bins=10)
data_new["educational_level"].value_counts()
f, (ax_box, ax_hist) = plt.subplots(2, sharex=True, gridspec_kw={"height_ratios": (0.25, 1)}, figsize=(16,8))
mean = data_new.value.mean()
median = data_new.value.median()

_ = sns.boxplot(data_new.value, ax=ax_box,  color=color, showfliers=False)
_ = ax_box.axvline(mean, color='r', linestyle='--')
_ = ax_box.axvline(median, color='g', linestyle='-')
_ = ax_box.set(xlabel='')

_ = sns.distplot(data_new.value, ax=ax_hist, color=color, kde=False)
_ = ax_hist.axvline(mean, color='r', linestyle='--')
_ = ax_hist.axvline(median, color='g', linestyle='-')
_ = ax_hist.xaxis.set_major_locator(ticker.MultipleLocator(5000))

_ = plt.legend({'Mean':mean,'Median':median}, fontsize='large')
data_new.boxplot(['value'], by = ['role_gp'], figsize = (15,8), rot=90, fontsize = 10)
gp = data_new.groupby('role_gp')
levels = data_new.role_gp.value_counts()

fig = plt.figure(figsize=(14,60))
fig.subplots_adjust(hspace=0.8, wspace=0.4)

total_plots = levels.shape[0]
i = 0
for level in levels.keys():
    ax = fig.add_subplot(total_plots, 2, i+1)
    ax.set_title(level)
    
    top = gp.get_group(level)["agency_gp"].value_counts()[:3]
    _ = sns.barplot(x=top.keys(), y=top.values, color=color, ax=ax)
    
    i += 1
gp = data_new.groupby('role_gp')
levels = data_new.role_gp.value_counts()

fig = plt.figure(figsize=(14,60))
fig.subplots_adjust(hspace=0.8, wspace=0.4)

total_plots = levels.shape[0]
i = 0
for level in levels.keys():
    ax = fig.add_subplot(total_plots, 2, i+1)
    ax.set_title(level)
    
    top = gp.get_group(level)["region"].value_counts()
    _ = sns.barplot(x=top.keys(), y=top.values, color=color, ax=ax)
    
    i += 1
print("Mediana:", data_new.value.median())
print("Média", data_new.value.mean())
print("Percentil", data_new.value.quantile(0.75))
print("Desvio Padrão", data_new.value.std())
states = data_new.groupby('state')
order = states.value.median().sort_values(ascending=True)
order
plt.figure(figsize=(15,8))
states = data_new.groupby('state')
order = states.value.median().sort_values(ascending=True).keys().to_list()

chart = sns.boxplot(
    x="upag_state", y="value", 
    data=data_new, color=color, showfliers=False, order=order
)

_ = chart.set_xticklabels(chart.get_xticklabels(), rotation=45, horizontalalignment='right')
plt.figure(figsize=(15,8))
states = data_new.groupby('region')
order = states.value.median().sort_values(ascending=True).keys().to_list()

chart = sns.boxplot(
    x="region", y="value", 
    data=data_new, color=color, showfliers=False, order = order
)

_ = chart.set_xticklabels(chart.get_xticklabels(), rotation=45, horizontalalignment='right')
plt.figure(figsize=(20,8))
_ = sns.lineplot(x="allowance_start_date", y="value", data=data_new)
# Prepare data
data_new['start_year'] = [d.year for d in data_new.allowance_start_date]
data_new['start_month'] = [d.strftime('%b') for d in data_new.allowance_start_date]
years = data_new['start_year'].unique()

fig, axes = plt.subplots(2, 1, figsize=(15,16), dpi= 80)
_ = sns.boxplot(x='start_year', y='value', data=data_new, ax=axes[0], showfliers=False)
_ = sns.boxplot(x='start_month', y='value', data=data_new.loc[~data_new.start_year.isin([2000, 2020]), :], showfliers=False)

# Set Title
axes[0].set_title('Year-wise Box Plot\n(The Trend)', fontsize=18); 
axes[1].set_title('Month-wise Box Plot\n(The Seasonality)', fontsize=18)
plt.show()
plt.figure(figsize=(15,8))

chart = sns.boxplot(
    x="educational_level", y="value", 
    data=data_new, color=color, showfliers=False
)

_ = chart.set_xticklabels(chart.get_xticklabels(), rotation=90, horizontalalignment='right')
ncount = data_new.shape[0]

plt.figure(figsize=(14,10))

order = [
    'alfabetizado', '1º grau incompleto', 'ensino fundamental',
    '2º grau incompleto', 'ensino medio', 'superior incompleto',
    'ensino superior',  'mestrado', 'doutorado'
]

ax = sns.countplot(
    x="educational_level", data=data_new, 
    color=color, order=order
)
plt.xlabel('educational level')

ax2=ax.twinx()

ax2.yaxis.tick_left()
ax.yaxis.tick_right()

ax.yaxis.set_label_position('right')
ax2.yaxis.set_label_position('left')

ax2.set_ylabel('Frequency [%]')

for p in ax.patches:
    x=p.get_bbox().get_points()[:,0]
    y=p.get_bbox().get_points()[1,1]
    ax.annotate('{:.1f}%'.format(100.*y/ncount), (x.mean(), y), 
            ha='center', va='bottom') 

ax.yaxis.set_major_locator(ticker.LinearLocator(11))

_ = ax.set_xticklabels(chart.get_xticklabels(), rotation=45, horizontalalignment='right')
data_new["educational_level"].value_counts()<jupyter_output><empty_output><jupyter_text>Análise dos Outliers<jupyter_code>plt.figure(figsize=(15,8))
ax = sns.distplot(data_outliers.value, bins=10)
profile_outliers = ProfileReport(data_new, title='Abono Permanencia - Outliers',html={'style':{'full_width':True}})
profile_outliers.to_notebook_iframe()
data_outliers.boxplot(['value'], by = ['agency_gp'], figsize = (15,8), rot=90, fontsize = 10)
data_outliers.boxplot(['value'], by = ['role_gp'], figsize = (15,8), rot=90, fontsize = 10)
gp = data_outliers.groupby('region')
levels = data_outliers.region.value_counts()

fig = plt.figure(figsize=(14,30))
fig.subplots_adjust(hspace=0.4, wspace=0.4)

total_plots = levels.shape[0]
i = 0
for level in levels.keys():
    ax = fig.add_subplot(total_plots, 2, i+1)
    ax.set_title(level)
    
    top = gp.get_group(level)["gender"].value_counts()[:3]
    _ = sns.barplot(x=top.keys(), y=top.values, color=color, ax=ax)
    
    i += 1
gp = data_outliers.groupby('role_gp')
levels = data_outliers.role_gp.value_counts()

fig = plt.figure(figsize=(14,60))
fig.subplots_adjust(hspace=0.8, wspace=0.4)

total_plots = levels.shape[0]
i = 0
for level in levels.keys():
    ax = fig.add_subplot(total_plots, 2, i+1)
    ax.set_title(level)
    
    top = gp.get_group(level)["region"].value_counts()
    _ = sns.barplot(x=top.keys(), y=top.values, color=color, ax=ax)
    
    i += 1
dataset[dataset.gender == 'M'].educational_level.value_counts()
plt.figure(figsize=(15,8))

chart = sns.countplot(x="educational_level", hue="gender", data=data_new)

_ = chart.set_xticklabels(chart.get_xticklabels(), rotation=45, horizontalalignment='right')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1770324771810656, 'fraction_numerical': 0.02398641477393335, 'mean_word_length': 5.7019914651493595, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '222539', 'license': 'MIT', 'n_tokens_mistral': 3868, 'n_tokens_neox': 3656, 'n_words': 556, 'path': 'notebooks/(abono) second study and pre processing.ipynb', 'repo_name': 'WagnerFLL/UFAL-DS', 'size': '9422'}