<jupyter_start><jupyter_text>BCG Gamma Challenge  Libraries<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from scipy import stats
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>df_municipios_2015 = pd.read_csv('../bcggammachallenge/municipios/municipios20150101.csv')
df_municipios_2016 = pd.read_csv('../bcggammachallenge/municipios/municipios20160101.csv')
df_municipios_2017 = pd.read_csv('../bcggammachallenge/municipios/municipios20170101.csv')
df_municipios_2015.shape
df_municipios_2016.shape
df_municipios_2017.shape
df = pd.concat([df_municipios_2015, df_municipios_2016, df_municipios_2017])
df.shape
df.head(10)
df['regiao'].value_counts()
df['ano_censo'].value_counts()
df[df['municipio'] == 'Jussara']
columns = [
    'ano_censo',
    'regiao',
    'unidade_federativa',
    'municipio',
    'num_escolas',
    'num_escolas_em_atividade',
    'num_professores',
    'num_estudantes',
    'num_funcionarios'    
]

df[columns].head()
df[columns].describe()
df.columns
df['cod_municipio'].dtypes
df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12956810631229235, 'fraction_numerical': 0.05647840531561462, 'mean_word_length': 7.087248322147651, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '59143', 'license': 'MIT', 'n_tokens_mistral': 529, 'n_tokens_neox': 470, 'n_words': 60, 'path': 'utils/municipios.ipynb', 'repo_name': 'alexandrerays/E-xray', 'size': '1204'}