<jupyter_start><jupyter_text>Parte 1: Árvore de Decisão  Parâmetros editáveis<jupyter_code># Arquivo de dados
file_name = './accident_data.csv'
#file_name = './debug.csv'

# Tamanho do set de treino
training_ratio = 0.8<jupyter_output><empty_output><jupyter_text>Antes de executar este caderno, pode ser necessário instalar previamente as dependências em seu sistema. Isso pode ser feito com os seguintes comandos:```bashpip install --user numpypip install --user pandaspip install seaborn```  Dependencias<jupyter_code>import numpy as np
import pandas as pd
import seaborn as sn
import matplotlib.pyplot as plt

from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import confusion_matrix<jupyter_output><empty_output><jupyter_text>Importando os dados<jupyter_code># Import
dataset = pd.read_csv(file_name, sep = ",")

# Cleaning
# dataset = dataset[]

# Printing
dataset<jupyter_output><empty_output><jupyter_text>Pré-Processamento<jupyter_code>def enumeracao(valores):
    mapping = {}
    translating = {}
    cont = 0
    for val in valores:
        if val not in mapping:
            mapping[val] = cont
            translating[cont] = val
            cont += 1
    return mapping, translating, cont
translators = {}
sizes = {}

for col in dataset.columns:
    mapping, translating, size = enumeracao(dataset[col])
    dataset[col] = dataset[col].map(mapping)
    
    translators[col] = translating
    sizes[col] = size

dataset<jupyter_output><empty_output><jupyter_text>Separando em treino e teste<jupyter_code># Treino
#train = dataset.sample(frac=training_ratio, random_state=21)
train = dataset.sample(frac=training_ratio)
# Teste
test = dataset.drop(train.index)<jupyter_output><empty_output><jupyter_text>Estrutura de Dados<jupyter_code>from abc import ABC

class DesisionTree(ABC):
    def inferencia(self, value):
        raise NotImplementedError()

class Node(DesisionTree):
    def __init__(self, test):
        self.test = test
        self.trees = np.empty((sizes[test],), dtype=object)
        self.num_trees = 0

    def add_tree(self, tree, idx):
        self.trees[idx] = tree
        self.num_trees += 1

    def inferencia(self, value):
        val = value[self.test]
        return self.trees[val].inferencia(value)

class Leaf(DesisionTree):
    def __init__(self, label):
        self.label = label

    def inferencia(self, value):
        return self.label<jupyter_output><empty_output><jupyter_text>Funções Auxiliares<jupyter_code>class_name = 'Accident Level'
quant_classes = sizes[class_name]

def distribuicao(valores):
    dist = {}
    for val in valores:
        if val in dist:
            dist[val] += 1
        else:
            dist[val] = 1
    return dist

def valor_maioria(dist_valores):
    max_quant = -1 
    max_class = ''
    for item in dist_valores.items():
        if item[1] > max_quant:
            max_quant = item[1]
            max_class = item[0]
    return max_class

def calcula_entropia(exemplos):
    probabilities = np.zeros((quant_classes))

    total_number = len(exemplos)
    for ex in exemplos[class_name]:
        probabilities[ex] += 1
    
    old_settings = np.seterr(all='ignore')  # Disable warnings
    probabilities /= total_number
    
    entropy = (-probabilities) * (np.log2(probabilities))
    np.seterr(**old_settings) # Enable warnings back
    return np.nan_to_num(entropy).sum()

def melhor_atributo(atributos, exemplos):
    total_number = len(exemplos)
    entropia = calcula_entropia(exemplos)

    max_ganho = -np.inf
    max_atrib = ''
    for atrib in atributos:
        ganho = entropia
        for val in range(sizes[atrib]):
            filterring = exemplos[atrib] == val
            exemplos_com = exemplos[filterring]

            ganho -= (len(exemplos_com) / total_number) * calcula_entropia(exemplos_com)
        if ganho > max_ganho:
            max_ganho = ganho
            max_atrib = atrib
    return max_atrib<jupyter_output><empty_output><jupyter_text>Algoritmo de Treinamento<jupyter_code>def treinamento(exemplos, atributos, padrao):
    if len(exemplos) == 0:
        return Leaf(padrao)

    dist = distribuicao(exemplos[class_name])
    if len(dist) == 1:
        return Leaf(list(dist.keys())[0])

    if len(atributos) == 0:
        return Leaf(valor_maioria(dist))
    
    melhor = melhor_atributo(atributos, exemplos)
    arvore = Node(melhor)
    maioria = valor_maioria(dist)

    for val in range(sizes[melhor]):
        filterring = exemplos[melhor] == val
        exemplos_i = exemplos[filterring]

        atributos_i = atributos.copy()
        atributos_i.discard(melhor)
        
        subarvore = treinamento(exemplos_i, atributos_i, maioria)
        arvore.add_tree(subarvore, val)
    return arvore
# Armazena o nome dos atributos em um conjunto
atributos = set()
for col in train.columns:
    atributos.add(col)

atributos.remove(class_name)

# Realiza o treinamento
solution = treinamento(train, atributos, '')<jupyter_output><empty_output><jupyter_text>Visualizando algumas respostas<jupyter_code># Visualizando algumas respostas
num_views = 5
for i in range(num_views):
    real = test.iloc[i][class_name]
    predicted = solution.inferencia(test.iloc[i])
    print('---------------------------')
    print('Real = ', translators[class_name][real], '| Predicted = ', translators[class_name][predicted])<jupyter_output>---------------------------
Real =  I | Predicted =  I
---------------------------
Real =  I | Predicted =  I
---------------------------
Real =  I | Predicted =  I
---------------------------
Real =  I | Predicted =  I
---------------------------
Real =  II | Predicted =  I<jupyter_text>Calculando a taxa de acerto no fração do dataset reservada para teste<jupyter_code>count_acertos = 0
for i in range(len(test)):
    real = test.iloc[i][class_name]
    predicted = solution.inferencia(test.iloc[i])
    if real == predicted:
        count_acertos += 1

print('Porcentagem de acertos = ', count_acertos / len(test) * 100, '%')<jupyter_output>Porcentagem de acertos =  61.36363636363637 %<jupyter_text>Matriz de confusão<jupyter_code>y_true = []
y_pred = []
for i in range(len(test)):
    y_true.append(test.iloc[i][class_name])
    y_pred.append(solution.inferencia(test.iloc[i]))

cm = confusion_matrix(y_true, y_pred)

df_cm = pd.DataFrame(cm, ['I', 'II', 'III', 'IV', 'V'], ['I', 'II', 'III', 'IV', 'V'])
sn.set(font_scale=1.4) # for label size
sn.heatmap(df_cm, annot=True, annot_kws={"size": 15}) # font size

plt.xlabel('Estimado', fontsize = 15)
plt.ylabel('Real', fontsize = 15)
plt.title('Classificador em arvore')
plt.show()
print("Estatistica Kappa: ", cohen_kappa_score(y_true, y_pred))<jupyter_output><empty_output><jupyter_text>--- Parte 2: Classificador à priori  Classificador por Média<jupyter_code># Classificador utilizando a média
media = 0
for i in range(len(train)):
    media += train.iloc[i][class_name]
class_media = round(media / len(train))

# Visualizando algumas respostas
num_views = 5
for i in range(num_views):
    real = test.iloc[i][class_name]
    predicted = class_media
    print('---------------------------')
    print('Real = ', translators[class_name][real], '| Predicted = ', translators[class_name][predicted])<jupyter_output>---------------------------
Real =  I | Predicted =  IV
---------------------------
Real =  I | Predicted =  IV
---------------------------
Real =  I | Predicted =  IV
---------------------------
Real =  I | Predicted =  IV
---------------------------
Real =  II | Predicted =  IV<jupyter_text>Desempenho<jupyter_code>count_acertos = 0
for i in range(len(test)):
    real = test.iloc[i][class_name]
    if real == class_media:
        count_acertos += 1

print('Porcentagem de acertos = ', count_acertos / len(test) * 100, '%')<jupyter_output>Porcentagem de acertos =  5.681818181818182 %<jupyter_text>Matriz de confusao e Estatística Kappa<jupyter_code>y_true = []
y_pred = []
for i in range(len(test)):
    y_true.append(test.iloc[i][class_name])
    y_pred.append(class_media)

cm = confusion_matrix(y_true, y_pred)

df_cm = pd.DataFrame(cm, ['I', 'II', 'III', 'IV', 'V'], ['I', 'II', 'III', 'IV', 'V'])
sn.set(font_scale=1.4) # for label size
sn.heatmap(df_cm, annot=True, annot_kws={"size": 15}) # font size

plt.xlabel('Estimado', fontsize = 15)
plt.ylabel('Real', fontsize = 15)
plt.title('Classificador em media')
plt.show()
print("Estatistica Kappa: ", cohen_kappa_score(y_true, y_pred))<jupyter_output><empty_output><jupyter_text>Classificador por Moda<jupyter_code># Classificador utilizando a moda
moda = {}
for i in range(len(train)):
    if train.iloc[i][class_name] in moda:
        moda[train.iloc[i][class_name]] += 1
    else:
        moda[train.iloc[i][class_name]] = 1

value_moda = -1
class_moda = -1
for key in moda:
    if moda[key] > value_moda:
        value_moda = moda[key]
        class_moda = key<jupyter_output><empty_output><jupyter_text>Desempenho<jupyter_code>count_acertos = 0
for i in range(len(test)):
    real = test.iloc[i][class_name]
    if real == class_moda:
        count_acertos += 1

print('Porcentagem de acertos = ', count_acertos / len(test) * 100, '%')<jupyter_output>Porcentagem de acertos =  71.5909090909091 %<jupyter_text>Matriz de Confusão e Estatística Kappa<jupyter_code>y_true = []
y_pred = []
for i in range(len(test)):
    y_true.append(test.iloc[i][class_name])
    y_pred.append(class_moda)

cm = confusion_matrix(y_true, y_pred)

df_cm = pd.DataFrame(cm, ['I', 'II', 'III', 'IV', 'V'], ['I', 'II', 'III', 'IV', 'V'])
sn.set(font_scale=1.4) # for label size
sn.heatmap(df_cm, annot=True, annot_kws={"size": 15}) # font size

plt.xlabel('Estimado', fontsize = 15)
plt.ylabel('Real', fontsize = 15)
plt.title('Classificador em moda')
plt.show()
print("Estatistica Kappa: ", cohen_kappa_score(y_true, y_pred))<jupyter_output><empty_output><jupyter_text>Parte3: Refazendo a análise desconsiderando alguns dados  Desconsiderando as datas e os países<jupyter_code>dataset.drop('Countries', axis=1,inplace=True)
# dataset.drop('Employee ou Terceiro', axis=1,inplace=True)
dataset.drop('Data', axis=1,inplace=True)
dataset<jupyter_output><empty_output><jupyter_text>Separa os dados de treino e de teste<jupyter_code># Treino
#train = dataset.sample(frac=training_ratio, random_state=21)
train = dataset.sample(frac=training_ratio)
# Teste
test = dataset.drop(train.index)<jupyter_output><empty_output><jupyter_text>Treinando<jupyter_code># Armazena o nome dos atributos em um conjunto
atributos = set()
for col in train.columns:
    atributos.add(col)

atributos.remove(class_name)

# Realiza o treinamento
solution = treinamento(train, atributos, '')<jupyter_output><empty_output><jupyter_text>Taxa de Acerto<jupyter_code>count_acertos = 0
for i in range(len(test)):
    real = test.iloc[i][class_name]
    predicted = solution.inferencia(test.iloc[i])
    if real == predicted:
        count_acertos += 1

print('Porcentagem de acertos = ', count_acertos / len(test) * 100, '%')<jupyter_output>Porcentagem de acertos =  81.81818181818183 %<jupyter_text>Matriz de Confusão e Estatística Kappa<jupyter_code>y_true = []
y_pred = []
for i in range(len(test)):
    y_true.append(test.iloc[i][class_name])
    y_pred.append(solution.inferencia(test.iloc[i]))

cm = confusion_matrix(y_true, y_pred)

df_cm = pd.DataFrame(cm, ['I', 'II', 'III', 'IV', 'V'], ['I', 'II', 'III', 'IV', 'V'])
sn.set(font_scale=1.4) # for label size
sn.heatmap(df_cm, annot=True, annot_kws={"size": 15}) # font size

plt.xlabel('Estimado', fontsize = 15)
plt.ylabel('Real', fontsize = 15)
plt.title('Classificador em arvore')
plt.show()
print("Estatistica Kappa: ", cohen_kappa_score(y_true, y_pred))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15285970761733778, 'fraction_numerical': 0.012823800974608874, 'mean_word_length': 3.823917525773196, 'pattern_counts': {'":': 4, '<': 83, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '226884', 'license': 'MIT', 'n_tokens_mistral': 4428, 'n_tokens_neox': 4254, 'n_words': 874, 'path': 'Projeto3-Aprendizado-de-Maquina/decision-tree.ipynb', 'repo_name': 'gitoso/ctc-17', 'size': '11697'}