<jupyter_start><jupyter_text>Import<jupyter_code>import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split


#se vc usa tensorflow 2.0 use o import abaixo
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()

# se vc usa o tensorflow 1.x comente aslinha acima e descomente a linha que segue
#import tensorflow as tf


import matplotlib.pyplot as plt
import seaborn as sns<jupyter_output>WARNING:tensorflow:From c:\users\busson_windows\appdata\local\programs\python\python37\lib\site-packages\tensorflow_core\python\compat\v2_compat.py:65: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
Instructions for updating:
non-resource variables are not supported in the long term<jupyter_text>Dataset<jupyter_code>#dataset com atributos sobre nódulos, indica se é benigno ou maligno
table_ = pd.read_csv("../data/cancer.csv")


print("Todos os nomes dos atributos")
table_.info()


#lista de atributos que serão usados no x
dataset_x_features = ["mean radius", "mean texture", "mean perimeter", "mean area", "mean smoothness", "mean compactness", "mean concavity",
                     "mean concave points", "mean symmetry", "mean fractal dimension", "radius error", "texture error", "perimeter error", 
                      "area error", "smoothness error", "compactness error", "concavity error", "concave points error", "symmetry error", 
                      "fractal dimension error", "worst radius", "worst texture", "worst perimeter", "worst area", "worst smoothness", 
                      "worst compactness", "worst concavity", "worst concave points", "worst symmetry", "worst fractal dimension"]

#normalização dos atributos
table_[dataset_x_features] = table_[dataset_x_features].apply(lambda x:(x-x.min()) / (x.max()-x.min()))

display(table_)

#convertendo o dataframe pandas para ndarray (x e y)
dataset_x =  table_[dataset_x_features].to_numpy()
dataset_y =  table_['target'].to_numpy()

#obtendo a quantidade de classes
num_labels = table_['target'].nunique()<jupyter_output><empty_output><jupyter_text>Divisão entre treino e validação<jupyter_code>#90% para treino e 10% para teste
TRAIN_X, VALID_X = train_test_split(dataset_x, test_size=0.1, random_state=0, stratify=dataset_y)
TRAIN_Y, VALID_Y = train_test_split(dataset_y, test_size=0.1, random_state=0, stratify=dataset_y)

#shapes dos conjuntos de treino e validação
print(TRAIN_X.shape, TRAIN_Y.shape)
print(VALID_X.shape, VALID_Y.shape)<jupyter_output>(512, 30) (512,)
(57, 30) (57,)<jupyter_text>Usando um modelo de REDE NEURAL<jupyter_code>#a função recebe como parametro a quantidade de features e número de classes no problema
def build_net(n_features, n_classes):
    
    #Por praticidade os tensores são salvos neste dicionário
    Dic = {}
    
    # ---- INICIO DA REDE NEURAL ----
    
    # Placeholders (X (entrada) e Y (saída) da rede)
    placeholder_X = tf.placeholder(dtype=tf.float32, shape=[None, n_features])
    Dic["placeholder_X"] = placeholder_X
    
    placeholder_Y = tf.placeholder(dtype=tf.int64, shape=[None])
    Dic["placeholder_Y"] = placeholder_Y

    # camada escondida
    hidden_layer1 = tf.layers.dense(placeholder_X, 10, activation=tf.nn.relu)
    Dic["layer1"] = hidden_layer1
    
    # camada escondida
    hidden_layer2 = tf.layers.dense(hidden_layer1, 20, activation=tf.nn.relu)
    Dic["layer2"] = hidden_layer2
    
    # camada escondida
    hidden_layer3 = tf.layers.dense(hidden_layer2, 10, activation=tf.nn.relu)
    Dic["layer3"] = hidden_layer3
    
    #camada de saída
    out = tf.layers.dense(hidden_layer3, n_classes, name="output")
     
    # ---- FIM DA REDE NEURAL ----
    
    #adaptando o Label Y para o modelo One-Hot Label
    one_hot = tf.one_hot(placeholder_Y, depth=n_classes)
    
    # Função de perda/custo/erro
    loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot, logits=out) 
    Dic["loss"] = loss
    
    # Otimizador
    opt = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(loss)
    Dic["opt"] = opt
    
    #Softmax
    softmax = tf.nn.softmax(out)
    Dic["softmax"] = softmax
    
    #Classe
    class_ = tf.argmax(softmax,1)
    Dic["class"] = class_
    
    #Acurácia
    compare_prediction = tf.equal(class_, placeholder_Y)
    accuracy = tf.reduce_mean(tf.cast(compare_prediction, tf.float32))
    Dic["accuracy"] = accuracy
    
    return Dic
#Iniciando
sess = tf.InteractiveSession()

#obtendo o número de features 
n_features = dataset_x.shape[1]

#construindo o modelo de rede
Dic_cg = build_net(n_features, num_labels)

#inicializando as variveis do tensorflow
sess.run(tf.global_variables_initializer())
#definindo o número de épocas
epochs = 1000

historic_train_error = []
historic_train_acc = []
historic_valid_acc = []

for i in range(epochs):
    
    sess.run(Dic_cg["opt"], feed_dict={Dic_cg["placeholder_X"]: TRAIN_X, Dic_cg["placeholder_Y"]: TRAIN_Y})
    
    # a cada 100 épocas o erro é impresso
    if  i % 100 == 0:
        erro_train = sess.run(Dic_cg["loss"], feed_dict={Dic_cg["placeholder_X"]: TRAIN_X, Dic_cg["placeholder_Y"]: TRAIN_Y})
        print("O erro na época", i,"é", erro_train)
        historic_train_error.append(erro_train)
        
        
        #após o fim do treino, é calculada a acurácia
        acc = sess.run(Dic_cg["accuracy"], feed_dict={Dic_cg["placeholder_X"]: TRAIN_X, Dic_cg["placeholder_Y"]: TRAIN_Y})
        print("A accurácia do treino é:", acc)
        historic_train_acc.append(acc)
        
        
        acc = sess.run(Dic_cg["accuracy"], feed_dict={Dic_cg["placeholder_X"]: VALID_X, Dic_cg["placeholder_Y"]: VALID_Y})
        print("A accurácia da validacao é:", acc)
        historic_valid_acc.append(acc)
        

        #após o fim do treino, é calculada a acurácia



print("erro train")
sns.relplot(data=historic_train_error, kind="line",  palette="cool", aspect=2)


plt.pause(0.1)
print("acc train")
sns.relplot(data=historic_train_acc, kind="line",  palette="cool", aspect=2)

plt.pause(0.1)
print("acc valid")
sns.relplot(data=historic_valid_acc, kind="line",  palette="cool", aspect=2)<jupyter_output>O erro na época 0 é 0.7077857
A accurácia do treino é: 0.37304688
A accurácia da validacao é: 0.36842105
O erro na época 100 é 0.13853297
A accurácia do treino é: 0.9628906
A accurácia da validacao é: 0.9649123
O erro na época 200 é 0.07960736
A accurácia do treino é: 0.98046875
A accurácia da validacao é: 0.9649123
O erro na época 300 é 0.06413698
A accurácia do treino é: 0.98046875
A accurácia da validacao é: 0.98245615
O erro na época 400 é 0.056469984
A accurácia do treino é: 0.9824219
A accurácia da validacao é: 0.98245615
O erro na época 500 é 0.051943105
A accurácia do treino é: 0.9824219
A accurácia da validacao é: 0.98245615
O erro na época 600 é 0.048501905
A accurácia do treino é: 0.98828125
A accurácia da validacao é: 0.98245615
O erro na época 700 é 0.048325542
A accurácia do treino é: 0.9902344
A accurácia da validacao é: 0.98245615
O erro na época 800 é 0.046993375
A accurácia do treino é: 0.9902344
A accurácia da validacao é: 0.98245615
O erro na época 900 é 0.0460212
A[...]<jupyter_text>Usando o modelo KNN<jupyter_code>from sklearn.neighbors import KNeighborsClassifier

logreg = KNeighborsClassifier(n_neighbors=20)
#treino
logreg.fit(TRAIN_X, TRAIN_Y)
#validação
logreg.score(VALID_X, VALID_Y)

## Usando uma
from sklearn.tree import DecisionTreeClassifier

clf = DecisionTreeClassifier()
#treino
clf.fit(TRAIN_X, TRAIN_Y)
#validação
clf.score(VALID_X, VALID_Y)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10601681555438781, 'fraction_numerical': 0.04703100367840252, 'mean_word_length': 4.257596685082873, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '588915', 'license': 'MIT', 'n_tokens_mistral': 3068, 'n_tokens_neox': 2792, 'n_words': 711, 'path': 'template_experiment/exemplo_em_aula.ipynb', 'repo_name': 'Busson/ComputacaoParaCienciaDeDados', 'size': '7612'}