<jupyter_start><jupyter_text>Uma análise de eficiencia entre *K-means* e *bisecting K-means* A idéia deste projeto é realizar um *benchmark* dos algoritmos K-means e *bisecting K-means* nos *DataSets* Iris e HertDesies.<jupyter_code>from sklearn import metrics
from time import time
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import math
import random as rd

filepath_trettel = '/home/trettel/Documents/projects/DATALAB/datasets/Iris_Data.csv'
filepath_eric = '/home/grayjack/MySources/WorkshopML/data/Iris_Data.csv'
filepath_marcelo = '/home/maru/Projects/ia_docs/Iris_Data.csv'
data = pd.read_csv(filepath_eric)
print(data.describe())
data.groupby('species').agg(['mean', 'median'])
data.groupby('species').agg([np.mean, np.median])
num = {val:i for i,val in enumerate(set(data.species))}
y_true = [num[item] for item in data.species]

data.plot.scatter(x="petal_length", y="petal_width", c=y_true, colormap='viridis')
def plotter(doc, ks, label):
    X,Y = zip(*doc)
    kx,ky = ks
    #print(ks)
    plt.scatter(x=X,y=Y, marker='o', c=label)
    plt.scatter(x=kx, y=ky, marker='x', color='black', s=150)

def plotter2(doc):
    X,Y = zip(*doc)
    plt.scatter(x=X,y=Y, marker='o')
def euclidianDistance(p1, p2):
    return math.sqrt( (p1[0] - p2[0])**2 + (p1[1] - p2[1])**2 )

MAX_ITER = 15
def goCondition(old_centroids, curr_centroids, itter):
    if itter >= MAX_ITER: return False    
    
    ctd = 0
    for old_centroid,curr_centroid in zip(old_centroids,curr_centroids):
        if abs(old_centroid[0]-curr_centroid[0]) <= 0.001 and\
           abs(old_centroid[1]-curr_centroid[1]) <= 0.001:
               ctd += 1
    if ctd == len(curr_centroids): return False
    
    return True
    
def definePatternship(S, centroids, dist_func):
    color = {cen:i for i,cen in enumerate(centroids)}
    labels = []
    
    for p in S:
        distances = [(dist_func(p, centroid),centroid) for centroid in centroids]
        
        closest_centroid = min(distances, key=lambda x: x[0])[1]
        
        labels.append(color[closest_centroid])
    
    return labels

def defineNewCentroids(S, labels):
    centroids = dict()
    for p,label in zip(S,labels):
        if label not in centroids:
            centroids[label] = {"x":p[0], "y":p[1], "len":1}
        else:
            centroids[label]["x"] += p[0]
            centroids[label]["y"] += p[1]
            centroids[label]["len"] += 1
 
        
    return [(c['x']/c['len'], c['y']/c['len']) for c in centroids.values()]


def kmeans(points, k, dist):
    l = len(points[0])
    c = [[],[]]
    sx = max(points[0]) - min(points[0])
    sy = max(points[1]) - min(points[1])
    for i in range(k):
        c[0].append(rd.uniform(min(points[0])-sx, max(points[0])+sx))
        c[1].append(rd.uniform(min(points[1])-sy, max(points[1])+sy))
    changed = False
    while(not changed):
        distc = []
        closestFlag = []
        cUpdaterX = []
        cUpdaterY = []
        for j in range(k):
            distc.append([])
            cUpdaterX.append([])
            cUpdaterY.append([])
            for i in range(l):
                distc[j].append(dist((c[0][j],c[1][j]),(points[0][i],points[1][i])))
                if(j>0):
                    if(distc[j][i]<distc[j-1][i]):
                        closestFlag[i] = j
                else:
                    closestFlag.append(0)
        for i in range(l):
            cUpdaterX[closestFlag[i]].append(points[0][i])
            cUpdaterY[closestFlag[i]].append(points[1][i])
        oldC = [c[0][:],c[1][:]]
        for i in range(k):
            c[0][i] = sum(cUpdaterX[i])/(len(cUpdaterX[i])+0.000000001)
            c[1][i] = sum(cUpdaterY[i])/(len(cUpdaterY[i])+0.000000001)
        changed = oldC==c
    
    distc = []
    closestFlag = []
    for j in range(k):
        distc.append([])
        for i in range(l):
            distc[j].append(dist((c[0][j],c[1][j]),(points[0][i],points[1][i])))
            if(j>0):
                if(distc[j][i]<distc[j-1][i]):
                    closestFlag[i] = j
            else:
                closestFlag.append(0)
    return closestFlag, c
document = [[pl, pw] for pl,pw in zip(list(data.petal_length), list(data.petal_width))]

best_accuracy = -1
best_centr = None
y_pred = []
best_label = []
best_k = 0
inicial = time()

# for _ in range(30):
for K in range(1, 50):
    y_pred, centroids = kmeans(list(zip(*document)), 50, euclidianDistance)

#     accuracy = metrics.adjusted_rand_score(y_true, y_pred)
    try:
       accuracy = metrics.v_measure_score(y_true, y_pred)
    except:
       print(y_pred)


    if accuracy >= best_accuracy:
        best_accuracy = accuracy
        best_centr = centroids
        best_k = K
        best_label = y_pred

final = time()
print(f"Melhor valor para o K:   {best_k}\nObtendo uma acurácia de: {best_accuracy} (v_measure_score)")
print(f"tempo: {final-inicial}")
plotter(document, best_centr, best_label)

doc =[[pl, pw] for pl,pw in zip(list(data.petal_length), list(data.petal_width))]
document = []

inicio = time()
for K in range(1, 20):
    y_pred, _ = kmeans(list(zip(*doc)), K, euclidianDistance)

#     accuracy = metrics.adjusted_rand_score(y_true, y_pred)
#     accuracy = metrics.v_measure_score(y_true, y_pred)
    try:
        accuracy = metrics.v_measure_score(y_true, y_pred)
    except:
        print(y_pred)

    document.append((K, accuracy))

final = time()

plotter2(document)

print(f"Time: {final-inicio}\n\nGráfico: K por Acurácia")<jupyter_output>Time: 0.232452392578125

Gráfico: K por Acurácia
{'fraction_non_alphanumeric': 0.15916023685627131, 'fraction_numerical': 0.02153238830073569, 'mean_word_length': 2.8951781970649897, 'pattern_counts': {'":': 3, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '640704', 'license': 'MIT', 'n_tokens_mistral': 2182, 'n_tokens_neox': 2122, 'n_words': 410, 'path': 'src/K-means-marcelo.ipynb', 'repo_name': 'GrayJack/Bisect-K-Means', 'size': '5573'}