<jupyter_start><jupyter_text>GrupoNomes: <NAME>, <NAME>  Ra: , 216179<jupyter_code>import math
import numpy as np
import json
import random
import matplotlib.pyplot as plt
import copy
from sklearn import decomposition
density = 5
density2 = 15
density3 = 20
epsilon = 0.01
epsilon2 = 0.005
epsilon3 = 0.02
def getDataset(dataset):
    cleanDataset = {
        "matrix" : [],
        "cluster": [],
    }
    for i in range(len(dataset)):
        point = getPoint(dataset[i],i)
        cleanDataset["matrix"].append(point)
        cleanDataset["cluster"].append(dataset[i]["cluster"])
    return cleanDataset
def getPoint(oPoint,t):
    point = {
        "coord" : [],
        "neigh" : [],
        "cluster" : -2,
        "pos" : t
    }
    for i in oPoint:
        if i != "cluster":
            point["coord"].append(oPoint[i])
    point["coord"] = np.array(point["coord"])
    return point
def normalize(matrix):
    lists = []
    for i in range(len(matrix[0]["coord"])):
        lists.append(np.array(getList(matrix,i)))
    for i in range(len(lists)):
        norm = np.linalg.norm(lists[i])
        lists[i] = lists[i]/norm
    for i in range(len(lists[0])):
        for j in range(len(lists)):
            matrix[i]["coord"][j] = lists[j][i]
    return matrix
def getList(matrix,t):
    #print(matrix[0])
    array = []
    for i in matrix:
        array.append(i["coord"][t])
    return array
def getDistance(point1,point2):
    distance = np.linalg.norm(point1 - point2)
    return distance
def dbscan(dataset,den,ep):
        corePoints = getNeighs(dataset["matrix"],ep,den)
        #dfs(dataset["matrix"],dataset["cluster"],den)
        getCluster(dataset["matrix"],dataset["cluster"],corePoints)
def getNeighs(dataset,ep,den):
    corePoints = []
    for i in range(len(dataset)):
        for j in range(i+1,len(dataset)):
            if getDistance(dataset[i]["coord"],dataset[j]["coord"]) < ep:
                dataset[i]["neigh"].append(j)
                dataset[j]["neigh"].append(i)
        if (len(dataset[i]["neigh"]) + 1) >= den:
            corePoints.append(i)
    return corePoints
def getCluster(matrix,clusters,corePoints):
    sumCluster = -1
    for i in corePoints:
        if matrix[i]["cluster"] == -2:
            sumCluster += 1
            matrix[i]["cluster"] = sumCluster
        clusters[i] = matrix[i]["cluster"]
        for j in matrix[i]["neigh"]:
            if matrix[j]["cluster"] > -2:
                continue
            matrix[j]["cluster"] = matrix[i]["cluster"]
            clusters[j] = matrix[i]["cluster"]
            
def dfs(matrix,clusters,den):
    sumCluster = -1
    for i in range(len(matrix)):
        if matrix[i]["cluster"] != -2:
            continue
        countNeigh = len(matrix[i]["neigh"])
        if countNeigh < den:
            matrix[i]["cluster"] = -1
            continue
        sumCluster += 1
        clusters[i] = sumCluster
        matrix[i]["cluster"] = sumCluster
        dfsColor(matrix,i,clusters)
def dfsColor(matrix,i,clusters):
    for j in matrix[i]["neigh"]:
        if matrix[j]["cluster"] > -1:
            continue
        else:
            matrix[j]["cluster"] = matrix[i]["cluster"]
            clusters[j] = matrix[i]["cluster"]
            dfsColor(matrix,j,clusters)
def cleanOutliers(dataset):
    clean = []
    for i in range(len(dataset["cluster"])):
        if(dataset["cluster"][i] <= -1):
            clean.append(i)
    for i in range(len(clean)):
        del dataset["cluster"][clean[i]-i]
        del dataset["matrix"][clean[i]-i]
def print2d(dataset,path):
    #print(getList(dataset["matrix"],0))
    #print(getList(dataset["matrix"],1))
    plt.scatter(getList(dataset["matrix"],0),getList(dataset["matrix"],1),c = dataset["cluster"])
    plt.title("clusteredData")
    plt.xlabel("x")
    plt.ylabel("y")
    plt.grid(True)
    plt.savefig(path)
def printCount(dataset,path):
    clusterCount = []
    manyCluster = getManyCluster(dataset["cluster"])
    for i in manyCluster:
        clusterCount.append(getClusterCount(dataset["matrix"],i))
    print(manyCluster)
    print(clusterCount)
    plt.bar(manyCluster,clusterCount)
    plt.title("Clusters length")
    plt.xlabel("Cluster")
    plt.ylabel("Number of elements in cluster")
    plt.grid(True)
    plt.savefig(path)
def getClusterCount(matrix,k):
    summ = 0
    for i in matrix:
        if i["cluster"] == k:
            summ += 1
    return summ
def getManyCluster(clusters):
    many = []
    for i in clusters:
        if len(many) == 0:
            many.append(i)
            continue
        tam = len(many) - 1
        k = binarySearch(i,many,0,tam)
        if many[k] == i:
            continue
        if many[k] > i:
            many.insert(k,i)
        if many[k] < i:
            if k == tam:
                many.append(i)
            else:
                many.insert(k+1,i)
    return many
def binarySearch(element,array,ini,end):
    if ini >= end:
        return ini
    mid = math.floor((ini+end)/2)
    if array[mid] == element:
        return mid
    elif array[mid] < element:
        return binarySearch(element,array,mid+1,end)
    else:
        return binarySearch(element,array,ini,mid)
def printDbScan(dataset,den,ep,printable,path):
    dbscan(dataset,den,ep)
    cleanOutliers(dataset)
    printable(dataset,path)
def getPoints(matrix):
    points = []
    for i in matrix:
        points.append(i["coord"])
    return np.array(points)
def restore(points,dataset):
    newDataset = copy.deepcopy(dataset)
    for i in range(len(newDataset["matrix"])):
        newDataset["matrix"][i]["coord"] = points[i]
    return newDataset
exist = True
try:
    f = open("../data/teacherData.json","r")
    stringSet = f.read()
    f.close()
except:
    print("Arquivo não existe")
    exist = False
if(exist):
    dataset = json.loads(stringSet)
    nDataset = getDataset(dataset)
    print2d(nDataset,"../images/originalData1.jpg")
    #plt.scatter(getList(nDataset["matrix"],0),getList(nDataset["matrix"],1))
    #plt.title("Original Data")
    #plt.xlabel("x")
    #plt.ylabel("y")
    #plt.grid(True)
    #plt.savefig("../images/originalData1.png")
    nDataset["matrix"] = normalize(nDataset["matrix"])
    #for i in nDataset["matrix"]:
       # print(i["coord"])
print2d(nDataset,"../images/test.jpg")
printDbScan(nDataset,density,epsilon,print2d,"../images/dbscanClustered1.jpg")
#print(nDataset["matrix"])
#for i in nDataset["matrix"]:
    #print(i["coord"])
exist = True
try:
    f = open("../data/strokeData.json","r")
    stringSet2 = f.read()
    f.close()
except:
    print("Arquivo não existe")
    exist = False
if(exist):
    dataset2 = json.loads(stringSet2)
    nDataset2 = getDataset(dataset2)
    nDataset2["matrix"] = normalize(nDataset2["matrix"])
printDbScan(nDataset2,density3,epsilon3,printCount,"../images/dbscanCount2_new.jpg")
printDbScan(nDataset2,60,epsilon3,printCount,"../images/dbscanCount2_60_002.jpg")
results1 = []
for i in range(10,101,10):
        tDataset = copy.deepcopy(nDataset2)
        dbscan(tDataset,i,epsilon3)
        cleanOutliers(tDataset)
        results1.append(tDataset)
printCount(results0[0],"../images/dbscanCount2_10_005.jpg")
printCount(results0[1],"../images/dbscanCount2_20_005.jpg")
printCount(results0[2],"../images/dbscanCount2_30_005.jpg")
printCount(results0[3],"../images/dbscanCount2_40_005.jpg")
printCount(results0[4],"../images/dbscanCount2_50_005.jpg")
printCount(results0[5],"../images/dbscanCount2_60_005.jpg")
printCount(results0[6],"../images/dbscanCount2_70_005.jpg")
printCount(results0[7],"../images/dbscanCount2_80_005.jpg")
printCount(results0[8],"../images/dbscanCount2_90_005.jpg")
printCount(results0[9],"../images/dbscanCount2_100_005.jpg")
results0 = []
for i in range(10,101,10):
        tDataset = copy.deepcopy(nDataset2)
        dbscan(tDataset,i,0.05)
        cleanOutliers(tDataset)
        results0.append(tDataset)
printCount(results1[0],"../images/dbscanCount2_10_002.jpg")
printCount(results1[0],"../images/dbscanCount2_10_002.jpg")
printCount(results1[0],"../images/dbscanCount2_10_002.jpg")
printCount(results1[0],"../images/dbscanCount2_10_002.jpg")
printCount(results1[0],"../images/dbscanCount2_10_002.jpg")
printCount(results1[0],"../images/dbscanCount2_10_002.jpg")
printCount(results1[0],"../images/dbscanCount2_10_002.jpg")
printCount(results1[0],"../images/dbscanCount2_10_002.jpg")
printCount(results1[0],"../images/dbscanCount2_10_002.jpg")
printCount(results1[0],"../images/dbscanCount2_10_002.jpg")
printDbScan(nDataset2,density3,0.05,printCount,"../images/dbscanCount2_new1.jpg")
points = getPoints(nDataset2["matrix"])
print(points[0])
pca = decomposition.PCA(n_components=2)
pca.fit(points)
pcPoints = pca.transform(points)
print(pcPoints[0])
pcaDataset = restore(pcPoints,nDataset2)
print2d(pcaDataset,"../images/pcaDatasetD.jpg")
print(pcaDataset["matrix"][0])
printDbScan(pcaDataset,density2,epsilon2,print2d,"../images/dbscanPCA1_0005_20.jpg")
printDbScan(pcaDataset,density2,epsilon2,print2d,"../images/dbscanPCA1_0005_15.jpg")
printDbScan(pcaDataset,18,0.008,print2d,"../images/dbscanPCA1_0008_18.jpg")
printDbScan(pcaDataset,density2,0.05,print2d,"../images/dbscanPCA2.jpg")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16389883859763377, 'fraction_numerical': 0.03538478237273418, 'mean_word_length': 3.431938431938432, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '651309', 'license': 'MIT', 'n_tokens_mistral': 3294, 'n_tokens_neox': 3208, 'n_words': 494, 'path': 'project1_ra216179/Notebooks/.ipynb_checkpoints/dbscan_ra216179-checkpoint.ipynb', 'repo_name': 'batataazul/projects_MachineLearning', 'size': '9245'}