<commit_before>import matplotlib.pyplot as plt
import logging
import os

from random import sample
from django.utils import timezone
from apps.data.songs.models import Song
from apps.similarities.Cosine.algorithm.algorithm import CosineSimilarity
from apps.similarities.Cosine.benchmark.models import BenchCosine_SongTitle

logger = logging.getLogger(__name__)


def CosineBenchmark(allSongs):
    logger.info("[Start Bench Cosine]")
    startedAt = timezone.now()
    CosineSimilarity([song.title for song in allSongs])
    BenchCosine_SongTitle.objects.create(
        setSize=len(allSongs),
        started_at=startedAt,
        finished_at=timezone.now()
    )
    logger.info("[Finish Bench Cosine]")


def RunCosineBenchmark(size_list=[1500, 3000, 4500]):
    logger.info("[Start Run Bench Cosine]")
    songs_list = Song.objects.all()
    for runner in size_list:
        for i in range(10):
            logger.info(
                "\tTamanho do banco ("
                + str(runner)
                + ") Ciclo: "
                + str(i)
            )
            CosineBenchmark(allSongs=sample(set(songs_list), runner))
    logger.info("[Finish Run Bench Cosine]")
<commit_msg>adicionando chamada no cosine e globalizando variaveis<commit_after>import matplotlib.pyplot as plt
import logging
import os

from random import sample
from django.utils import timezone
from apps.CONSTANTS import COSINE_TOTAL_RUN, COSINE_SET_SIZE
from apps.data.songs.models import Song
from apps.similarities.Cosine.algorithm.algorithm import CosineSimilarity
from apps.similarities.Cosine.benchmark.models import BenchCosine_SongTitle

logger = logging.getLogger(__name__)


def CosineBenchmark(allSongs):
    logger.info("[Start Bench Cosine]")
    startedAt = timezone.now()
    similarityMatrix = CosineSimilarity([song.title for song in allSongs])
    finishedAt = timezone.now()
    similaritySum = 0.0
    countTotal = 0.0
    for line in range(len(allSongs)):
        for column in range(line, len(allSongs)):
            countTotal += 1.0
            similaritySum += similarityMatrix[line][column]
    BenchCosine_SongTitle.objects.create(
        setSize=len(allSongs),
        similarity=(similaritySum/countTotal),
        started_at=startedAt,
        finished_at=finishedAt
    )
    logger.info("[Finish Bench Cosine]")


def RunCosineBenchmark(size_list=COSINE_SET_SIZE):
    logger.info("[Start Run Bench Cosine]")
    songs_list = Song.objects.all()
    for runner in size_list:
        for i in range(COSINE_TOTAL_RUN):
            logger.info(
                "\tTamanho do banco ("
                + str(runner)
                + ") Ciclo: "
                + str(i)
            )
            CosineBenchmark(allSongs=sample(set(songs_list), runner))
    logger.info("[Finish Run Bench Cosine]")

{'fraction_non_alphanumeric': 0.09366096866096867, 'fraction_numerical': 0.007122507122507123, 'mean_word_length': 3.071014492753623, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3859417', 'n_tokens_mistral': 918, 'n_tokens_neox': 843, 'n_words': 190}