<commit_before>import json
import pkgutil


def levenshtein(s1, s2):
    l1 = len(s1)
    l2 = len(s2)

    matrix = [range(l1 + 1)] * (l2 + 1)

    for zz in range(l2 + 1):
        matrix[zz] = range(zz, zz + l1 + 1)

    for zz in range(l2):
        for sz in range(l1):
            if s1[sz] == s2[zz]:
                matrix[zz + 1][sz + 1] = min(matrix[zz + 1][sz] + 1, matrix[zz][sz + 1] + 1, matrix[zz][sz])
            else:
                matrix[zz + 1][sz + 1] = min(matrix[zz + 1][sz] + 1, matrix[zz][sz + 1] + 1, matrix[zz][sz] + 1)

    return matrix[l2][l1]


def classify_is_sus(database_dict, new_url):
    sus_sum = 0
    notsus_sum = 0
    sind = 0
    nind = 0

    for k, v in database_dict.items():
        dist = levenshtein(new_url, k)
        if v == 1:
            sus_sum = sus_sum + dist
            sind = sind + 1
        else:
            notsus_sum = notsus_sum + dist
            nind = nind + 1

    avg_sus = sus_sum / sind
    avg_notsus = notsus_sum / nind
<commit_msg>separate out matrix part of levenshtein<commit_after>import json
import pkgutil


def levenshtein_matrix(s1, s2):
    l1 = len(s1)
    l2 = len(s2)

    matrix = [range(l1 + 1)] * (l2 + 1)

    for zz in range(l2 + 1):
        matrix[zz] = range(zz, zz + l1 + 1)

    for zz in range(l2):
        for sz in range(l1):
            if s1[sz] == s2[zz]:
                matrix[zz + 1][sz + 1] = min(matrix[zz + 1][sz] + 1, matrix[zz][sz + 1] + 1, matrix[zz][sz])
            else:
                matrix[zz + 1][sz + 1] = min(matrix[zz + 1][sz] + 1, matrix[zz][sz + 1] + 1, matrix[zz][sz] + 1)
    return matrix


def levenshtein(s1, s2):
    return levenshtein_matrix(s1, s2)[len(s2)][len(s1)]


def classify_is_sus(database_dict, new_url):
    sus_sum = 0
    notsus_sum = 0
    sind = 0
    nind = 0

    for k, v in database_dict.items():
        dist = levenshtein(new_url, k)
        if v == 1:
            sus_sum = sus_sum + dist
            sind = sind + 1
        else:
            notsus_sum = notsus_sum + dist
            nind = nind + 1

    avg_sus = sus_sum / sind
    avg_notsus = notsus_sum / nind

{'fraction_non_alphanumeric': 0.13827277017461068, 'fraction_numerical': 0.03964134025483719, 'mean_word_length': 1.9160935350756534, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '187248', 'n_tokens_mistral': 915, 'n_tokens_neox': 821, 'n_words': 215}