<commit_before>#!/usr/bin/env python3
import argparse
from Bio import PDB, SeqIO, SeqRecord, Seq
from Bio.Data import CodonTable
from collections import Counter, defaultdict
from dna_features_viewer import BiopythonTranslator, CircularGraphicRecord, GraphicFeature, GraphicRecord
from itertools import chain, product
from matplotlib import pyplot as plt
import multiprocessing as mp
import os
from pathlib import Path
import sys
import textdistance as TD
from typing import Iterable, List

plt.style.use('ggplot')

PLOTDIR = 'plots'
if not os.path.exists(PLOTDIR):
    os.mkdir(os.path.join(os.curdir, PLOTDIR))

DATADIR = 'data'
if not os.path.exists(DATADIR):
    os.mkdir(os.path.join(os.curdir, DATADIR))

textdistfuncs = [
    'cosine',
    'damerau_levenshtein',
    'editex',
    'entropy_ncd',
    'gotoh',
    'hamming',
    'identity',
    'jaccard',
    'jaro',
    'jaro_winkler',
    'lcsseq',
    'lcsstr',
    'length',
    'levenshtein',
    'lzma_ncd',
    'matrix',
    'mlipns',
    'monge_elkan',
    'mra',
    'needleman_wunsch',
    'overlap',
    'postfix',
    'prefix',
    'ratcliff_obershelp',
    'rle_ncd',
    'smith_waterman',
    'sorensen',
    'sorensen_dice',
    'sqrt_ncd',
    'strcmp95',
    'tversky',
    'zlib_ncd',
]

def create_distance_matrix(pdbfile, quiet=False):
    derp = PDB.PDBParser(QUIET=quiet).get_structure('pdbfile', pdbfile)<commit_msg>calc lev distance, ration, cosine distance<commit_after>#!/usr/bin/env python3
import argparse
from Bio import PDB, SeqIO, SeqRecord, Seq
from Bio.Data import CodonTable
from collections import Counter, defaultdict
from dna_features_viewer import BiopythonTranslator, CircularGraphicRecord, GraphicFeature, GraphicRecord
from itertools import chain, product
from matplotlib import pyplot as plt
import multiprocessing as mp
import numpy as np
import os
from pathlib import Path
import sys
from sklearn.feature_extraction.text import TfidfVectorizer
import textdistance as TD
from typing import Iterable, List

plt.style.use('ggplot')

PLOTDIR = 'plots'
if not os.path.exists(PLOTDIR):
    os.mkdir(os.path.join(os.curdir, PLOTDIR))

DATADIR = 'data'
if not os.path.exists(DATADIR):
    os.mkdir(os.path.join(os.curdir, DATADIR))

textdistfuncs = [
    'cosine',
    'damerau_levenshtein',
    'editex',
    'entropy_ncd',
    'gotoh',
    'hamming',
    'identity',
    'jaccard',
    'jaro',
    'jaro_winkler',
    'lcsseq',
    'lcsstr',
    'length',
    'levenshtein',
    'lzma_ncd',
    'matrix',
    'mlipns',
    'monge_elkan',
    'mra',
    'needleman_wunsch',
    'overlap',
    'postfix',
    'prefix',
    'ratcliff_obershelp',
    'rle_ncd',
    'smith_waterman',
    'sorensen',
    'sorensen_dice',
    'sqrt_ncd',
    'strcmp95',
    'tversky',
    'zlib_ncd',
]

def lev_distance(str1, str2):
    # calculates levenshtein similarity between 2 strings
    xlen, ylen = len(str1) + 1, len(str2) + 1
    distmatrix = np.zeros((xlen, ylen))
    for x in range(xlen): distmatrix[x,0] = x
    for y in range(ylen): distmatrix[0,y] = y
    for x in range(1, xlen):
        for y in range(1, ylen):
            if str1[x-1] == str2[y-1]:
                distmatrix[x,y] = min(
                    distmatrix[x-1,y] + 1,
                    distmatrix[x-1,y-1],
                    distmatrix[x,y-1] + 1)
            else:
                distmatrix[x,y] = min(
                    distmatrix[x-1,y] + 1,
                    distmatrix[x-1,y-1] + 1,
                    distmatrix[x,y-1] + 1)
    return distmatrix[xlen-1, ylen-1]

def lev_ratio(str1, str2):
    # calculates levenshtein distance as a ratio of the maximum edit distance
    upper_bound = max(len(str1), len(str2))
    return 1 - lev_distance(str1, str2) / upper_bound

def tfidf_cosine_distance(str1, str2):
    # returns a float, a higher number is closer/better result
    vectors = TfidfVectorizer(min_df=1)
    tfidf = vectors.fit_transform([str1, str2])
    return (tfidf*tfidf.T).A[0,1]

def create_distance_matrix(pdbfile, quiet=False):
    derp = PDB.PDBParser(QUIET=quiet).get_structure('pdbfile', pdbfile)
{'fraction_non_alphanumeric': 0.12209444580376805, 'fraction_numerical': 0.012233912405187179, 'mean_word_length': 3.218782249742002, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4249438', 'n_tokens_mistral': 1576, 'n_tokens_neox': 1516, 'n_words': 341}