<commit_before>#!/usr/bin/env python
from __future__ import print_function

import unicodecsv as csv
import argparse
import panphon
import Levenshtein
import munkres
from functools import partial


def levenshtein_dist(_, a, b):
    return Levenshtein.distance(a, b)


def feature_hamming_dist(ft, a, b):
    return ft.feature_edit_distance(ft.word_to_vector_list(a),
                                    ft.word_to_vector_list(b))


def feature_weighted_dist(ft, a, b):
    return ft.weighted_feature_edit_distance(ft.word_to_vector_list(a),
                                             ft.word_to_vector_list(b))


def construct_cost_matrix(words_a, words_b, dist):
    def matrix_row(word_a, words_b):
        return [dist(word_a, word_b) for (word_b, _) in words_b]
    return [matrix_row(word_a, words_b) for (word_a, _) in words_a]


def score(indices):
    pairs, errors = 0, 0
    for row, column in indices:
        pairs += 1
        if row != column:
            errors += 1
    return pairs, errors


def main(wordlist1, wordlist2, dist):
    with open(wordlist1, 'rb') as file_a, open(wordlist2, 'rb') as file_b:
        reader_a = csv.reader(file_a, encoding='utf-8')
        reader_b = csv.reader(file_b, encoding='utf-8')
        print('Reading word lists...')
        words_a = [(w, g) for (g, w) in reader_a]
        words_b = [(w, g) for (g, w) in reader_b]
        print('Constructing cost matrix...')
        matrix = construct_cost_matrix(words_a, words_b, dist)
        m = munkres.Munkres()
        print('Computing matrix using Hungarian Algorithm...')
        indices = m.compute(matrix)
        print(score(indices))
        print('Done.')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(usage='Align two lists of "cognates" using a specified distance metric.')
    parser.add_argument('wordlists', nargs=2, help='Filenames of two wordlists in corresponding order.')
    parser.add_argument('-d', '--dist', default='hamming', help='Distance metric (e.g. Hamming).')
    args = parser.parse_args()
    dists = {'levenshtein': levenshtein_dist,
             'hamming': feature_hamming_dist,
             'weighted': feature_weighted_dist}
    ft = panphon.FeatureTable()
    dist = partial(dists[args.dist], ft)<commit_msg>Added dogol_leven mode<commit_after>#!/usr/bin/env python
from __future__ import print_function

import unicodecsv as csv
import argparse
import panphon
import Levenshtein
import munkres
from functools import partial


def levenshtein_dist(_, a, b):
    return Levenshtein.distance(a, b)


def dogol_leven_dist(_, a, b):
    return Levenshtein.distance(ft.map_to_dogol_prime(a),
                                ft.map_to_dogol_prime(b))


def feature_hamming_dist(ft, a, b):
    return ft.feature_edit_distance(ft.word_to_vector_list(a),
                                    ft.word_to_vector_list(b))


def feature_weighted_dist(ft, a, b):
    return ft.weighted_feature_edit_distance(ft.word_to_vector_list(a),
                                             ft.word_to_vector_list(b))


def construct_cost_matrix(words_a, words_b, dist):
    def matrix_row(word_a, words_b):
        return [dist(word_a, word_b) for (word_b, _) in words_b]
    return [matrix_row(word_a, words_b) for (word_a, _) in words_a]


def score(indices):
    pairs, errors = 0, 0
    for row, column in indices:
        pairs += 1
        if row != column:
            errors += 1
    return pairs, errors


def main(wordlist1, wordlist2, dist):
    with open(wordlist1, 'rb') as file_a, open(wordlist2, 'rb') as file_b:
        reader_a = csv.reader(file_a, encoding='utf-8')
        reader_b = csv.reader(file_b, encoding='utf-8')
        print('Reading word lists...')
        words_a = [(w, g) for (g, w) in reader_a]
        words_b = [(w, g) for (g, w) in reader_b]
        print('Constructing cost matrix...')
        matrix = construct_cost_matrix(words_a, words_b, dist)
        m = munkres.Munkres()
        print('Computing matrix using Hungarian Algorithm...')
        indices = m.compute(matrix)
        print(score(indices))
        print('Done.')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(usage='Align two lists of "cognates" using a specified distance metric.')
    parser.add_argument('wordlists', nargs=2, help='Filenames of two wordlists in corresponding order.')
    parser.add_argument('-d', '--dist', default='hamming', help='Distance metric (e.g. Hamming).')
    args = parser.parse_args()
    dists = {'levenshtein': levenshtein_dist,
             'dogol_leven': dogol_leven_dist,
             'hamming': feature_hamming_dist,
             'weighted': feature_weighted_dist}
    ft = panphon.FeatureTable()
    dist = partial(dists[args.dist], ft)
{'fraction_non_alphanumeric': 0.11554710604140261, 'fraction_numerical': 0.004647232784114913, 'mean_word_length': 2.9856902356902357, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3197329', 'n_tokens_mistral': 1608, 'n_tokens_neox': 1541, 'n_words': 380}