<commit_before>import logging

from pymachine.src.machine import MachineGraph

class WordSimilarity():
    def __init__(self, wrapper):
        self.wrapper = wrapper

    def get_links(self, machine):
        self.seen = set()
        return self._get_links(machine)

    def _get_links(self, machine):
        if machine in self.seen:
            return
        self.seen.add(machine)
        for hypernym in machine.partitions[0]:
            name = hypernym.printname()
            if name.isupper():
                continue
            yield name
            for link in self.get_links(hypernym):
                yield link

        for link in self.get_binary_links(machine):
            yield link

    def get_binary_links(self, machine):
        for parent, partition in machine.parents:
            parent_pn = parent.printname()
            if not parent_pn.isupper() or partition == 0:
                continue
            if partition == 1:
                links = set(["{0} {1}".format(parent_pn, other.printname())
                            for other in parent.partitions[2]])
            elif partition == 2:
                links = set(["{0} {1}".format(other.printname(), parent_pn)
                            for other in parent.partitions[1]])
            else:
                raise Exception(
                    'machine {0} has more than 3 partitions!'.format(machine))

            for link in links:
                yield link

    def word_similarity(self, word1, word2, pos1, pos2):
        lemma1, lemma2 = map(self.wrapper.get_lemma, (word1, word2))
        #logging.info(u'lemma1: {0}, lemma2: {1}'.format(lemma1, lemma2))
        if lemma1 == lemma2:
            return 1
        oov = filter(lambda l: l not in self.wrapper.definitions,
                     (lemma1, lemma2))
        if oov:
            logging.debug(u'OOV: {0}, no machine similarity')
            return None

        machine1, machine2 = map(self.wrapper.definitions.get,
                                 (lemma1, lemma2))<commit_msg>limiting the length of zero-paths<commit_after>import logging

from pymachine.src.machine import MachineGraph

class WordSimilarity():
    def __init__(self, wrapper):
        self.wrapper = wrapper

    def get_links(self, machine):
        self.seen = set()
        return self._get_links(machine, depth=0)

    def _get_links(self, machine, depth):
        if machine in self.seen or depth > 5:
            return
        self.seen.add(machine)
        for hypernym in machine.partitions[0]:
            name = hypernym.printname()
            if name.isupper():
                continue
            yield name
            for link in self._get_links(hypernym, depth=depth+1):
                yield link

        for link in self.get_binary_links(machine):
            yield link

    def get_binary_links(self, machine):
        for parent, partition in machine.parents:
            parent_pn = parent.printname()
            if not parent_pn.isupper() or partition == 0:
                continue
            if partition == 1:
                links = set(["{0} {1}".format(parent_pn, other.printname())
                            for other in parent.partitions[2]])
            elif partition == 2:
                links = set(["{0} {1}".format(other.printname(), parent_pn)
                            for other in parent.partitions[1]])
            else:
                raise Exception(
                    'machine {0} has more than 3 partitions!'.format(machine))

            for link in links:
                yield link

    def word_similarity(self, word1, word2, pos1, pos2):
        lemma1, lemma2 = map(self.wrapper.get_lemma, (word1, word2))
        #logging.warning(u'lemma1: {0}, lemma2: {1}'.format(lemma1, lemma2))
        if lemma1 == lemma2:
            return 1
        oov = filter(lambda l: l not in self.wrapper.definitions,
                     (lemma1, lemma2))
        if oov:
            #logging.warning(u'OOV: {0}, no machine similarity'.format(oov))
            return None

        machine1, machine2 = map(self.wrapper.definitions.get,
                                 (lemma1, lemma2))
{'fraction_non_alphanumeric': 0.09891956782713085, 'fraction_numerical': 0.01800720288115246, 'mean_word_length': 1.7193211488250653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3695660', 'n_tokens_mistral': 1185, 'n_tokens_neox': 1126, 'n_words': 311}