<commit_before>from __future__ import unicode_literals, print_function
import codecs
import pathlib

try:
    import ujson as json
except ImportError:
    import json

from .parts_of_speech import NOUN, VERB, ADJ, PUNCT


class Lemmatizer(object):
    @classmethod
    def load(cls, path):
        index = {}
        exc = {}
        for pos in ['adj', 'noun', 'verb']:
            pos_index_path = path / 'wordnet' / 'index.{pos}'.format(pos=pos)
            if pos_index_path.exists():
                with pos_index_path.open() as file_:
                    index[pos] = read_index(file_)
            else:
                index[pos] = set()
            pos_exc_path = path / 'wordnet' / '{pos}.exc'.format(pos=pos)
            if pos_exc_path.exists():
                with pos_exc_path.open() as file_:
                    exc[pos] = read_exc(file_)
            else:
                exc[pos] = {}
        with (path / 'vocab' / 'lemma_rules.json').open() as file_:
            rules = json.load(file_)
        return cls(index, exc, rules)

    def __init__(self, index, exceptions, rules):
        self.index = index
        self.exc = exceptions
        self.rules = rules

    def __call__(self, string, pos, **morphology):
        if pos == NOUN:
            pos = 'noun'
        elif pos == VERB:
            pos = 'verb'
        elif pos == ADJ:
            pos = 'adj'
        elif pos == PUNCT:
            pos = 'punct'
        # See Issue #435 for example of where this logic is requied.
        if self.is_base_form(pos, **morphology):
            return set([string.lower()])
        lemmas = lemmatize(string, self.index.get(pos, {}), self.exc.get(pos, {}), self.rules.get(pos, []))
        return lemmas

    def is_base_form(self, pos, **morphology):
        '''Check whether we're dealing with an uninflected paradigm, so we can
        avoid lemmatization entirely.'''
        if pos == 'noun' and morphology.get('number') == 'sing' and len(morphology) == 1:
            return True
        elif pos == 'verb' and morphology.get('verbform') == 'inf' and len(morphology) == 1:
            return True
        else:
            return False

    def noun(self, string, **morphology):
        return self(string, 'noun', **morphology)

    def verb(self, string, **morphology):
        return self(string, 'verb', **morphology)

    def adj(self, string, **morphology):
        return self(string, 'adj', **morphology)

    def punct(self, string, **morphology):
        return self(string, 'punct', **morphology)


def lemmatize(string, index, exceptions, rules):
    string = string.lower()
    forms = []
    # TODO: Is this correct? See discussion in Issue #435.
    #if string in index:
    #    forms.append(string)
    forms.extend(exceptions.get(string, []))
    for old, new in rules:
        if string.endswith(old):
            form = string[:len(string) - len(old)] + new
            if form in index or not form.isalpha():
                forms.append(form)
    if not forms:<commit_msg>Tweak the new is_base_form logic, because we can expect the 'pos' key in the morphology we're passed.<commit_after>from __future__ import unicode_literals, print_function
import codecs
import pathlib

try:
    import ujson as json
except ImportError:
    import json

from .parts_of_speech import NOUN, VERB, ADJ, PUNCT


class Lemmatizer(object):
    @classmethod
    def load(cls, path):
        index = {}
        exc = {}
        for pos in ['adj', 'noun', 'verb']:
            pos_index_path = path / 'wordnet' / 'index.{pos}'.format(pos=pos)
            if pos_index_path.exists():
                with pos_index_path.open() as file_:
                    index[pos] = read_index(file_)
            else:
                index[pos] = set()
            pos_exc_path = path / 'wordnet' / '{pos}.exc'.format(pos=pos)
            if pos_exc_path.exists():
                with pos_exc_path.open() as file_:
                    exc[pos] = read_exc(file_)
            else:
                exc[pos] = {}
        with (path / 'vocab' / 'lemma_rules.json').open() as file_:
            rules = json.load(file_)
        return cls(index, exc, rules)

    def __init__(self, index, exceptions, rules):
        self.index = index
        self.exc = exceptions
        self.rules = rules

    def __call__(self, string, pos, **morphology):
        if pos == NOUN:
            pos = 'noun'
        elif pos == VERB:
            pos = 'verb'
        elif pos == ADJ:
            pos = 'adj'
        elif pos == PUNCT:
            pos = 'punct'
        # See Issue #435 for example of where this logic is requied.
        if self.is_base_form(pos, **morphology):
            return set([string.lower()])
        lemmas = lemmatize(string, self.index.get(pos, {}), self.exc.get(pos, {}), self.rules.get(pos, []))
        return lemmas

    def is_base_form(self, pos, **morphology):
        '''Check whether we're dealing with an uninflected paradigm, so we can
        avoid lemmatization entirely.'''
        others = [key for key in morphology if key not in ('number', 'pos', 'verbform')]
        if pos == 'noun' and morphology.get('number') == 'sing' and not others:
            return True
        elif pos == 'verb' and morphology.get('verbform') == 'inf' and not others:
            return True
        else:
            return False

    def noun(self, string, **morphology):
        return self(string, 'noun', **morphology)

    def verb(self, string, **morphology):
        return self(string, 'verb', **morphology)

    def adj(self, string, **morphology):
        return self(string, 'adj', **morphology)

    def punct(self, string, **morphology):
        return self(string, 'punct', **morphology)


def lemmatize(string, index, exceptions, rules):
    string = string.lower()
    forms = []
    # TODO: Is this correct? See discussion in Issue #435.
    #if string in index:
    #    forms.append(string)
    forms.extend(exceptions.get(string, []))
    for old, new in rules:
        if string.endswith(old):
            form = string[:len(string) - len(old)] + new
            if form in index or not form.isalpha():
                forms.append(form)
    if not forms:
{'fraction_non_alphanumeric': 0.12425367113119251, 'fraction_numerical': 0.002259157656930773, 'mean_word_length': 2.123991935483871, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7515568', 'n_tokens_mistral': 1928, 'n_tokens_neox': 1856, 'n_words': 546}