<commit_before>##
# Copyright (c) 2008-2010 MagicStack Inc.
# All rights reserved.
#
# See LICENSE for details.
##


from metamagic.utils.algos.persistent_hash import persistent_hash


class NaturalLanguageObject(object):
    pass


class GrammaticalCategoryMeta(type):
    index = {}

    def __init__(cls, name, bases, dct):
        super(GrammaticalCategoryMeta, cls).__init__(name, bases, dct)

        name = dct.get('name')
        if name:
            idx = GrammaticalCategoryMeta.index
            if name in idx:
                raise Exception('Grammatical category "%s" is already used by %s.%s' %
                                (name, idx[name].__module__, idx[name].__name__))
            else:
                idx[name] = cls
        else:
            raise Exception('Missing required grammatical category name')


def is_valid_category(category):
    return category in GrammaticalCategoryMeta.index


def form(category, value):
    cls = GrammaticalCategoryMeta.index.get(category)
    if cls:
        return cls(value)
    else:
        raise Exception('reference to an invalid grammatical category: %s' % category)


class GrammaticalCategory(object, metaclass=GrammaticalCategoryMeta):
    name = 'category'

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return self.value

    def __repr__(self):
        return '<%s %r>' % (self.__class__.name, self.value)

    def __eq__(self, other):
        if isinstance(other, GrammaticalCategory):
            return self.value == other.value
        elif isinstance(other, str):
            return self.value == other
        else:
            return False

    def __hash__(self):
        return hash(self.__class__.name + ':' + self.value)

    def persistent_hash(self):
        return persistent_hash(self.__class__.name + ':' + self.value)


class GrammaticalNumber(GrammaticalCategory):
    name = 'number'

    def format_count(self, count):
        pass


class Singular(GrammaticalNumber):
    name = 'singular'


class Plural(GrammaticalNumber):
    name = 'plural'


class WordCombination(NaturalLanguageObject):
    def __new__(cls, value):
        self = super().__new__(cls)

        if isinstance(value, str):
            value = {Singular(value)}

        self.forms = {}

        for form in value:
            assert isinstance(form, GrammaticalCategory), type(form)
            self.forms[form.__class__.name] = form.__class__(form.value)
<commit_msg>utils.nlang.morphology: Allow dicts to WordCombination constructor<commit_after>##
# Copyright (c) 2008-2010 MagicStack Inc.
# All rights reserved.
#
# See LICENSE for details.
##


from metamagic.utils.algos.persistent_hash import persistent_hash


class NaturalLanguageObject(object):
    pass


class GrammaticalCategoryMeta(type):
    index = {}

    def __init__(cls, name, bases, dct):
        super(GrammaticalCategoryMeta, cls).__init__(name, bases, dct)

        name = dct.get('name')
        if name:
            idx = GrammaticalCategoryMeta.index
            if name in idx:
                raise Exception('Grammatical category "%s" is already used by %s.%s' %
                                (name, idx[name].__module__, idx[name].__name__))
            else:
                idx[name] = cls
        else:
            raise Exception('Missing required grammatical category name')


def is_valid_category(category):
    return category in GrammaticalCategoryMeta.index


def form(category, value):
    cls = GrammaticalCategoryMeta.index.get(category)
    if cls:
        return cls(value)
    else:
        raise Exception('reference to an invalid grammatical category: %s' % category)


def forms(mapping):
    for k, v in mapping.items():
        yield form(k, v)


class GrammaticalCategory(object, metaclass=GrammaticalCategoryMeta):
    name = 'category'

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return self.value

    def __repr__(self):
        return '<%s %r>' % (self.__class__.name, self.value)

    def __eq__(self, other):
        if isinstance(other, GrammaticalCategory):
            return self.value == other.value
        elif isinstance(other, str):
            return self.value == other
        else:
            return False

    def __hash__(self):
        return hash(self.__class__.name + ':' + self.value)

    def persistent_hash(self):
        return persistent_hash(self.__class__.name + ':' + self.value)


class GrammaticalNumber(GrammaticalCategory):
    name = 'number'

    def format_count(self, count):
        pass


class Singular(GrammaticalNumber):
    name = 'singular'


class Plural(GrammaticalNumber):
    name = 'plural'


class WordCombination(NaturalLanguageObject):
    def __new__(cls, value):
        self = super().__new__(cls)

        if isinstance(value, str):
            value = {Singular(value)}
        elif isinstance(value, dict):
            value = forms(value)

        self.forms = {}

        for form in value:
            assert isinstance(form, GrammaticalCategory), type(form)
            self.forms[form.__class__.name] = form.__class__(form.value)

{'fraction_non_alphanumeric': 0.10170476559473073, 'fraction_numerical': 0.0030995738086013174, 'mean_word_length': 2.5879082696316886, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2218241', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1450, 'n_words': 374}