<commit_before># -*- coding: utf-8 -*-

import os
import re
from .cache_utils import cpickle_cache, cached_property
current_dir = os.path.dirname(os.path.realpath(__file__))

from singleton import singleton

@singleton()
class LazyData(object):

    @cached_property
    def en_us_dict(self):
        """
        werid example.
        >>> en_us_dict.check(u"跑")
        True
        """
        import enchant
        return enchant.Dict("en_US")

    @cached_property
    def two_length_words(self):
        """
        检查 2个字符 的字符组 是否是正常单词 或 词组
        e.g. `"At" in two_length_words`
        """
        def load__two_length_words__func():
            import nltk
            import marisa_trie
            return marisa_trie.Trie( \
                        sorted( \
                        list( \
                        set(\
                            [w1 for w1 in nltk.corpus.abc.words() \
                                if (len(w1) == 2) and \
                                    re.compile("[a-z]", re.IGNORECASE).match(w1[0]) \
                            ] \
                        ))))


        return cpickle_cache(current_dir + "/two_length_words.cPickle", load__two_length_words__func)

    @cached_property
    def regular_words(self):
        import nltk
        import marisa_trie
        from .regexp import word_regexp
        return marisa_trie.Trie([unicode(w1) for w1 in nltk.corpus.abc.words() if word_regexp.match(w1)])

    @cached_property
    def lemmatize(self):
        """
        return lemmatize method.

        必须得注明 词形 才行
        >>> lmtzr.lemmatize("having")<commit_msg>Fix ld.regular_words<commit_after># -*- coding: utf-8 -*-

import os
import re
from .cache_utils import cpickle_cache, cached_property
current_dir = os.path.dirname(os.path.realpath(__file__))

from singleton import singleton

@singleton()
class LazyData(object):

    @cached_property
    def en_us_dict(self):
        """
        werid example.
        >>> en_us_dict.check(u"跑")
        True
        """
        import enchant
        return enchant.Dict("en_US")

    @cached_property
    def two_length_words(self):
        """
        检查 2个字符 的字符组 是否是正常单词 或 词组
        e.g. `"At" in two_length_words`
        """
        def load__two_length_words__func():
            import nltk
            import marisa_trie
            return marisa_trie.Trie( \
                        sorted( \
                        list( \
                        set(\
                            [w1 for w1 in nltk.corpus.abc.words() \
                                if (len(w1) == 2) and \
                                    re.compile("[a-z]", re.IGNORECASE).match(w1[0]) \
                            ] \
                        ))))


        return cpickle_cache(current_dir + "/two_length_words.cPickle", load__two_length_words__func)

    @cached_property
    def regular_words(self):
        import nltk
        import marisa_trie
        from .regexp_utils import regexp
        return marisa_trie.Trie([unicode(w1) for w1 in nltk.corpus.abc.words() if regexp.word.match(w1)])

    @cached_property
    def lemmatize(self):
        """
        return lemmatize method.

        必须得注明 词形 才行
        >>> lmtzr.lemmatize("having")
{'fraction_non_alphanumeric': 0.10006176652254478, 'fraction_numerical': 0.0067943174799258805, 'mean_word_length': 1.5747217806041336, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2060957', 'n_tokens_mistral': 1067, 'n_tokens_neox': 1045, 'n_words': 200}