<commit_before># coding: utf8
from __future__ import absolute_import, unicode_literals
from contextlib import contextmanager
import shutil

from .tokenizer import Tokenizer
from .vocab import Vocab
from .tagger import Tagger
from .matcher import Matcher
from .lemmatizer import Lemmatizer
from .train import Trainer
from .syntax.parser import get_templates
from .syntax.nonproj import PseudoProjectivity
from .pipeline import DependencyParser, EntityRecognizer
from .syntax.arc_eager import ArcEager
from .syntax.ner import BiluoPushDown
from .compat import json_dumps
from .attrs import IS_STOP
from . import attrs
from . import orth
from . import util
from . import language_data


class BaseDefaults(object):
    @classmethod
    def create_lemmatizer(cls, nlp=None):
        return Lemmatizer(cls.lemma_index, cls.lemma_exc, cls.lemma_rules)

    @classmethod
    def create_vocab(cls, nlp=None):
        lemmatizer = cls.create_lemmatizer(nlp)
        if nlp is None or nlp.path is None:
            lex_attr_getters = dict(cls.lex_attr_getters)
            # This is very messy, but it's the minimal working fix to Issue #639.
            # This defaults stuff needs to be refactored (again)
            lex_attr_getters[IS_STOP] = lambda string: string.lower() in cls.stop_words
            vocab = Vocab(lex_attr_getters=lex_attr_getters, tag_map=cls.tag_map,
                         lemmatizer=lemmatizer)
        else:
            vocab = Vocab.load(nlp.path, lex_attr_getters=cls.lex_attr_getters,
                             tag_map=cls.tag_map, lemmatizer=lemmatizer)
        for tag_str, exc in cls.morph_rules.items():
            for orth_str, attrs in exc.items():
                vocab.morphology.add_special_case(tag_str, orth_str, attrs)
        return vocab

    @classmethod
    def add_vectors(cls, nlp=None):
        if nlp is None or nlp.path is None:
            return False
        else:
            vec_path = nlp.path / 'vocab' / 'vec.bin'
            if vec_path.exists():
                return lambda vocab: vocab.load_vectors_from_bin_loc(vec_path)

    @classmethod
    def create_tokenizer(cls, nlp=None):
        rules = cls.tokenizer_exceptions
        if cls.token_match:
            token_match = cls.token_match
        if cls.prefixes:
            prefix_search  = util.compile_prefix_regex(cls.prefixes).search
        else:
            prefix_search = None
        if cls.suffixes:
            suffix_search  = util.compile_suffix_regex(cls.suffixes).search
        else:
            suffix_search = None
        if cls.infixes:
            infix_finditer = util.compile_infix_regex(cls.infixes).finditer
        else:
            infix_finditer = None
        vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp)
        return Tokenizer(vocab, rules=rules,
                         prefix_search=prefix_search, suffix_search=suffix_search,
                         infix_finditer=infix_finditer, token_match=token_match)

    @classmethod
    def create_tagger(cls, nlp=None):
        if nlp is None:
            return Tagger(cls.create_vocab(), features=cls.tagger_features)
        elif nlp.path is False:
            return Tagger(nlp.vocab, features=cls.tagger_features)
        elif nlp.path is None or not (nlp.path / 'pos').exists():
            return None
        else:
            return Tagger.load(nlp.path / 'pos', nlp.vocab)

    @classmethod
    def create_parser(cls, nlp=None, **cfg):
        if nlp is None:
            return DependencyParser(cls.create_vocab(), features=cls.parser_features,
                                    **cfg)
        elif nlp.path is False:
            return DependencyParser(nlp.vocab, features=cls.parser_features, **cfg)
        elif nlp.path is None or not (nlp.path / 'deps').exists():
            return None
        else:
            return DependencyParser.load(nlp.path / 'deps', nlp.vocab, **cfg)

    @classmethod
    def create_entity(cls, nlp=None, **cfg):
        if nlp is None:
            return EntityRecognizer(cls.create_vocab(), features=cls.entity_features, **cfg)
        elif nlp.path is False:
            return EntityRecognizer(nlp.vocab, features=cls.entity_features, **cfg)
        elif nlp.path is None or not (nlp.path / 'ner').exists():
            return None
        else:
            return EntityRecognizer.load(nlp.path / 'ner', nlp.vocab, **cfg)

    @classmethod
    def create_matcher(cls, nlp=None):
        if nlp is None:
            return Matcher(cls.create_vocab())
        elif nlp.path is False:
            return Matcher(nlp.vocab)
        elif nlp.path is None or not (nlp.path / 'vocab').exists():
            return None
        else:
            return Matcher.load(nlp.path / 'vocab', nlp.vocab)

    @classmethod
    def create_pipeline(self, nlp=None):
        pipeline = []
        if nlp is None:
            return []
        if nlp.tagger:
            pipeline.append(nlp.tagger)
        if nlp.parser:
            pipeline.append(nlp.parser)
            pipeline.append(PseudoProjectivity.deprojectivize)
        if nlp.entity:
            pipeline.append(nlp.entity)
        return pipeline

    token_match = language_data.TOKEN_MATCH
    prefixes = tuple(language_data.TOKENIZER_PREFIXES)
    suffixes = tuple(language_data.TOKENIZER_SUFFIXES)
    infixes = tuple(language_data.TOKENIZER_INFIXES)
    tag_map = dict(language_data.TAG_MAP)
    tokenizer_exceptions = {}
    parser_features = get_templates('parser')
    entity_features = get_templates('ner')
    tagger_features = Tagger.feature_templates # TODO -- fix this
    stop_words = set()
    lemma_rules = {}
    lemma_exc = {}
    lemma_index = {}
    morph_rules = {}

    lex_attr_getters = {
        attrs.LOWER: lambda string: string.lower(),
        attrs.NORM: lambda string: string,
        attrs.SHAPE: orth.word_shape,
        attrs.PREFIX: lambda string: string[0],
        attrs.SUFFIX: lambda string: string[-3:],
        attrs.CLUSTER: lambda string: 0,
        attrs.IS_ALPHA: orth.is_alpha,
        attrs.IS_ASCII: orth.is_ascii,
        attrs.IS_DIGIT: lambda string: string.isdigit(),
        attrs.IS_LOWER: orth.is_lower,
        attrs.IS_PUNCT: orth.is_punct,
        attrs.IS_SPACE: lambda string: string.isspace(),
        attrs.IS_TITLE: orth.is_title,
        attrs.IS_UPPER: orth.is_upper,
        attrs.IS_BRACKET: orth.is_bracket,
        attrs.IS_QUOTE: orth.is_quote,
        attrs.IS_LEFT_PUNCT: orth.is_left_punct,
        attrs.IS_RIGHT_PUNCT: orth.is_right_punct,
        attrs.LIKE_URL: orth.like_url,
        attrs.LIKE_NUM: orth.like_number,
        attrs.LIKE_EMAIL: orth.like_email,
        attrs.IS_STOP: lambda string: False,
        attrs.IS_OOV: lambda string: True
    }


class Language(object):
    """
    A text-processing pipeline. Usually you'll load this once per process, and
    pass the instance around your program.
    """
    Defaults = BaseDefaults
    lang = None

    @classmethod
    def setup_directory(cls, path, **configs):
        """
        Initialise a model directory.
        """
        for name, config in configs.items():
            directory = path / name
            if directory.exists():
                shutil.rmtree(str(directory))
            directory.mkdir()
            with (directory / 'config.json').open('wb') as file_:
                data = json_dumps(config)
                file_.write(data)
        if not (path / 'vocab').exists():
            (path / 'vocab').mkdir()

    @classmethod
    @contextmanager
    def train(cls, path, gold_tuples, **configs):
        parser_cfg = configs.get('deps', {})
        if parser_cfg.get('pseudoprojective'):
            # preprocess training data here before ArcEager.get_labels() is called
            gold_tuples = PseudoProjectivity.preprocess_training_data(gold_tuples)

        for subdir in ('deps', 'ner', 'pos'):
            if subdir not in configs:
                configs[subdir] = {}
        if parser_cfg:
            configs['deps']['actions'] = ArcEager.get_actions(gold_parses=gold_tuples)
        if 'ner' in configs:
            configs['ner']['actions'] = BiluoPushDown.get_actions(gold_parses=gold_tuples)

        cls.setup_directory(path, **configs)

        self = cls(
                path=path,<commit_msg>Fix json dump for Python3<commit_after># coding: utf8
from __future__ import absolute_import, unicode_literals
from contextlib import contextmanager
import shutil

from .tokenizer import Tokenizer
from .vocab import Vocab
from .tagger import Tagger
from .matcher import Matcher
from .lemmatizer import Lemmatizer
from .train import Trainer
from .syntax.parser import get_templates
from .syntax.nonproj import PseudoProjectivity
from .pipeline import DependencyParser, EntityRecognizer
from .syntax.arc_eager import ArcEager
from .syntax.ner import BiluoPushDown
from .compat import json_dumps
from .attrs import IS_STOP
from . import attrs
from . import orth
from . import util
from . import language_data


class BaseDefaults(object):
    @classmethod
    def create_lemmatizer(cls, nlp=None):
        return Lemmatizer(cls.lemma_index, cls.lemma_exc, cls.lemma_rules)

    @classmethod
    def create_vocab(cls, nlp=None):
        lemmatizer = cls.create_lemmatizer(nlp)
        if nlp is None or nlp.path is None:
            lex_attr_getters = dict(cls.lex_attr_getters)
            # This is very messy, but it's the minimal working fix to Issue #639.
            # This defaults stuff needs to be refactored (again)
            lex_attr_getters[IS_STOP] = lambda string: string.lower() in cls.stop_words
            vocab = Vocab(lex_attr_getters=lex_attr_getters, tag_map=cls.tag_map,
                         lemmatizer=lemmatizer)
        else:
            vocab = Vocab.load(nlp.path, lex_attr_getters=cls.lex_attr_getters,
                             tag_map=cls.tag_map, lemmatizer=lemmatizer)
        for tag_str, exc in cls.morph_rules.items():
            for orth_str, attrs in exc.items():
                vocab.morphology.add_special_case(tag_str, orth_str, attrs)
        return vocab

    @classmethod
    def add_vectors(cls, nlp=None):
        if nlp is None or nlp.path is None:
            return False
        else:
            vec_path = nlp.path / 'vocab' / 'vec.bin'
            if vec_path.exists():
                return lambda vocab: vocab.load_vectors_from_bin_loc(vec_path)

    @classmethod
    def create_tokenizer(cls, nlp=None):
        rules = cls.tokenizer_exceptions
        if cls.token_match:
            token_match = cls.token_match
        if cls.prefixes:
            prefix_search  = util.compile_prefix_regex(cls.prefixes).search
        else:
            prefix_search = None
        if cls.suffixes:
            suffix_search  = util.compile_suffix_regex(cls.suffixes).search
        else:
            suffix_search = None
        if cls.infixes:
            infix_finditer = util.compile_infix_regex(cls.infixes).finditer
        else:
            infix_finditer = None
        vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp)
        return Tokenizer(vocab, rules=rules,
                         prefix_search=prefix_search, suffix_search=suffix_search,
                         infix_finditer=infix_finditer, token_match=token_match)

    @classmethod
    def create_tagger(cls, nlp=None):
        if nlp is None:
            return Tagger(cls.create_vocab(), features=cls.tagger_features)
        elif nlp.path is False:
            return Tagger(nlp.vocab, features=cls.tagger_features)
        elif nlp.path is None or not (nlp.path / 'pos').exists():
            return None
        else:
            return Tagger.load(nlp.path / 'pos', nlp.vocab)

    @classmethod
    def create_parser(cls, nlp=None, **cfg):
        if nlp is None:
            return DependencyParser(cls.create_vocab(), features=cls.parser_features,
                                    **cfg)
        elif nlp.path is False:
            return DependencyParser(nlp.vocab, features=cls.parser_features, **cfg)
        elif nlp.path is None or not (nlp.path / 'deps').exists():
            return None
        else:
            return DependencyParser.load(nlp.path / 'deps', nlp.vocab, **cfg)

    @classmethod
    def create_entity(cls, nlp=None, **cfg):
        if nlp is None:
            return EntityRecognizer(cls.create_vocab(), features=cls.entity_features, **cfg)
        elif nlp.path is False:
            return EntityRecognizer(nlp.vocab, features=cls.entity_features, **cfg)
        elif nlp.path is None or not (nlp.path / 'ner').exists():
            return None
        else:
            return EntityRecognizer.load(nlp.path / 'ner', nlp.vocab, **cfg)

    @classmethod
    def create_matcher(cls, nlp=None):
        if nlp is None:
            return Matcher(cls.create_vocab())
        elif nlp.path is False:
            return Matcher(nlp.vocab)
        elif nlp.path is None or not (nlp.path / 'vocab').exists():
            return None
        else:
            return Matcher.load(nlp.path / 'vocab', nlp.vocab)

    @classmethod
    def create_pipeline(self, nlp=None):
        pipeline = []
        if nlp is None:
            return []
        if nlp.tagger:
            pipeline.append(nlp.tagger)
        if nlp.parser:
            pipeline.append(nlp.parser)
            pipeline.append(PseudoProjectivity.deprojectivize)
        if nlp.entity:
            pipeline.append(nlp.entity)
        return pipeline

    token_match = language_data.TOKEN_MATCH
    prefixes = tuple(language_data.TOKENIZER_PREFIXES)
    suffixes = tuple(language_data.TOKENIZER_SUFFIXES)
    infixes = tuple(language_data.TOKENIZER_INFIXES)
    tag_map = dict(language_data.TAG_MAP)
    tokenizer_exceptions = {}
    parser_features = get_templates('parser')
    entity_features = get_templates('ner')
    tagger_features = Tagger.feature_templates # TODO -- fix this
    stop_words = set()
    lemma_rules = {}
    lemma_exc = {}
    lemma_index = {}
    morph_rules = {}

    lex_attr_getters = {
        attrs.LOWER: lambda string: string.lower(),
        attrs.NORM: lambda string: string,
        attrs.SHAPE: orth.word_shape,
        attrs.PREFIX: lambda string: string[0],
        attrs.SUFFIX: lambda string: string[-3:],
        attrs.CLUSTER: lambda string: 0,
        attrs.IS_ALPHA: orth.is_alpha,
        attrs.IS_ASCII: orth.is_ascii,
        attrs.IS_DIGIT: lambda string: string.isdigit(),
        attrs.IS_LOWER: orth.is_lower,
        attrs.IS_PUNCT: orth.is_punct,
        attrs.IS_SPACE: lambda string: string.isspace(),
        attrs.IS_TITLE: orth.is_title,
        attrs.IS_UPPER: orth.is_upper,
        attrs.IS_BRACKET: orth.is_bracket,
        attrs.IS_QUOTE: orth.is_quote,
        attrs.IS_LEFT_PUNCT: orth.is_left_punct,
        attrs.IS_RIGHT_PUNCT: orth.is_right_punct,
        attrs.LIKE_URL: orth.like_url,
        attrs.LIKE_NUM: orth.like_number,
        attrs.LIKE_EMAIL: orth.like_email,
        attrs.IS_STOP: lambda string: False,
        attrs.IS_OOV: lambda string: True
    }


class Language(object):
    """
    A text-processing pipeline. Usually you'll load this once per process, and
    pass the instance around your program.
    """
    Defaults = BaseDefaults
    lang = None

    @classmethod
    def setup_directory(cls, path, **configs):
        """
        Initialise a model directory.
        """
        for name, config in configs.items():
            directory = path / name
            if directory.exists():
                shutil.rmtree(str(directory))
            directory.mkdir()
            with (directory / 'config.json').open('w') as file_:
                data = json_dumps(config)
                file_.write(data)
        if not (path / 'vocab').exists():
            (path / 'vocab').mkdir()

    @classmethod
    @contextmanager
    def train(cls, path, gold_tuples, **configs):
        parser_cfg = configs.get('deps', {})
        if parser_cfg.get('pseudoprojective'):
            # preprocess training data here before ArcEager.get_labels() is called
            gold_tuples = PseudoProjectivity.preprocess_training_data(gold_tuples)

        for subdir in ('deps', 'ner', 'pos'):
            if subdir not in configs:
                configs[subdir] = {}
        if parser_cfg:
            configs['deps']['actions'] = ArcEager.get_actions(gold_parses=gold_tuples)
        if 'ner' in configs:
            configs['ner']['actions'] = BiluoPushDown.get_actions(gold_parses=gold_tuples)

        cls.setup_directory(path, **configs)

        self = cls(
                path=path,
{'fraction_non_alphanumeric': 0.09202047576031316, 'fraction_numerical': 0.0009033423667570009, 'mean_word_length': 2.3285227500501104, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4180662', 'n_tokens_mistral': 5242, 'n_tokens_neox': 5151, 'n_words': 1175}