<commit_before># cython: infer_types
# coding: utf8
from __future__ import unicode_literals

from libc.string cimport memset
import srsly
from collections import Counter

from .compat import basestring_
from .strings import get_string_id
from . import symbols
from .attrs cimport POS, IS_SPACE
from .attrs import LEMMA, intify_attrs
from .parts_of_speech cimport SPACE
from .parts_of_speech import IDS as POS_IDS
from .lexeme cimport Lexeme
from .errors import Errors
from .util import ensure_path


cdef enum univ_field_t:
    Field_POS
    Field_Abbr
    Field_AdpType
    Field_AdvType
    Field_Animacy
    Field_Aspect
    Field_Case
    Field_ConjType
    Field_Connegative
    Field_Definite
    Field_Degree
    Field_Derivation
    Field_Echo
    Field_Foreign
    Field_Gender
    Field_Hyph
    Field_InfForm
    Field_Mood
    Field_NameType
    Field_Negative
    Field_NounType
    Field_Number
    Field_NumForm
    Field_NumType
    Field_NumValue
    Field_PartForm
    Field_PartType
    Field_Person
    Field_Polarity
    Field_Polite
    Field_Poss
    Field_Prefix
    Field_PrepCase
    Field_PronType
    Field_PunctSide
    Field_PunctType
    Field_Reflex
    Field_Style
    Field_StyleVariant
    Field_Tense
    Field_Typo
    Field_VerbForm
    Field_VerbType
    Field_Voice


def _normalize_props(props):
    """Transform deprecated string keys to correct names."""
    out = {}
    props = dict(props)
    for key in FIELDS:
        if key in props:
            value = str(props[key]).lower()
            # We don't have support for disjunctive int|rel features, so
            # just take the first one :(
            if "|" in value:
                value = value.split("|")[0]
            attr = '%s_%s' % (key, value)
            if attr in FEATURES:
                props.pop(key)
                props[attr] = True
    for key, value in props.items():
        if key == POS:
            if hasattr(value, 'upper'):
                value = value.upper()
            if value in POS_IDS:
                value = POS_IDS[value]
            out[key] = value
        elif isinstance(key, int):
            out[key] = value
        elif value is True:
            out[key] = value
        elif key.lower() == 'pos':
            out[POS] = POS_IDS[value.upper()]
        elif key.lower() != 'morph':
            out[key] = value
    return out


class MorphologyClassMap(object):
    def __init__(self, features):
        self.features = tuple(features)
        self.fields = []
        self.feat2field = {}
        seen_fields = set()
        for feature in features:
            field = feature.split("_", 1)[0]
            if field not in seen_fields:
                self.fields.append(field)
                seen_fields.add(field)
            self.feat2field[feature] = FIELDS[field]
        self.id2feat = {get_string_id(name): name for name in features}
        self.field2feats = {"POS": []}
        self.col2info = []
        self.attr2field = dict(LOWER_FIELDS.items())
        self.feat2offset = {}
        self.field2col = {}
        self.field2id = dict(FIELDS.items())
        self.fieldid2field = {field_id: field for field, field_id in FIELDS.items()}
        for feature in features:
            field = self.fields[self.feat2field[feature]]
            if field not in self.field2col:
                self.field2col[field] = len(self.col2info)
            if field != "POS" and field not in self.field2feats:
                self.col2info.append((field, 0, "NIL"))
            self.field2feats.setdefault(field, ["NIL"])
            offset = len(self.field2feats[field])
            self.field2feats[field].append(feature)
            self.col2info.append((field, offset, feature))
            self.feat2offset[feature] = offset

    @property
    def field_sizes(self):
        return [len(self.field2feats[field]) for field in self.fields]

    def get_field_offset(self, field):
        return self.field2col[field]


cdef class Morphology:
    '''Store the possible morphological analyses for a language, and index them
    by hash.

    To save space on each token, tokens only know the hash of their morphological
    analysis, so queries of morphological attributes are delegated
    to this class.
    '''
    def __init__(self, StringStore string_store, tag_map, lemmatizer, exc=None):
        self.mem = Pool()
        self.strings = string_store
        self.tags = PreshMap()
        # Add special space symbol. We prefix with underscore, to make sure it
        # always sorts to the end.
        space_attrs = tag_map.get('SP', {POS: SPACE})
        if '_SP' not in tag_map:
            self.strings.add('_SP')
            tag_map = dict(tag_map)
            tag_map['_SP'] = space_attrs
        self.tag_names = tuple(sorted(tag_map.keys()))
        self.tag_map = {}
        self.lemmatizer = lemmatizer
        self.n_tags = len(tag_map)
        self.reverse_index = {}
        self._feat_map = MorphologyClassMap(FEATURES)
        self._load_from_tag_map(tag_map)

        self._cache = PreshMapArray(self.n_tags)
        self.exc = {}
        if exc is not None:
            for (tag, orth), attrs in exc.items():
                attrs = _normalize_props(attrs)
                self.add_special_case(
                    self.strings.as_string(tag), self.strings.as_string(orth), attrs)

    def _load_from_tag_map(self, tag_map):
        for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())):
            attrs = _normalize_props(attrs)
            self.add({self._feat_map.id2feat[feat] for feat in attrs
                      if feat in self._feat_map.id2feat})
            self.tag_map[tag_str] = dict(attrs)
            self.reverse_index[self.strings.add(tag_str)] = i
<commit_msg>Prefer _SP over SP for default tag map space attrs<commit_after># cython: infer_types
# coding: utf8
from __future__ import unicode_literals

from libc.string cimport memset
import srsly
from collections import Counter

from .compat import basestring_
from .strings import get_string_id
from . import symbols
from .attrs cimport POS, IS_SPACE
from .attrs import LEMMA, intify_attrs
from .parts_of_speech cimport SPACE
from .parts_of_speech import IDS as POS_IDS
from .lexeme cimport Lexeme
from .errors import Errors
from .util import ensure_path


cdef enum univ_field_t:
    Field_POS
    Field_Abbr
    Field_AdpType
    Field_AdvType
    Field_Animacy
    Field_Aspect
    Field_Case
    Field_ConjType
    Field_Connegative
    Field_Definite
    Field_Degree
    Field_Derivation
    Field_Echo
    Field_Foreign
    Field_Gender
    Field_Hyph
    Field_InfForm
    Field_Mood
    Field_NameType
    Field_Negative
    Field_NounType
    Field_Number
    Field_NumForm
    Field_NumType
    Field_NumValue
    Field_PartForm
    Field_PartType
    Field_Person
    Field_Polarity
    Field_Polite
    Field_Poss
    Field_Prefix
    Field_PrepCase
    Field_PronType
    Field_PunctSide
    Field_PunctType
    Field_Reflex
    Field_Style
    Field_StyleVariant
    Field_Tense
    Field_Typo
    Field_VerbForm
    Field_VerbType
    Field_Voice


def _normalize_props(props):
    """Transform deprecated string keys to correct names."""
    out = {}
    props = dict(props)
    for key in FIELDS:
        if key in props:
            value = str(props[key]).lower()
            # We don't have support for disjunctive int|rel features, so
            # just take the first one :(
            if "|" in value:
                value = value.split("|")[0]
            attr = '%s_%s' % (key, value)
            if attr in FEATURES:
                props.pop(key)
                props[attr] = True
    for key, value in props.items():
        if key == POS:
            if hasattr(value, 'upper'):
                value = value.upper()
            if value in POS_IDS:
                value = POS_IDS[value]
            out[key] = value
        elif isinstance(key, int):
            out[key] = value
        elif value is True:
            out[key] = value
        elif key.lower() == 'pos':
            out[POS] = POS_IDS[value.upper()]
        elif key.lower() != 'morph':
            out[key] = value
    return out


class MorphologyClassMap(object):
    def __init__(self, features):
        self.features = tuple(features)
        self.fields = []
        self.feat2field = {}
        seen_fields = set()
        for feature in features:
            field = feature.split("_", 1)[0]
            if field not in seen_fields:
                self.fields.append(field)
                seen_fields.add(field)
            self.feat2field[feature] = FIELDS[field]
        self.id2feat = {get_string_id(name): name for name in features}
        self.field2feats = {"POS": []}
        self.col2info = []
        self.attr2field = dict(LOWER_FIELDS.items())
        self.feat2offset = {}
        self.field2col = {}
        self.field2id = dict(FIELDS.items())
        self.fieldid2field = {field_id: field for field, field_id in FIELDS.items()}
        for feature in features:
            field = self.fields[self.feat2field[feature]]
            if field not in self.field2col:
                self.field2col[field] = len(self.col2info)
            if field != "POS" and field not in self.field2feats:
                self.col2info.append((field, 0, "NIL"))
            self.field2feats.setdefault(field, ["NIL"])
            offset = len(self.field2feats[field])
            self.field2feats[field].append(feature)
            self.col2info.append((field, offset, feature))
            self.feat2offset[feature] = offset

    @property
    def field_sizes(self):
        return [len(self.field2feats[field]) for field in self.fields]

    def get_field_offset(self, field):
        return self.field2col[field]


cdef class Morphology:
    '''Store the possible morphological analyses for a language, and index them
    by hash.

    To save space on each token, tokens only know the hash of their morphological
    analysis, so queries of morphological attributes are delegated
    to this class.
    '''
    def __init__(self, StringStore string_store, tag_map, lemmatizer, exc=None):
        self.mem = Pool()
        self.strings = string_store
        self.tags = PreshMap()
        # Add special space symbol. We prefix with underscore, to make sure it
        # always sorts to the end.
        if '_SP' in tag_map:
            space_attrs = tag_map.get('_SP')
        else:
            space_attrs = tag_map.get('SP', {POS: SPACE})
        if '_SP' not in tag_map:
            self.strings.add('_SP')
            tag_map = dict(tag_map)
            tag_map['_SP'] = space_attrs
        self.tag_names = tuple(sorted(tag_map.keys()))
        self.tag_map = {}
        self.lemmatizer = lemmatizer
        self.n_tags = len(tag_map)
        self.reverse_index = {}
        self._feat_map = MorphologyClassMap(FEATURES)
        self._load_from_tag_map(tag_map)

        self._cache = PreshMapArray(self.n_tags)
        self.exc = {}
        if exc is not None:
            for (tag, orth), attrs in exc.items():
                attrs = _normalize_props(attrs)
                self.add_special_case(
                    self.strings.as_string(tag), self.strings.as_string(orth), attrs)

    def _load_from_tag_map(self, tag_map):
        for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())):
            attrs = _normalize_props(attrs)
            self.add({self._feat_map.id2feat[feat] for feat in attrs
                      if feat in self._feat_map.id2feat})
            self.tag_map[tag_str] = dict(attrs)
            self.reverse_index[self.strings.add(tag_str)] = i

{'fraction_non_alphanumeric': 0.08628527317951797, 'fraction_numerical': 0.005146238957028905, 'mean_word_length': 2.1989026063100137, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4079388', 'n_tokens_mistral': 3616, 'n_tokens_neox': 3522, 'n_words': 904}