<commit_before>from libc.stdio cimport fopen, fclose, fread, fwrite, FILE
from libc.string cimport memset
from libc.stdint cimport int32_t

import bz2
from os import path
import codecs
import math

from .lexeme cimport EMPTY_LEXEME
from .lexeme cimport set_lex_struct_props
from .lexeme cimport Lexeme
from .strings cimport slice_unicode
from .strings cimport hash_string
from .orth cimport word_shape
from .typedefs cimport attr_t

from cymem.cymem cimport Address


DEF MAX_VEC_SIZE = 100000


cdef float[MAX_VEC_SIZE] EMPTY_VEC
memset(EMPTY_VEC, 0, sizeof(EMPTY_VEC))
memset(&EMPTY_LEXEME, 0, sizeof(LexemeC))
EMPTY_LEXEME.repvec = EMPTY_VEC


cdef class Vocab:
    '''A map container for a language's LexemeC structs.
    '''
    def __init__(self, data_dir=None, get_lex_props=None, load_vectors=True,
                 pos_tags=None):
        self.mem = Pool()
        self._map = PreshMap(2 ** 20)
        self.strings = StringStore()
        self.pos_tags = pos_tags if pos_tags is not None else {}
        self.lexemes.push_back(&EMPTY_LEXEME)
        self.lexeme_props_getter = get_lex_props
        self.repvec_length = 0
        if data_dir is not None:
            if not path.exists(data_dir):
                raise IOError("Directory %s not found -- cannot load Vocab." % data_dir)
        if data_dir is not None:
            if not path.isdir(data_dir):
                raise IOError("Path %s is a file, not a dir -- cannot load Vocab." % data_dir)
            self.load_lexemes(path.join(data_dir, 'strings.txt'),
                              path.join(data_dir, 'lexemes.bin'))
            if load_vectors and path.exists(path.join(data_dir, 'vec.bin')):
                self.repvec_length = self.load_rep_vectors(path.join(data_dir, 'vec.bin'))
        self._codec = None

    def __len__(self):
        """The current number of lexemes stored."""
        return self.lexemes.size()

    cdef const LexemeC* get(self, Pool mem, UniStr* c_str) except NULL:
        '''Get a pointer to a LexemeC from the lexicon, creating a new Lexeme
        if necessary, using memory acquired from the given pool.  If the pool
        is the lexicon's own memory, the lexeme is saved in the lexicon.'''
        cdef LexemeC* lex
        lex = <LexemeC*>self._map.get(c_str.key)
        if lex != NULL:
            return lex
        if c_str.n < 3:
            mem = self.mem
        cdef unicode py_str = c_str.chars[:c_str.n]
        lex = <LexemeC*>mem.alloc(sizeof(LexemeC), 1)
        props = self.lexeme_props_getter(py_str)
        set_lex_struct_props(lex, props, self.strings, EMPTY_VEC)
        if mem is self.mem:<commit_msg>* Remove codec property from Vocab<commit_after>from libc.stdio cimport fopen, fclose, fread, fwrite, FILE
from libc.string cimport memset
from libc.stdint cimport int32_t

import bz2
from os import path
import codecs
import math

from .lexeme cimport EMPTY_LEXEME
from .lexeme cimport set_lex_struct_props
from .lexeme cimport Lexeme
from .strings cimport slice_unicode
from .strings cimport hash_string
from .orth cimport word_shape
from .typedefs cimport attr_t

from cymem.cymem cimport Address


DEF MAX_VEC_SIZE = 100000


cdef float[MAX_VEC_SIZE] EMPTY_VEC
memset(EMPTY_VEC, 0, sizeof(EMPTY_VEC))
memset(&EMPTY_LEXEME, 0, sizeof(LexemeC))
EMPTY_LEXEME.repvec = EMPTY_VEC


cdef class Vocab:
    '''A map container for a language's LexemeC structs.
    '''
    def __init__(self, data_dir=None, get_lex_props=None, load_vectors=True,
                 pos_tags=None):
        self.mem = Pool()
        self._map = PreshMap(2 ** 20)
        self.strings = StringStore()
        self.pos_tags = pos_tags if pos_tags is not None else {}
        self.lexemes.push_back(&EMPTY_LEXEME)
        self.lexeme_props_getter = get_lex_props
        self.repvec_length = 0
        if data_dir is not None:
            if not path.exists(data_dir):
                raise IOError("Directory %s not found -- cannot load Vocab." % data_dir)
        if data_dir is not None:
            if not path.isdir(data_dir):
                raise IOError("Path %s is a file, not a dir -- cannot load Vocab." % data_dir)
            self.load_lexemes(path.join(data_dir, 'strings.txt'),
                              path.join(data_dir, 'lexemes.bin'))
            if load_vectors and path.exists(path.join(data_dir, 'vec.bin')):
                self.repvec_length = self.load_rep_vectors(path.join(data_dir, 'vec.bin'))

    def __len__(self):
        """The current number of lexemes stored."""
        return self.lexemes.size()

    cdef const LexemeC* get(self, Pool mem, UniStr* c_str) except NULL:
        '''Get a pointer to a LexemeC from the lexicon, creating a new Lexeme
        if necessary, using memory acquired from the given pool.  If the pool
        is the lexicon's own memory, the lexeme is saved in the lexicon.'''
        cdef LexemeC* lex
        lex = <LexemeC*>self._map.get(c_str.key)
        if lex != NULL:
            return lex
        if c_str.n < 3:
            mem = self.mem
        cdef unicode py_str = c_str.chars[:c_str.n]
        lex = <LexemeC*>mem.alloc(sizeof(LexemeC), 1)
        props = self.lexeme_props_getter(py_str)
        set_lex_struct_props(lex, props, self.strings, EMPTY_VEC)
        if mem is self.mem:
{'fraction_non_alphanumeric': 0.08851402577710386, 'fraction_numerical': 0.006444275966641395, 'mean_word_length': 2.6696801112656465, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7342482', 'n_tokens_mistral': 1834, 'n_tokens_neox': 1736, 'n_words': 506}