<commit_before>"""
Module contains a Base LM class for subclassing and a generic client-side LM
"""

from cython.operator cimport dereference as deref
from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
from c_vocab cimport VocabIndex, Vocab_None
from stats cimport Stats
from common cimport Boolean, File, LogP, Prob, LogP2, TextStats, LogPtoPPL, _fill_buffer_with_array, _create_array_from_buffer
from vocab cimport Vocab

cdef tuple _compute_ppl(TextStats *tsptr):
    cdef LogP2 prob = tsptr.prob
    cdef double denom = tsptr.numWords - tsptr.numOOVs - tsptr.zeroProbs + tsptr.numSentences
    cdef Prob ppl = LogPtoPPL(prob / denom) if denom > 0 else float('NaN')
    return (prob, denom, ppl)

cdef class Lm:
    """Base class to share common code and to encourage uniform interface"""
    def __cinit__(self, Vocab v, unsigned order, *args, **kwargs):
        self.keysptr = <VocabIndex *>PyMem_Malloc((order+1) * sizeof(VocabIndex))
        if self.keysptr == NULL:
            raise MemoryError
        self._vocab = v
        self._order = order

    def __dealloc__(self):
        PyMem_Free(self.keysptr)

    property order:
        """Order of the language model

        Most language models make a Markovian assumption that the predicted word is fully
        specified by its history/context of length (order-1).
        """
        def __get__(self):
            return self._order

    def prob(self, VocabIndex word, context):
        """Compute log probability of p(word | context)

        Note that the context is an ngram context in reverse order, 
        i.e., if the text is ... w_0 w_1 w_2 ..., then this function computes p(w_2 | w_1, w_0)
        and 'context' should be (w_1, w_0), *not* (w_0, w_1).
        """
        if not context:
            self.keysptr[0] = Vocab_None
            return self.lmptr.wordProb(word, self.keysptr)
        else:
            _fill_buffer_with_array(self._order, self.keysptr, context)
            return self.lmptr.wordProb(word, self.keysptr)

    def prob_ngram(self, ngram):
        """Compute log probability of p(ngram[-1] | ngram[-2], ngram[-3], ...)

           Note that this function takes ngram in its *natural* order.
        """
        cdef VocabIndex word = ngram[-1]
        context = ngram[:-1].reverse()
        return self.prob(word, context)
    
    def read(self, const char *fname, Boolean limitVocab = False, binary = False):
        """Read the language model from a file"""
        mode = 'rb' if binary else 'r'
        cdef File *fptr = new File(fname, mode, 0)
        if fptr == NULL:
            raise MemoryError
        elif fptr.error():
            raise IOError
        ok = self.lmptr.read(deref(fptr), limitVocab)
        del fptr
        return ok

    def write(self, const char *fname, binary = False):
        """Write the language to a file"""
        mode = 'wb' if binary else 'w'
        cdef File *fptr = new File(fname, mode, 0)
        if fptr == NULL:
            raise MemoryError
        elif fptr.error():
            raise IOError
        if binary:
            self.lmptr.writeBinary(deref(fptr))
        else:
            self.lmptr.write(deref(fptr))
        del fptr

    def train(self, Stats ts):<commit_msg>fixed a typo<commit_after>"""
Module contains a Base LM class for subclassing and a generic client-side LM
"""

from cython.operator cimport dereference as deref
from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
from c_vocab cimport VocabIndex, Vocab_None
from stats cimport Stats
from common cimport Boolean, File, LogP, Prob, LogP2, TextStats, LogPtoPPL, _fill_buffer_with_array, _create_array_from_buffer
from vocab cimport Vocab

cdef tuple _compute_ppl(TextStats *tsptr):
    cdef LogP2 prob = tsptr.prob
    cdef double denom = tsptr.numWords - tsptr.numOOVs - tsptr.zeroProbs + tsptr.numSentences
    cdef Prob ppl = LogPtoPPL(prob / denom) if denom > 0 else float('NaN')
    return (prob, denom, ppl)

cdef class Lm:
    """Base class to share common code and to encourage uniform interface"""
    def __cinit__(self, Vocab v, unsigned order, *args, **kwargs):
        self.keysptr = <VocabIndex *>PyMem_Malloc((order+1) * sizeof(VocabIndex))
        if self.keysptr == NULL:
            raise MemoryError
        self._vocab = v
        self._order = order

    def __dealloc__(self):
        PyMem_Free(self.keysptr)

    property order:
        """Order of the language model

        Most language models make a Markovian assumption that the predicted word is fully
        specified by its history/context of length (order-1).
        """
        def __get__(self):
            return self._order

    def prob(self, VocabIndex word, context):
        """Compute log probability of p(word | context)

        Note that the context is an ngram context in reverse order, 
        i.e., if the text is ... w_0 w_1 w_2 ..., then this function computes p(w_2 | w_1, w_0)
        and 'context' should be (w_1, w_0), *not* (w_0, w_1).
        """
        if not context:
            self.keysptr[0] = Vocab_None
            return self.lmptr.wordProb(word, self.keysptr)
        else:
            _fill_buffer_with_array(self._order, self.keysptr, context)
            return self.lmptr.wordProb(word, self.keysptr)

    def prob_ngram(self, ngram):
        """Compute log probability of p(ngram[-1] | ngram[-2], ngram[-3], ...)

           Note that this function takes ngram in its *natural* order.
        """
        cdef VocabIndex word = ngram[-1]
        context = ngram[:-1].reverse()
        return self.prob(word, context)
    
    def read(self, const char *fname, Boolean limitVocab = False, binary = False):
        """Read the language model from a file"""
        mode = 'rb' if binary else 'r'
        cdef File *fptr = new File(fname, mode, 0)
        if fptr == NULL:
            raise MemoryError
        elif fptr.error():
            raise IOError
        ok = self.lmptr.read(deref(fptr), limitVocab)
        del fptr
        return ok

    def write(self, const char *fname, binary = False):
        """Write the language model to a file"""
        mode = 'wb' if binary else 'w'
        cdef File *fptr = new File(fname, mode, 0)
        if fptr == NULL:
            raise MemoryError
        elif fptr.error():
            raise IOError
        if binary:
            self.lmptr.writeBinary(deref(fptr))
        else:
            self.lmptr.write(deref(fptr))
        del fptr

    def train(self, Stats ts):
{'fraction_non_alphanumeric': 0.09564816238659081, 'fraction_numerical': 0.007073658311548516, 'mean_word_length': 2.408805031446541, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2043453', 'n_tokens_mistral': 2081, 'n_tokens_neox': 2020, 'n_words': 664}