<commit_before>from cpython.ref cimport PyObject
from libc.stddef cimport wchar_t
from libc.stdlib cimport malloc, free
from libcpp.vector cimport vector
from libcpp.set cimport set
from libcpp.string cimport string

cimport _clstm

from collections import namedtuple


CharPrediction = namedtuple("CharPrediction",
                            ("x_position", "char", "confidence"))


cdef extern from "Python.h":
    ctypedef PyObject PyUnicodeObject
    Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *o, wchar_t *w,
                                    Py_ssize_t size)
    PyObject* PyUnicode_FromWideChar(wchar_t *w, Py_ssize_t size)


cpdef double levenshtein(unicode a, unicode b):
    return _clstm.levenshtein[string, string](
        a.encode('utf8'), b.encode('utf8'))


cdef load_img(img, _clstm.Tensor2 *data):
    """ Copy image data from a PIL image to an Eigen tensor.

    :param img:     Image
    :type img:      :py:class:`PIL.Image.Image`
    """
    data.resize(img.width, img.height)
    imgdata = img.load()
    for i in range(img.width):
        for j in range(img.height):
            px = imgdata[i, j]
            # Pillow returns pixels as [0, 255], but we need [0, 1]
            if isinstance(px, tuple):
                # For color images, we use the mean across all channels
                px = (sum(px)/len(px))/255.
            else:
                px = px/255.
            px = -px + 1.
            data[0].put(px, i, j)


cdef class ClstmOcr:
    cdef _clstm.CLSTMOCR *_ocr

    def __cinit__(self, str fname=None):
        self._ocr = new _clstm.CLSTMOCR()

    cpdef load(self, str fname):
        cdef bint rv = self._ocr.maybe_load(fname)
        if not rv:
            raise IOError("Could not load model from {}".format(fname))

    cpdef save(self, str fname):
        cdef bint rv = self._ocr.maybe_save(fname)
        if not rv:
            raise IOError("Could not save model to {}".format(fname))
<commit_msg>python: allow loading of model in constructor<commit_after>from cpython.ref cimport PyObject
from libc.stddef cimport wchar_t
from libc.stdlib cimport malloc, free
from libcpp.vector cimport vector
from libcpp.set cimport set
from libcpp.string cimport string

cimport _clstm

from collections import namedtuple


CharPrediction = namedtuple("CharPrediction",
                            ("x_position", "char", "confidence"))


cdef extern from "Python.h":
    ctypedef PyObject PyUnicodeObject
    Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *o, wchar_t *w,
                                    Py_ssize_t size)
    PyObject* PyUnicode_FromWideChar(wchar_t *w, Py_ssize_t size)


cpdef double levenshtein(unicode a, unicode b):
    return _clstm.levenshtein[string, string](
        a.encode('utf8'), b.encode('utf8'))


cdef load_img(img, _clstm.Tensor2 *data):
    """ Copy image data from a PIL image to an Eigen tensor.

    :param img:     Image
    :type img:      :py:class:`PIL.Image.Image`
    """
    data.resize(img.width, img.height)
    imgdata = img.load()
    for i in range(img.width):
        for j in range(img.height):
            px = imgdata[i, j]
            # Pillow returns pixels as [0, 255], but we need [0, 1]
            if isinstance(px, tuple):
                # For color images, we use the mean across all channels
                px = (sum(px)/len(px))/255.
            else:
                px = px/255.
            px = -px + 1.
            data[0].put(px, i, j)


cdef class ClstmOcr:
    cdef _clstm.CLSTMOCR *_ocr

    def __cinit__(self, str fname=None):
        self._ocr = new _clstm.CLSTMOCR()
        if fname:
            self.load(fname)

    cpdef load(self, str fname):
        cdef bint rv = self._ocr.maybe_load(fname)
        if not rv:
            raise IOError("Could not load model from {}".format(fname))

    cpdef save(self, str fname):
        cdef bint rv = self._ocr.maybe_save(fname)
        if not rv:
            raise IOError("Could not save model to {}".format(fname))

{'fraction_non_alphanumeric': 0.10037546933667084, 'fraction_numerical': 0.00851063829787234, 'mean_word_length': 2.321695760598504, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '703760', 'n_tokens_mistral': 1353, 'n_tokens_neox': 1294, 'n_words': 381}