<commit_before>from libc.stdlib cimport malloc, free

from cpython cimport (PyObject, PyString_FromString,
                      PyString_AsString, PyString_Check)

cdef extern from "stdlib.h":
    void memcpy(void *dst, void *src, size_t n)

cimport numpy as cnp

from numpy cimport ndarray, uint8_t, uint64_t

import numpy as np

# cdef extern from "Python.h":
#     void Py_INCREF(PyObject*)
#     void Py_XDECREF(PyObject*)

cimport util

import pandas.lib as lib

import time<commit_msg>DOC: sublicense parser-tokenizer cython code<commit_after># Copyright (c) 2012, Lambda Foundry, Inc.
# All rights reserved.

# Released subject to the terms of the GNU Lesser Public License


from libc.stdlib cimport malloc, free

from cpython cimport (PyObject, PyString_FromString,
                      PyString_AsString, PyString_Check)

cdef extern from "stdlib.h":
    void memcpy(void *dst, void *src, size_t n)

cimport numpy as cnp

from numpy cimport ndarray, uint8_t, uint64_t

import numpy as np

# cdef extern from "Python.h":
#     void Py_INCREF(PyObject*)
#     void Py_XDECREF(PyObject*)

cimport util

import pandas.lib as lib

import time
{'fraction_non_alphanumeric': 0.072236727589208, 'fraction_numerical': 0.008703220191470844, 'mean_word_length': 3.831932773109244, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '620843', 'n_tokens_mistral': 389, 'n_tokens_neox': 375, 'n_words': 130}