<commit_before># Monary - Copyright 2011-2014 <NAME>
# Please see the included LICENSE.TXT and NOTICE.TXT for licensing information.

import atexit
import os.path
import platform
import sys
from copy import deepcopy
from ctypes import *

PY3 = sys.version_info[0] >= 3
if PY3:
    # Python 3
    bytes_type = bytes
    string_type = str
    from urllib.parse import urlencode
else:
    # Python 2.6/2.7
    bytes_type = basestring
    string_type = basestring
    from urllib import urlencode

try:
    # if we are using Python 2.7+
    from collections import OrderedDict
except ImportError:
    # for Python 2.6 and earlier
    from .ordereddict import OrderedDict

import numpy
import bson

cmonary = None

ERROR_LEN = 504
ERROR_ARR = c_char * ERROR_LEN


class bson_error_t(Structure):
    _fields_ = [
        ("domain", c_uint),
        ("code", c_uint),
        ("message", ERROR_ARR)
    ]


class MonaryError(RuntimeError):
    pass


def _load_cmonary_lib():
    """Loads the cmonary CDLL library (from the directory containing this module)."""
    global cmonary
    thismodule = __file__
    abspath = os.path.abspath(thismodule)
    moduledir = list(os.path.split(abspath))[:-1]
    if platform.system() == 'Windows':
        libbson = CDLL(os.path.join(*(moduledir + ['libbson-1.0.dll'])))<commit_msg>changed parent of MonaryError from RuntimeError to Exception<commit_after># Monary - Copyright 2011-2014 <NAME>
# Please see the included LICENSE.TXT and NOTICE.TXT for licensing information.

import atexit
import os.path
import platform
import sys
from copy import deepcopy
from ctypes import *

PY3 = sys.version_info[0] >= 3
if PY3:
    # Python 3
    bytes_type = bytes
    string_type = str
    from urllib.parse import urlencode
else:
    # Python 2.6/2.7
    bytes_type = basestring
    string_type = basestring
    from urllib import urlencode

try:
    # if we are using Python 2.7+
    from collections import OrderedDict
except ImportError:
    # for Python 2.6 and earlier
    from .ordereddict import OrderedDict

import numpy
import bson

cmonary = None

ERROR_LEN = 504
ERROR_ARR = c_char * ERROR_LEN


class bson_error_t(Structure):
    _fields_ = [
        ("domain", c_uint),
        ("code", c_uint),
        ("message", ERROR_ARR)
    ]


class MonaryError(Exception):
    pass


def _load_cmonary_lib():
    """Loads the cmonary CDLL library (from the directory containing this module)."""
    global cmonary
    thismodule = __file__
    abspath = os.path.abspath(thismodule)
    moduledir = list(os.path.split(abspath))[:-1]
    if platform.system() == 'Windows':
        libbson = CDLL(os.path.join(*(moduledir + ['libbson-1.0.dll'])))
{'fraction_non_alphanumeric': 0.09349289454001496, 'fraction_numerical': 0.02019446522064323, 'mean_word_length': 3.3006430868167205, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1825627', 'n_tokens_mistral': 924, 'n_tokens_neox': 873, 'n_words': 262}