<commit_before>import ctypes
import numpy
import sys
import os
import os.path
from numpy.compat import asbytes, asstr


def _generate_candidate_libs():
    # look for likely library files in the following dirs:
    lib_dirs = [os.path.dirname(__file__),
                '/lib',
                '/usr/lib',
                '/usr/local/lib',
                '/opt/local/lib',
                os.path.join(sys.prefix, 'lib'),
                os.path.join(sys.prefix, 'DLLs')
                ]
    if 'HOME' in os.environ:
        lib_dirs.append(os.path.join(os.environ['HOME'], 'lib'))
    lib_dirs = [ld for ld in lib_dirs if os.path.exists(ld)]

    lib_names = ['libfreeimage', 'freeimage']  # should be lower-case!
    # Now attempt to find libraries of that name in the given directory
    # (case-insensitive and without regard for extension)
    lib_paths = []
    for lib_dir in lib_dirs:
        for lib_name in lib_names:
            files = os.listdir(lib_dir)
            lib_paths += [os.path.join(lib_dir, lib) for lib in files
                          if lib.lower().startswith(lib_name) and not
                          os.path.splitext(lib)[1] in ('.py', '.pyc', '.ini')]
    lib_paths = [lp for lp in lib_paths if os.path.exists(lp)]

    return lib_dirs, lib_paths

if sys.platform == 'win32':
    LOADER = ctypes.windll
    FUNCTYPE = ctypes.WINFUNCTYPE
else:
    LOADER = ctypes.cdll
    FUNCTYPE = ctypes.CFUNCTYPE


def handle_errors():
    global FT_ERROR_STR
    if FT_ERROR_STR:
        tmp = FT_ERROR_STR
        FT_ERROR_STR = None
        raise RuntimeError(tmp)


FT_ERROR_STR = None


# This MUST happen in module scope, or the function pointer is garbage
# collected, leading to a segfault when error_handler is called.
@FUNCTYPE(None, ctypes.c_int, ctypes.c_char_p)
def c_error_handler(fif, message):
    global FT_ERROR_STR
    FT_ERROR_STR = 'FreeImage error: %s' % message


def load_freeimage():
    freeimage = None
    errors = []
    # First try a few bare library names that ctypes might be able to find
    # in the default locations for each platform. Win DLL names don't need the
    # extension, but other platforms do.
    bare_libs = ['FreeImage', 'libfreeimage.dylib', 'libfreeimage.so',
                 'libfreeimage.so.3']
    lib_dirs, lib_paths = _generate_candidate_libs()
    lib_paths = bare_libs + lib_paths
    for lib in lib_paths:
        try:
            freeimage = LOADER.LoadLibrary(lib)
            break
        except Exception:
            if lib not in bare_libs:
                # Don't record errors when it couldn't load the library from
                # a bare name -- this fails often, and doesn't provide any
                # useful debugging information anyway, beyond "couldn't find
                # library..."
                # Get exception instance in Python 2.x/3.x compatible manner
                e_type, e_value, e_tb = sys.exc_info()
                del e_tb
                errors.append((lib, e_value))

    if freeimage is None:
        if errors:
            # No freeimage library loaded, and load-errors reported for some
            # candidate libs
            err_txt = ['%s:\n%s' % (l, str(e)) for l, e in errors]
            raise RuntimeError('One or more FreeImage libraries were found, '
                               'but could not be loaded due to the following '
                               'errors:\n\n\n'.join(err_txt))
        else:
            # No errors, because no potential libraries found at all!
            raise RuntimeError('Could not find a FreeImage library in any of:'
                               '\n\n'.join(lib_dirs))

    # FreeImage found
    freeimage.FreeImage_SetOutputMessage(c_error_handler)
    return freeimage

_FI = load_freeimage()

API = {
    # All we're doing here is telling ctypes that some of the FreeImage
    # functions return pointers instead of integers. (On 64-bit systems,
    # without this information the pointers get truncated and crashes result).
    # There's no need to list functions that return ints, or the types of the
    # parameters to these or other functions -- that's fine to do implicitly.

    # Note that the ctypes immediately converts the returned void_p back to a
    # python int again! This is really not helpful, because then passing it
    # back to another library call will cause truncation-to-32-bits on 64-bit
    # systems. Thanks, ctypes! So after these calls one must immediately
    # re-wrap the int as a c_void_p if it is to be passed back into FreeImage.
    'FreeImage_AllocateT': (ctypes.c_void_p, None),
    'FreeImage_FindFirstMetadata': (ctypes.c_void_p, None),
    'FreeImage_GetBits': (ctypes.c_void_p, None),
    'FreeImage_GetPalette': (ctypes.c_void_p, None),
    'FreeImage_GetTagKey': (ctypes.c_char_p, None),
    'FreeImage_GetTagValue': (ctypes.c_void_p, None),
    'FreeImage_Load': (ctypes.c_void_p, None),
    'FreeImage_LockPage': (ctypes.c_void_p, None),
    'FreeImage_OpenMultiBitmap': (ctypes.c_void_p, None)<commit_msg>Fix display problem when printing error messages<commit_after>import ctypes
import numpy
import sys
import os
import os.path
from numpy.compat import asbytes, asstr


def _generate_candidate_libs():
    # look for likely library files in the following dirs:
    lib_dirs = [os.path.dirname(__file__),
                '/lib',
                '/usr/lib',
                '/usr/local/lib',
                '/opt/local/lib',
                os.path.join(sys.prefix, 'lib'),
                os.path.join(sys.prefix, 'DLLs')
                ]
    if 'HOME' in os.environ:
        lib_dirs.append(os.path.join(os.environ['HOME'], 'lib'))
    lib_dirs = [ld for ld in lib_dirs if os.path.exists(ld)]

    lib_names = ['libfreeimage', 'freeimage']  # should be lower-case!
    # Now attempt to find libraries of that name in the given directory
    # (case-insensitive and without regard for extension)
    lib_paths = []
    for lib_dir in lib_dirs:
        for lib_name in lib_names:
            files = os.listdir(lib_dir)
            lib_paths += [os.path.join(lib_dir, lib) for lib in files
                          if lib.lower().startswith(lib_name) and not
                          os.path.splitext(lib)[1] in ('.py', '.pyc', '.ini')]
    lib_paths = [lp for lp in lib_paths if os.path.exists(lp)]

    return lib_dirs, lib_paths

if sys.platform == 'win32':
    LOADER = ctypes.windll
    FUNCTYPE = ctypes.WINFUNCTYPE
else:
    LOADER = ctypes.cdll
    FUNCTYPE = ctypes.CFUNCTYPE


def handle_errors():
    global FT_ERROR_STR
    if FT_ERROR_STR:
        tmp = FT_ERROR_STR
        FT_ERROR_STR = None
        raise RuntimeError(tmp)


FT_ERROR_STR = None


# This MUST happen in module scope, or the function pointer is garbage
# collected, leading to a segfault when error_handler is called.
@FUNCTYPE(None, ctypes.c_int, ctypes.c_char_p)
def c_error_handler(fif, message):
    global FT_ERROR_STR
    FT_ERROR_STR = 'FreeImage error: %s' % message


def load_freeimage():
    freeimage = None
    errors = []
    # First try a few bare library names that ctypes might be able to find
    # in the default locations for each platform. Win DLL names don't need the
    # extension, but other platforms do.
    bare_libs = ['FreeImage', 'libfreeimage.dylib', 'libfreeimage.so',
                 'libfreeimage.so.3']
    lib_dirs, lib_paths = _generate_candidate_libs()
    lib_paths = bare_libs + lib_paths
    for lib in lib_paths:
        try:
            freeimage = LOADER.LoadLibrary(lib)
            break
        except Exception:
            if lib not in bare_libs:
                # Don't record errors when it couldn't load the library from
                # a bare name -- this fails often, and doesn't provide any
                # useful debugging information anyway, beyond "couldn't find
                # library..."
                # Get exception instance in Python 2.x/3.x compatible manner
                e_type, e_value, e_tb = sys.exc_info()
                del e_tb
                errors.append((lib, e_value))

    if freeimage is None:
        if errors:
            # No freeimage library loaded, and load-errors reported for some
            # candidate libs
            err_txt = ['%s:\n%s' % (l, str(e)) for l, e in errors]
            raise RuntimeError('One or more FreeImage libraries were found, '
                               'but could not be loaded due to the following '
                               'errors:\n',
                               '\n\n'.join(err_txt))
        else:
            # No errors, because no potential libraries found at all!
            raise RuntimeError('Could not find a FreeImage library in any of:'
                               '\n', 
                               '\n\n'.join(lib_dirs))

    # FreeImage found
    freeimage.FreeImage_SetOutputMessage(c_error_handler)
    return freeimage

_FI = load_freeimage()

API = {
    # All we're doing here is telling ctypes that some of the FreeImage
    # functions return pointers instead of integers. (On 64-bit systems,
    # without this information the pointers get truncated and crashes result).
    # There's no need to list functions that return ints, or the types of the
    # parameters to these or other functions -- that's fine to do implicitly.

    # Note that the ctypes immediately converts the returned void_p back to a
    # python int again! This is really not helpful, because then passing it
    # back to another library call will cause truncation-to-32-bits on 64-bit
    # systems. Thanks, ctypes! So after these calls one must immediately
    # re-wrap the int as a c_void_p if it is to be passed back into FreeImage.
    'FreeImage_AllocateT': (ctypes.c_void_p, None),
    'FreeImage_FindFirstMetadata': (ctypes.c_void_p, None),
    'FreeImage_GetBits': (ctypes.c_void_p, None),
    'FreeImage_GetPalette': (ctypes.c_void_p, None),
    'FreeImage_GetTagKey': (ctypes.c_char_p, None),
    'FreeImage_GetTagValue': (ctypes.c_void_p, None),
    'FreeImage_Load': (ctypes.c_void_p, None),
    'FreeImage_LockPage': (ctypes.c_void_p, None),
    'FreeImage_OpenMultiBitmap': (ctypes.c_void_p, None)
{'fraction_non_alphanumeric': 0.08770379100373207, 'fraction_numerical': 0.002357100766057749, 'mean_word_length': 2.3485695494902994, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5203056', 'n_tokens_mistral': 3098, 'n_tokens_neox': 2991, 'n_words': 995}