<commit_before>from libc.stdint cimport int64_t, uint8_t, uint64_t

from fractions import Fraction
from threading import local
import sys
import traceback

cimport libav as lib

from av.logging cimport get_last_error


# === ERROR HANDLING ===
# ======================

# Would love to use the built-in constant, but it doesn't appear to
# exist on Travis, or my Linux workstation. Could this be because they
# are actually libav?
cdef int AV_ERROR_MAX_STRING_SIZE = 64

# Our custom error.
cdef int PYAV_ERROR = -0x50794156 # 'PyAV'


class AVError(EnvironmentError):
    """Exception class for errors from within FFmpeg."""
    def __init__(self, code, message, filename=None, log=None):
        if filename:
            super(AVError, self).__init__(code, message, filename)
        else:
            super(AVError, self).__init__(code, message)
        self.log = log
    def __str__(self):
        strerror = super(AVError, self).__str__()
        if self.log:
            return '%s (%s: %s)' % (strerror, self.log[0], self.log[1])
        else:
            return strerror
AVError.__module__ = 'av'


cdef object _local = local()
cdef int _err_count = 0

cdef int stash_exception(exc_info=None):

    global _err_count

    existing = getattr(_local, 'exc_info', None)
    if existing is not None:
        print >> sys.stderr, 'PyAV library exception being dropped:'
        traceback.print_exception(*existing)
        _err_count -= 1

    exc_info = exc_info or sys.exc_info()
    _local.exc_info = exc_info
    if exc_info:
        _err_count += 1

    return PYAV_ERROR


cdef int _last_log_count = 0

cpdef int err_check(int res=0, filename=None) except -1:

    global _err_count
    global _last_log_count

    # Check for stashed exceptions.
    if _err_count:
        exc_info = getattr(_local, 'exc_info', None)
        if exc_info is not None:
            _err_count -= 1
            _local.exc_info = None
            raise exc_info[0], exc_info[1], exc_info[2]

    if res >= 0:
        return res

    cdef bytes py_buffer
    cdef char *c_buffer


    if res == PYAV_ERROR:
        py_buffer = b'Error in PyAV callback'

    else:
        # This is kinda gross.
        py_buffer = b"\0" * AV_ERROR_MAX_STRING_SIZE
        c_buffer = py_buffer
        lib.av_strerror(res, c_buffer, AV_ERROR_MAX_STRING_SIZE)
        py_buffer = c_buffer
    cdef unicode message = py_buffer.decode('latin1')

    # Add details from the last log onto the end.
    log_count, last_log = get_last_error()
    if log_count > _last_log_count:
        _last_log_count = log_count
        log = last_log
    else:
        log = None

    if filename:
        raise AVError(-res, message, filename, log)
    else:
        raise AVError(-res, message, None,     log)

    return res



# === DICTIONARIES ===
# ====================

cdef bint _py3k = str is unicode

cdef _decode(char *s, encoding, errors):
    if encoding is None:
        return s
    else:
        return (<bytes>s).decode(encoding, errors)

cdef bytes _encode(s, encoding, errors):
    if isinstance(s, unicode):
        return s.encode(encoding, errors)
    return s

cdef dict avdict_to_dict(lib.AVDictionary *input, str encoding=None, str errors='strict'):
    if _py3k and encoding is None:
        encoding = 'utf8'

    cdef lib.AVDictionaryEntry *element = NULL
    cdef dict output = {}
    while True:
        element = lib.av_dict_get(input, "", element, lib.AV_DICT_IGNORE_SUFFIX)
        if element == NULL:
            break
        output[_decode(element.key, encoding, errors)] = _decode(element.value, encoding, errors)
    return output


cdef dict_to_avdict(lib.AVDictionary **dst, dict src, bint clear=True, str encoding=None, str errors='strict'):
    if clear:
        lib.av_dict_free(dst)
    if encoding is None:
        encoding = 'utf8'
    for key, value in src.items():
        err_check(lib.av_dict_set(dst, _encode(key, encoding, errors),
                                  _encode(value, encoding, errors), 0))


<commit_msg>Deterministically perform bytes -> unicode decoding<commit_after>from libc.stdint cimport int64_t, uint8_t, uint64_t

from fractions import Fraction
from threading import local
import sys
import traceback

cimport libav as lib

from av.logging cimport get_last_error


# === ERROR HANDLING ===
# ======================

# Would love to use the built-in constant, but it doesn't appear to
# exist on Travis, or my Linux workstation. Could this be because they
# are actually libav?
cdef int AV_ERROR_MAX_STRING_SIZE = 64

# Our custom error.
cdef int PYAV_ERROR = -0x50794156 # 'PyAV'


class AVError(EnvironmentError):
    """Exception class for errors from within FFmpeg."""
    def __init__(self, code, message, filename=None, log=None):
        if filename:
            super(AVError, self).__init__(code, message, filename)
        else:
            super(AVError, self).__init__(code, message)
        self.log = log
    def __str__(self):
        strerror = super(AVError, self).__str__()
        if self.log:
            return '%s (%s: %s)' % (strerror, self.log[0], self.log[1])
        else:
            return strerror
AVError.__module__ = 'av'


cdef object _local = local()
cdef int _err_count = 0

cdef int stash_exception(exc_info=None):

    global _err_count

    existing = getattr(_local, 'exc_info', None)
    if existing is not None:
        print >> sys.stderr, 'PyAV library exception being dropped:'
        traceback.print_exception(*existing)
        _err_count -= 1

    exc_info = exc_info or sys.exc_info()
    _local.exc_info = exc_info
    if exc_info:
        _err_count += 1

    return PYAV_ERROR


cdef int _last_log_count = 0

cpdef int err_check(int res=0, filename=None) except -1:

    global _err_count
    global _last_log_count

    # Check for stashed exceptions.
    if _err_count:
        exc_info = getattr(_local, 'exc_info', None)
        if exc_info is not None:
            _err_count -= 1
            _local.exc_info = None
            raise exc_info[0], exc_info[1], exc_info[2]

    if res >= 0:
        return res

    cdef bytes py_buffer
    cdef char *c_buffer


    if res == PYAV_ERROR:
        py_buffer = b'Error in PyAV callback'

    else:
        # This is kinda gross.
        py_buffer = b"\0" * AV_ERROR_MAX_STRING_SIZE
        c_buffer = py_buffer
        lib.av_strerror(res, c_buffer, AV_ERROR_MAX_STRING_SIZE)
        py_buffer = c_buffer
    cdef unicode message = py_buffer.decode('latin1')

    # Add details from the last log onto the end.
    log_count, last_log = get_last_error()
    if log_count > _last_log_count:
        _last_log_count = log_count
        log = last_log
    else:
        log = None

    if filename:
        raise AVError(-res, message, filename, log)
    else:
        raise AVError(-res, message, None,     log)

    return res



# === DICTIONARIES ===
# ====================

cdef _decode(char *s, encoding, errors):
    return (<bytes>s).decode(encoding, errors)

cdef bytes _encode(s, encoding, errors):
    if isinstance(s, unicode):
        return s.encode(encoding, errors)
    return s

cdef dict avdict_to_dict(lib.AVDictionary *input, str encoding=None, str errors='strict'):
    if encoding is None:
        encoding = 'utf8'

    cdef lib.AVDictionaryEntry *element = NULL
    cdef dict output = {}
    while True:
        element = lib.av_dict_get(input, "", element, lib.AV_DICT_IGNORE_SUFFIX)
        if element == NULL:
            break
        output[_decode(element.key, encoding, errors)] = _decode(element.value, encoding, errors)
    return output


cdef dict_to_avdict(lib.AVDictionary **dst, dict src, bint clear=True, str encoding=None, str errors='strict'):
    if clear:
        lib.av_dict_free(dst)
    if encoding is None:
        encoding = 'utf8'
    for key, value in src.items():
        err_check(lib.av_dict_set(dst, _encode(key, encoding, errors),
                                  _encode(value, encoding, errors), 0))



{'fraction_non_alphanumeric': 0.10107715430861723, 'fraction_numerical': 0.00876753507014028, 'mean_word_length': 2.734798877455566, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3891776', 'n_tokens_mistral': 2606, 'n_tokens_neox': 2494, 'n_words': 747}