<commit_before># imagecodecs/_deflate.pyx
# distutils: language = c
# cython: language_level = 3
# cython: boundscheck=False
# cython: wraparound=False
# cython: cdivision=True
# cython: nonecheck=False

# Copyright (c) 2020-2021, <NAME>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""Deflate and GZIP codecs for the imagecodecs package."""

__version__ = '2020.12.22'

include '_shared.pxi'

from libdeflate cimport *


class DEFLATE:
    """Libdeflate Constants."""


class DeflateError(RuntimeError):
    """Libdeflate Exceptions."""

    def __init__(self, func, err):
        msg = {
            LIBDEFLATE_SUCCESS: 'LIBDEFLATE_SUCCESS',
            LIBDEFLATE_BAD_DATA: 'LIBDEFLATE_BAD_DATA',
            LIBDEFLATE_SHORT_OUTPUT: 'LIBDEFLATE_SHORT_OUTPUT',
            LIBDEFLATE_INSUFFICIENT_SPACE: 'LIBDEFLATE_INSUFFICIENT_SPACE',
        }.get(err, f'unknown error {err!r}')
        msg = f'{func} returned {msg}'
        super().__init__(msg)


def deflate_version():
    """Return libdeflate library version string."""
    return 'libdeflate ' + LIBDEFLATE_VERSION_STRING.decode()


def deflate_check(data):
    """Return True if data likely contains Deflate/Zlib data."""


def deflate_encode(data, level=None, bint raw=False, out=None):
    """Compress Deflate/Zlib.

    """
    cdef:
        const uint8_t[::1] src = _readable_input(data)
        const uint8_t[::1] dst  # must be const to write to bytes
        ssize_t srcsize = src.size
        ssize_t dstsize
        size_t srclen, dstlen
        libdeflate_compressor* compressor = NULL
        int compression_level = _default_value(level, 6, 0, 12)

    if data is out:
        raise ValueError('cannot encode in-place')

    out, dstsize, outgiven, outtype = _parse_output(out)

    compressor = libdeflate_alloc_compressor(compression_level)
    if compressor == NULL:
        raise DeflateError('libdeflate_alloc_compressor', 'NULL')

    try:
        if out is None:
            if dstsize < 0:
                if raw:
                    dstlen = libdeflate_deflate_compress_bound(
                        compressor, <size_t> srcsize
                    )
                    if dstlen == 0:
                        raise DeflateError(
                            'libdeflate_deflate_compress_bound', '0'
                        )
                else:
                    dstlen = libdeflate_zlib_compress_bound(
                        compressor, <size_t> srcsize
                    )
                    if dstlen == 0:
                        raise DeflateError(
                            'libdeflate_zlib_compress_bound', '0'
                        )
                dstsize = <ssize_t> dstlen
            out = _create_output(outtype, dstsize)

        dst = out
        dstsize = dst.size
        dstlen = <size_t> dstsize
        srclen = <size_t> srcsize

        with nogil:
            if raw:
                dstlen = libdeflate_deflate_compress(
                    compressor,
                    <const void*> &src[0],
                    srclen,
                    <void*> &dst[0],
                    dstlen
                )
                if dstlen == 0:
                    raise DeflateError('libdeflate_deflate_compress', '0')
            else:
                dstlen = libdeflate_zlib_compress(
                    compressor,
                    <const void*> &src[0],
                    srclen,
                    <void*> &dst[0],
                    dstlen
                )
                if dstlen == 0:
                    raise DeflateError('libdeflate_zlib_compress', '0')
    finally:
        if compressor != NULL:
            libdeflate_free_compressor(compressor)

    del dst
    return _return_output(out, dstsize, dstlen, outgiven)


def deflate_decode(data, bint raw=False, out=None):
    """Decompress Deflate/Zlib.

    """
    cdef:
        const uint8_t[::1] src = data
        const uint8_t[::1] dst  # must be const to write to bytes
        ssize_t srcsize = src.size
        ssize_t dstsize
        size_t srclen, dstlen
        size_t actual_out_nbytes_ret
        libdeflate_result ret
        libdeflate_decompressor* decompressor = NULL

    if data is out:
        raise ValueError('cannot decode in-place')

    out, dstsize, outgiven, outtype = _parse_output(out)

    if out is None:
        if dstsize < 0:
            if raw:
                raise NotImplementedError

            # use Python's zlib module if output size is unknwon
            import zlib

            return zlib.decompress(data)

        out = _create_output(outtype, dstsize)

    dst = out
    dstsize = dst.size
    dstlen = <size_t> dstsize
    srclen = <size_t> srcsize

    try:
        with nogil:
            decompressor = libdeflate_alloc_decompressor()
            if decompressor == NULL:
                raise DeflateError('libdeflate_alloc_decompressor', 'NULL')

            if raw:
                ret = libdeflate_deflate_decompress(
                    decompressor,
                    <const void*> &src[0],
                    srclen,
                    <void*> &dst[0],
                    dstlen,
                    &actual_out_nbytes_ret
                )<commit_msg>Update imagecodecs/_deflate.pyx<commit_after># imagecodecs/_deflate.pyx
# distutils: language = c
# cython: language_level = 3
# cython: boundscheck=False
# cython: wraparound=False
# cython: cdivision=True
# cython: nonecheck=False

# Copyright (c) 2020-2021, <NAME>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""Deflate and GZIP codecs for the imagecodecs package."""

__version__ = '2020.12.22'

include '_shared.pxi'

from libdeflate cimport *


class DEFLATE:
    """Libdeflate Constants."""


class DeflateError(RuntimeError):
    """Libdeflate Exceptions."""

    def __init__(self, func, err):
        msg = {
            LIBDEFLATE_SUCCESS: 'LIBDEFLATE_SUCCESS',
            LIBDEFLATE_BAD_DATA: 'LIBDEFLATE_BAD_DATA',
            LIBDEFLATE_SHORT_OUTPUT: 'LIBDEFLATE_SHORT_OUTPUT',
            LIBDEFLATE_INSUFFICIENT_SPACE: 'LIBDEFLATE_INSUFFICIENT_SPACE',
        }.get(err, f'unknown error {err!r}')
        msg = f'{func} returned {msg}'
        super().__init__(msg)


def deflate_version():
    """Return libdeflate library version string."""
    return 'libdeflate ' + LIBDEFLATE_VERSION_STRING.decode()


def deflate_check(data):
    """Return True if data likely contains Deflate/Zlib data."""


def deflate_encode(data, level=None, bint raw=False, out=None):
    """Compress Deflate/Zlib.

    """
    cdef:
        const uint8_t[::1] src = _readable_input(data)
        const uint8_t[::1] dst  # must be const to write to bytes
        ssize_t srcsize = src.size
        ssize_t dstsize
        size_t srclen, dstlen
        libdeflate_compressor* compressor = NULL
        int compression_level = _default_value(level, 6, 0, 12)

    if data is out:
        raise ValueError('cannot encode in-place')

    out, dstsize, outgiven, outtype = _parse_output(out)

    compressor = libdeflate_alloc_compressor(compression_level)
    if compressor == NULL:
        raise DeflateError('libdeflate_alloc_compressor', 'NULL')

    try:
        if out is None:
            if dstsize < 0:
                if raw:
                    dstlen = libdeflate_deflate_compress_bound(
                        compressor, <size_t> srcsize
                    )
                    if dstlen == 0:
                        raise DeflateError(
                            'libdeflate_deflate_compress_bound', '0'
                        )
                else:
                    dstlen = libdeflate_zlib_compress_bound(
                        compressor, <size_t> srcsize
                    )
                    if dstlen == 0:
                        raise DeflateError(
                            'libdeflate_zlib_compress_bound', '0'
                        )
                dstsize = <ssize_t> dstlen
            out = _create_output(outtype, dstsize)

        dst = out
        dstsize = dst.size
        dstlen = <size_t> dstsize
        srclen = <size_t> srcsize

        with nogil:
            if raw:
                dstlen = libdeflate_deflate_compress(
                    compressor,
                    <const void*> &src[0],
                    srclen,
                    <void*> &dst[0],
                    dstlen
                )
                if dstlen == 0:
                    raise DeflateError('libdeflate_deflate_compress', '0')
            else:
                dstlen = libdeflate_zlib_compress(
                    compressor,
                    <const void*> &src[0],
                    srclen,
                    <void*> &dst[0],
                    dstlen
                )
                if dstlen == 0:
                    raise DeflateError('libdeflate_zlib_compress', '0')
    finally:
        if compressor != NULL:
            libdeflate_free_compressor(compressor)

    del dst
    return _return_output(out, dstsize, dstlen, outgiven)


def deflate_decode(data, bint raw=False, out=None):
    """Decompress Deflate/Zlib.

    """
    cdef:
        const uint8_t[::1] src = data
        const uint8_t[::1] dst  # must be const to write to bytes
        ssize_t srcsize = src.size
        ssize_t dstsize
        size_t srclen, dstlen
        size_t actual_out_nbytes_ret
        libdeflate_result ret
        libdeflate_decompressor* decompressor = NULL

    if data is out:
        raise ValueError('cannot decode in-place')

    out, dstsize, outgiven, outtype = _parse_output(out)

    if out is None:
        if dstsize < 0:
            if raw:
                raise NotImplementedError  # TODO

            # use Python's zlib module if output size is unknwon
            import zlib

            return zlib.decompress(data)

        out = _create_output(outtype, dstsize)

    dst = out
    dstsize = dst.size
    dstlen = <size_t> dstsize
    srclen = <size_t> srcsize

    try:
        with nogil:
            decompressor = libdeflate_alloc_decompressor()
            if decompressor == NULL:
                raise DeflateError('libdeflate_alloc_decompressor', 'NULL')

            if raw:
                ret = libdeflate_deflate_decompress(
                    decompressor,
                    <const void*> &src[0],
                    srclen,
                    <void*> &dst[0],
                    dstlen,
                    &actual_out_nbytes_ret
                )
{'fraction_non_alphanumeric': 0.07446967433522557, 'fraction_numerical': 0.00717060053779504, 'mean_word_length': 2.033990482664854, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5797428', 'n_tokens_mistral': 4023, 'n_tokens_neox': 3702, 'n_words': 1154}