<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
import zlib as _zlib


import numpy as np


from .abc import Codec
from .compat import buffer_copy, handle_datetime, buffer_tobytes, PY2


class GZip(Codec):
    """Codec providing gzip compression using zlib via the Python standard library.

    Parameters
    ----------
    level : int
        Compression level.

    """

    codec_id = 'gzip'

    def __init__(self, level=1):
        self.level = level

    def encode(self, buf):

        # deal with lack of buffer support for datetime64 and timedelta64
        buf = handle_datetime(buf)

        if isinstance(buf, np.ndarray):

            # cannot compress object array
            if buf.dtype == object:
                raise ValueError('cannot encode object array')

            # if numpy array, can only handle C contiguous directly
            if not buf.flags.c_contiguous:
                buf = buf.tobytes(order='A')

        if PY2:  # pragma: py3 no cover
            # ensure bytes, PY2 cannot handle things like bytearray
            buf = buffer_tobytes(buf)

        # do compression
        compress = _zlib.compressobj(self.level, _zlib.DEFLATED, 16 + 15)
        compressed = compress.compress(buf)
        compressed += compress.flush()

        return compressed

    # noinspection PyMethodMayBeStatic
    def decode(self, buf, out=None):

        if PY2:  # pragma: py3 no cover
            # ensure bytes, PY2 cannot handle things like bytearray
            buf = buffer_tobytes(buf)

        # do decompression
        dec = _zlib.decompress(buf, 15 + 32)

        # handle destination - Python standard library zlib module does not
        # support direct decompression into buffer, so we have to copy into
        # out if given
        return buffer_copy(dec, out)
<commit_msg>Fix flake8 W391 error<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
import zlib as _zlib


import numpy as np


from .abc import Codec
from .compat import buffer_copy, handle_datetime, buffer_tobytes, PY2


class GZip(Codec):
    """Codec providing gzip compression using zlib via the Python standard library.

    Parameters
    ----------
    level : int
        Compression level.

    """

    codec_id = 'gzip'

    def __init__(self, level=1):
        self.level = level

    def encode(self, buf):

        # deal with lack of buffer support for datetime64 and timedelta64
        buf = handle_datetime(buf)

        if isinstance(buf, np.ndarray):

            # cannot compress object array
            if buf.dtype == object:
                raise ValueError('cannot encode object array')

            # if numpy array, can only handle C contiguous directly
            if not buf.flags.c_contiguous:
                buf = buf.tobytes(order='A')

        if PY2:  # pragma: py3 no cover
            # ensure bytes, PY2 cannot handle things like bytearray
            buf = buffer_tobytes(buf)

        # do compression
        compress = _zlib.compressobj(self.level, _zlib.DEFLATED, 16 + 15)
        compressed = compress.compress(buf)
        compressed += compress.flush()

        return compressed

    # noinspection PyMethodMayBeStatic
    def decode(self, buf, out=None):

        if PY2:  # pragma: py3 no cover
            # ensure bytes, PY2 cannot handle things like bytearray
            buf = buffer_tobytes(buf)

        # do decompression
        dec = _zlib.decompress(buf, 15 + 32)

        # handle destination - Python standard library zlib module does not
        # support direct decompression into buffer, so we have to copy into
        # out if given
        return buffer_copy(dec, out)
{'fraction_non_alphanumeric': 0.07575353427580689, 'fraction_numerical': 0.012269938650306749, 'mean_word_length': 2.224419604471195, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2594148', 'n_tokens_mistral': 1076, 'n_tokens_neox': 1013, 'n_words': 369}