<commit_before># coding=UTF-8
"""
Allows you to preserve entire stack frame along with all variables (even pickles them).

After this, you can send this report somewhere. At target, it will be unpickled in a safe way
(without importing anything extra from environment). You can unpickle particular variables stored,
but that may involve an import.

Use in such a way:

    try:
        ...
    except WhateverError as e:
        tb = Traceback()
        print(tb.pretty_print())
        # you can now pickle it if you wish to
"""
import sys
import zlib
import io
import six
import traceback
import inspect
try:
    import cPickle as pickle
except ImportError:
    import pickle


class GenerationPolicy(object):
    """
    A policy that manages generating a traceback

    For a "default" policy it's pretty customizable

    Override if need be, and pass the class (or instance) to Traceback
    """

    def __init__(self, enable_pickling=True, compress_at=128*1024, repr_length_limit=128*1024):
        """
        :param enable_pickling: bool, whether to enable pickling at all
        :param compress_at: pickles longer than this (bytes) will be compressed
        :param repr_length_limit: maximum length of __repr__. None for no limit.
        """
        self.enable_pickling = enable_pickling
        self.compress_at = compress_at
        self.repr_length_limit = repr_length_limit

    def should_pickle(self, value):
        """
        Should this value be pickled?

        :param value: value candidate
        :return: bool
        """
        return self.enable_pickling

    def should_compress(self, pickledata):
        """
        Should this pickle undergo compression?
        :param pickledata: bytes, pickle value
        :return: bool
        """
        return len(pickledata) > self.compress_at

    def get_compression_level(self, pickledata):
        """
        What compression level to use to pickle this?
        :param pickledata: bytes, pickle value
        :return: int, 1-9, where "1" is the fastest, and "9" is the slowest, but produces best compression
        """
        return 6

    def process_repr(self, r):
        """
        Process the string obtained from __repr__ing
        :param r: result of a __repr__ on value
        :return: processed result
        """
        if self.repr_length_limit is None:
            return r
        else:
            if len(r) > self.repr_length_limit:
                return r[:self.repr_length_limit] + u'...'
            else:
                return r


class StoredVariable(object):
    """
    Class used to store a variable value. Picklable.

    Attributes are:
        .repr - a text representation obtained using repr
        .typeinfo - a text representation of variable's type
        .pickle - bytes with pickled (optionally processed) value, or None if not available
        .pickle_type - what is stored in .pickle?
            None - nothing
            "pickle" - normal Python pickle
            "pickle/gzip" - Python pickle treated with zlib.compress
            "failed" - could not pickle, pickle contains a text with human-readable reason

    """
    __slots__ = ('repr', 'type_', 'pickle', 'pickle_type')

    def __init__(self, value, policy):
        """
        If value cannot be pickled, it's repr will be at least preserved

        :param value: any Python value to preserve
        :param policy: policy to use (instance)
        """
        self.repr = repr(value)
        self.type_ = repr(type(value))
        if six.PY2:
            self.repr = six.text_type(self.repr, 'utf8')
            self.type_ = six.text_type(self.type_, 'utf8')

        self.repr = policy.process_repr(self.repr)

        self.pickle = None
        self.pickle_type = None

        if policy.should_pickle(value):
            try:
                self.pickle = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
                self.pickle_type = 'pickle'
            except pickle.PicklingError as e:
                self.pickle = repr(e.args)
                self.pickle_type = "failed"
            else:
                if policy.should_compress(self.pickle):
                    try:
                        self.pickle = zlib.compress(self.pickle, policy.get_compression_level(self.pickle))
                        self.pickle_type = "pickle/gzip"
                    except zlib.error:
                        pass    # ok, keep normal

    def load_value(self):
        """
        Return the value that this represents.

        WARNING! This may result in importing things from environment, as pickle.loads will be called.

        :return: stored value - if picklable and was pickled
        :raises ValueError: value has failed to be pickled or was never pickled
        """
        if self.pickle_type is None:
            raise ValueError('value was never pickled')
        elif self.pickle_type == 'failed':
            raise ValueError('Value has failed to be pickled, reason is %s' % (self.pickle, ))<commit_msg>coverage<commit_after># coding=UTF-8
"""
Allows you to preserve entire stack frame along with all variables (even pickles them).

After this, you can send this report somewhere. At target, it will be unpickled in a safe way
(without importing anything extra from environment). You can unpickle particular variables stored,
but that may involve an import.

Use in such a way:

    try:
        ...
    except WhateverError as e:
        tb = Traceback()
        print(tb.pretty_print())
        # you can now pickle it if you wish to
"""
import sys
import zlib
import io
import six
import traceback
import inspect
try:
    import cPickle as pickle
except ImportError:
    import pickle


class GenerationPolicy(object):
    """
    A policy that manages generating a traceback

    For a "default" policy it's pretty customizable

    Override if need be, and pass the class (or instance) to Traceback
    """

    def __init__(self, enable_pickling=True, compress_at=128*1024, repr_length_limit=128*1024):
        """
        :param enable_pickling: bool, whether to enable pickling at all
        :param compress_at: pickles longer than this (bytes) will be compressed
        :param repr_length_limit: maximum length of __repr__. None for no limit.
        """
        self.enable_pickling = enable_pickling
        self.compress_at = compress_at
        self.repr_length_limit = repr_length_limit

    def should_pickle(self, value):
        """
        Should this value be pickled?

        :param value: value candidate
        :return: bool
        """
        return self.enable_pickling

    def should_compress(self, pickledata):
        """
        Should this pickle undergo compression?
        :param pickledata: bytes, pickle value
        :return: bool
        """
        return len(pickledata) > self.compress_at

    def get_compression_level(self, pickledata):
        """
        What compression level to use to pickle this?
        :param pickledata: bytes, pickle value
        :return: int, 1-9, where "1" is the fastest, and "9" is the slowest, but produces best compression
        """
        return 6

    def process_repr(self, r):
        """
        Process the string obtained from __repr__ing
        :param r: result of a __repr__ on value
        :return: processed result
        """
        if self.repr_length_limit is None:
            return r
        else:
            if len(r) > self.repr_length_limit:
                return r[:self.repr_length_limit] + u'...'
            else:
                return r


class StoredVariable(object):
    """
    Class used to store a variable value. Picklable.

    Attributes are:
        .repr - a text representation obtained using repr
        .typeinfo - a text representation of variable's type
        .pickle - bytes with pickled (optionally processed) value, or None if not available
        .pickle_type - what is stored in .pickle?
            None - nothing
            "pickle" - normal Python pickle
            "pickle/gzip" - Python pickle treated with zlib.compress
            "failed" - could not pickle, pickle contains a text with human-readable reason

    """
    __slots__ = ('repr', 'type_', 'pickle', 'pickle_type')

    def __init__(self, value, policy):
        """
        If value cannot be pickled, it's repr will be at least preserved

        :param value: any Python value to preserve
        :param policy: policy to use (instance)
        """
        self.repr = repr(value)
        self.type_ = repr(type(value))
        if six.PY2:
            self.repr = six.text_type(self.repr, 'utf8')
            self.type_ = six.text_type(self.type_, 'utf8')

        self.repr = policy.process_repr(self.repr)

        self.pickle = None
        self.pickle_type = None

        if policy.should_pickle(value):
            try:
                self.pickle = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
                self.pickle_type = 'pickle'
            except (pickle.PicklingError, TypeError) as e:
                self.pickle = repr(e.args)
                self.pickle_type = "failed"
            else:
                if policy.should_compress(self.pickle):
                    try:
                        self.pickle = zlib.compress(self.pickle, policy.get_compression_level(self.pickle))
                        self.pickle_type = "pickle/gzip"
                    except zlib.error:
                        pass    # ok, keep normal

    def load_value(self):
        """
        Return the value that this represents.

        WARNING! This may result in importing things from environment, as pickle.loads will be called.

        :return: stored value - if picklable and was pickled
        :raises ValueError: value has failed to be pickled or was never pickled
        """
        if self.pickle_type is None:
            raise ValueError('value was never pickled')
        elif self.pickle_type == 'failed':
            raise ValueError('Value has failed to be pickled, reason is %s' % (self.pickle, ))
{'fraction_non_alphanumeric': 0.07830845771144279, 'fraction_numerical': 0.004577114427860697, 'mean_word_length': 2.1175558312655087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3000167', 'n_tokens_mistral': 2857, 'n_tokens_neox': 2730, 'n_words': 969}