<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2008 <NAME> (john -at- paulett.org)
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.
import types
import jsonpickle.util as util
import jsonpickle.tags as tags
import jsonpickle.handlers as handlers


class Pickler(object):
    """Converts a Python object to a JSON representation.

    Setting unpicklable to False removes the ability to regenerate
    the objects into object types beyond what the standard
    simplejson library supports.

    Setting max_depth to a negative number means there is no
    limit to the depth jsonpickle should recurse into an
    object.  Setting it to zero or higher places a hard limit
    on how deep jsonpickle recurses into objects, dictionaries, etc.

    >>> p = Pickler()
    >>> p.flatten('hello world')
    'hello world'
    """

    def __init__(self, unpicklable=True, max_depth=None):
        self.unpicklable = unpicklable
        ## The current recursion depth
        self._depth = -1
        ## The maximal recursion depth
        self._max_depth = max_depth
        ## Maps id(obj) to reference names
        self._objs = {}
        ## The namestack grows whenever we recurse into a child object
        self._namestack = []

    def _reset(self):
        self._objs = {}
        self._namestack = []

    def _push(self):
        """Steps down one level in the namespace.
        """
        self._depth += 1

    def _pop(self, value):
        """Step up one level in the namespace and return the value.
        If we're at the root, reset the pickler's state.
        """
        self._depth -= 1
        if self._depth == -1:
            self._reset()
        return value

    def _mkref(self, obj):
        objid = id(obj)
        if objid not in self._objs:<commit_msg>Don't use py/ref when unpicklable=False.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2008 <NAME> (john -at- paulett.org)
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.
import types
import jsonpickle.util as util
import jsonpickle.tags as tags
import jsonpickle.handlers as handlers


class Pickler(object):
    """Converts a Python object to a JSON representation.

    Setting unpicklable to False removes the ability to regenerate
    the objects into object types beyond what the standard
    simplejson library supports.

    Setting max_depth to a negative number means there is no
    limit to the depth jsonpickle should recurse into an
    object.  Setting it to zero or higher places a hard limit
    on how deep jsonpickle recurses into objects, dictionaries, etc.

    >>> p = Pickler()
    >>> p.flatten('hello world')
    'hello world'
    """

    def __init__(self, unpicklable=True, max_depth=None):
        self.unpicklable = unpicklable
        ## The current recursion depth
        self._depth = -1
        ## The maximal recursion depth
        self._max_depth = max_depth
        ## Maps id(obj) to reference names
        self._objs = {}
        ## The namestack grows whenever we recurse into a child object
        self._namestack = []

    def _reset(self):
        self._objs = {}
        self._namestack = []

    def _push(self):
        """Steps down one level in the namespace.
        """
        self._depth += 1

    def _pop(self, value):
        """Step up one level in the namespace and return the value.
        If we're at the root, reset the pickler's state.
        """
        self._depth -= 1
        if self._depth == -1:
            self._reset()
        return value

    def _mkref(self, obj):
        # Do not use references if not unpicklable.
        if self.unpicklable is False:
            return True
        objid = id(obj)
        if objid not in self._objs:
{'fraction_non_alphanumeric': 0.08122605363984674, 'fraction_numerical': 0.004597701149425287, 'mean_word_length': 2.4081810269799826, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3179546', 'n_tokens_mistral': 1126, 'n_tokens_neox': 1076, 'n_words': 444}