<commit_before>"""
Extensions to pickle allowing items in __main__ to be saved.
"""

import new
import pickle
import new
import types
import __main__
from StringIO import StringIO

import copy

class PickleMain(object):
    """
    Pickle support for types and functions defined in __main__.
    
    When pickled, saves types and functions defined in __main__ by
    value (i.e. as bytecode). When unpickled, loads previously saved
    types and functions back into __main__.
    """
    def _create_pickler(self):
        # Usually we use the cPickle module rather than the pickle
        # module (because cPickle is faster), but here we need control
        # over the pickling process so we use pickle.
        #
        # Additionally, we create a Pickler instance to avoid changing
        # defaults in the pickle module itself, so that there are no side
        # effects for code elsewhere (although we don't use pickle
        # anywhere else ourselves...).

        self.pickled_bytecode = StringIO()
        self.pickler = pickle.Pickler(self.pickled_bytecode,-1)

        # CB: pickle.Pickler's dispatch attribute is a class
        # attribute (I don't know why, but it is...), so before
        # modifying this instance's dispatch attribute we make sure
        # modifications will affect only this instance.
        self.pickler.__dict__['dispatch'] = copy.copy(self.pickler.dispatch)

        self.pickler.dispatch[new.code] = save_code
        self.pickler.dispatch[new.function] = save_function
        self.pickler.dispatch[dict] = save_module_dict<commit_msg>Minor simplification (should have no effect).<commit_after>"""
Extensions to pickle allowing items in __main__ to be saved.
"""

import new
import pickle
import new
import types
import __main__
from StringIO import StringIO

import copy

class PickleMain(object):
    """
    Pickle support for types and functions defined in __main__.
    
    When pickled, saves types and functions defined in __main__ by
    value (i.e. as bytecode). When unpickled, loads previously saved
    types and functions back into __main__.
    """
    def _create_pickler(self):
        # Usually we use the cPickle module rather than the pickle
        # module (because cPickle is faster), but here we need control
        # over the pickling process so we use pickle.
        #
        # Additionally, we create a Pickler instance to avoid changing
        # defaults in the pickle module itself, so that there are no side
        # effects for code elsewhere (although we don't use pickle
        # anywhere else ourselves...).

        self.pickled_bytecode = StringIO()
        self.pickler = pickle.Pickler(self.pickled_bytecode,-1)

        # CB: pickle.Pickler's dispatch attribute is a class
        # attribute (I don't know why, but it is...), so before
        # modifying this instance's dispatch attribute we make sure
        # modifications will affect only this instance.
        self.pickler.dispatch = copy.copy(self.pickler.dispatch)

        self.pickler.dispatch[new.code] = save_code
        self.pickler.dispatch[new.function] = save_function
        self.pickler.dispatch[dict] = save_module_dict
{'fraction_non_alphanumeric': 0.0687382297551789, 'fraction_numerical': 0.0006277463904582549, 'mean_word_length': 2.8913308913308913, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7023513', 'n_tokens_mistral': 860, 'n_tokens_neox': 835, 'n_words': 342}