<commit_before>"""
Extensions to pickle allowing items in __main__ to be saved.
"""

import new
import pickle
import new
import types
import __main__
from StringIO import StringIO



class PickleMain(object):
    """
    Pickle support for types and functions defined in __main__.
    
    When pickled, saves types and functions defined in __main__ by
    value (i.e. as bytecode). When unpickled, loads previously saved
    types and functions back into __main__.
    """
    def _create_pickler(self):
        # Usually we use the cPickle module rather than the pickle
        # module (because cPickle is faster), but here we need control
        # over the pickling process so we use pickle.
        #
        # Additionally, we create a Pickler instance to avoid changing
        # defaults in the pickle module itself, so that there are no side
        # effects for code elsewhere (although we don't use pickle
        # anywhere else ourselves...).

        self.pickled_bytecode = StringIO()
        self.pickler = pickle.Pickler(self.pickled_bytecode,-1)

        import picklesupport
        self.pickler.dispatch[new.code] = save_code
        self.pickler.dispatch[new.function] = save_function
        self.pickler.dispatch[dict] = save_module_dict
        self.pickler.dispatch[new.classobj] = save_classobj
        self.pickler.dispatch[new.instancemethod] = save_instancemethod
        self.pickler.dispatch[new.module] = save_module
        self.pickler.dispatch[type] = save_type
        # CB: maybe this should be registered from elsewhere
        from topo import param
        self.pickler.dispatch[param.parameterized.ParameterizedMetaclass] = save_type

<commit_msg>Removed stray line from previous checkin.<commit_after>"""
Extensions to pickle allowing items in __main__ to be saved.
"""

import new
import pickle
import new
import types
import __main__
from StringIO import StringIO



class PickleMain(object):
    """
    Pickle support for types and functions defined in __main__.
    
    When pickled, saves types and functions defined in __main__ by
    value (i.e. as bytecode). When unpickled, loads previously saved
    types and functions back into __main__.
    """
    def _create_pickler(self):
        # Usually we use the cPickle module rather than the pickle
        # module (because cPickle is faster), but here we need control
        # over the pickling process so we use pickle.
        #
        # Additionally, we create a Pickler instance to avoid changing
        # defaults in the pickle module itself, so that there are no side
        # effects for code elsewhere (although we don't use pickle
        # anywhere else ourselves...).

        self.pickled_bytecode = StringIO()
        self.pickler = pickle.Pickler(self.pickled_bytecode,-1)

        self.pickler.dispatch[new.code] = save_code
        self.pickler.dispatch[new.function] = save_function
        self.pickler.dispatch[dict] = save_module_dict
        self.pickler.dispatch[new.classobj] = save_classobj
        self.pickler.dispatch[new.instancemethod] = save_instancemethod
        self.pickler.dispatch[new.module] = save_module
        self.pickler.dispatch[type] = save_type
        # CB: maybe this should be registered from elsewhere
        from topo import param
        self.pickler.dispatch[param.parameterized.ParameterizedMetaclass] = save_type


{'fraction_non_alphanumeric': 0.06717963894643386, 'fraction_numerical': 0.0005918910920390648, 'mean_word_length': 2.9717978848413633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6476726', 'n_tokens_mistral': 923, 'n_tokens_neox': 905, 'n_words': 319}