<commit_before>import sys
import os
import errno
import warnings
import inspect
import re
import collections

def Observe(*args):
    return SelfExporter.default_model.add_observable(*args)

def Initial(*args):
    return SelfExporter.default_model.initial(*args)

def MatchOnce(pattern):
    cp = as_complex_pattern(pattern).copy()
    cp.match_once = True
    return cp


# Internal helper to implement the magic of making model components
# appear in the calling module's namespace.  Do not construct any
# instances; we just use the class for namespace containment.
class SelfExporter(object):

    do_export = True
    default_model = None
    target_globals = None   # the globals dict to which we'll export our symbols
    target_module = None    # the module to which we've exported

    @staticmethod
    def export(obj):
        if not SelfExporter.do_export:
            return
        if not isinstance(obj, (Model, Component)):
            raise Exception("%s is not a type that is understood by SelfExporter" % str(type(obj)))

        # determine the module from which we were called (we need to do this here so we can
        # calculate stacklevel for use in the warning at the bottom of this method)
        cur_module = inspect.getmodule(inspect.currentframe())
        caller_frame = inspect.currentframe()
        # walk up through the stack until we hit a different module
        stacklevel = 1
        while inspect.getmodule(caller_frame) == cur_module:
            stacklevel += 1
            caller_frame = caller_frame.f_back

        # use obj's name as the symbol to export it to (unless modified below)
        export_name = obj.name

        if isinstance(obj, Model):
            if SelfExporter.default_model is not None:
                warnings.warn("Redefining model! (You can probably ignore this if you are running"
                              " code interactively)", ModelExistsWarning, stacklevel);
                # delete previously exported symbols to prevent extra SymbolExistsWarnings
                for name in [c.name for c in SelfExporter.default_model.all_components()] + ['model']:
                    if name in SelfExporter.target_globals:
                        del SelfExporter.target_globals[name]
            SelfExporter.target_module = inspect.getmodule(caller_frame)
            SelfExporter.target_globals = caller_frame.f_globals
            SelfExporter.default_model = obj
            # if not set, assign model's name from the module it lives in. very sneaky and fragile.
            if obj.name is None:
                if SelfExporter.target_module == sys.modules['__main__']:
                    # user ran model .py directly
                    model_filename = inspect.getfile(sys.modules['__main__'])
                    module_name = re.sub(r'\.py$', '', model_filename)
                elif SelfExporter.target_module is not None:
                    # model is imported by some other script (typical case)
                    module_name = SelfExporter.target_module.__name__
                else:
                    # user is defining a model interactively (not really supported, but we'll try)
                    module_name = '<interactive>'
                obj.name = module_name   # internal name for identification<commit_msg>improve model name autodetection for paths with slashes<commit_after>import sys
import os
import errno
import warnings
import inspect
import re
import collections

def Observe(*args):
    return SelfExporter.default_model.add_observable(*args)

def Initial(*args):
    return SelfExporter.default_model.initial(*args)

def MatchOnce(pattern):
    cp = as_complex_pattern(pattern).copy()
    cp.match_once = True
    return cp


# Internal helper to implement the magic of making model components
# appear in the calling module's namespace.  Do not construct any
# instances; we just use the class for namespace containment.
class SelfExporter(object):

    do_export = True
    default_model = None
    target_globals = None   # the globals dict to which we'll export our symbols
    target_module = None    # the module to which we've exported

    @staticmethod
    def export(obj):
        if not SelfExporter.do_export:
            return
        if not isinstance(obj, (Model, Component)):
            raise Exception("%s is not a type that is understood by SelfExporter" % str(type(obj)))

        # determine the module from which we were called (we need to do this here so we can
        # calculate stacklevel for use in the warning at the bottom of this method)
        cur_module = inspect.getmodule(inspect.currentframe())
        caller_frame = inspect.currentframe()
        # walk up through the stack until we hit a different module
        stacklevel = 1
        while inspect.getmodule(caller_frame) == cur_module:
            stacklevel += 1
            caller_frame = caller_frame.f_back

        # use obj's name as the symbol to export it to (unless modified below)
        export_name = obj.name

        if isinstance(obj, Model):
            if SelfExporter.default_model is not None:
                warnings.warn("Redefining model! (You can probably ignore this if you are running"
                              " code interactively)", ModelExistsWarning, stacklevel);
                # delete previously exported symbols to prevent extra SymbolExistsWarnings
                for name in [c.name for c in SelfExporter.default_model.all_components()] + ['model']:
                    if name in SelfExporter.target_globals:
                        del SelfExporter.target_globals[name]
            SelfExporter.target_module = inspect.getmodule(caller_frame)
            SelfExporter.target_globals = caller_frame.f_globals
            SelfExporter.default_model = obj
            # if not set, assign model's name from the module it lives in. very sneaky and fragile.
            if obj.name is None:
                if SelfExporter.target_module == sys.modules['__main__']:
                    # user ran model .py directly
                    model_path = inspect.getfile(sys.modules['__main__'])
                    model_filename = os.path.basename(model_path)
                    module_name = re.sub(r'\.py$', '', model_filename)
                elif SelfExporter.target_module is not None:
                    # model is imported by some other script (typical case)
                    module_name = SelfExporter.target_module.__name__
                else:
                    # user is defining a model interactively (not really supported, but we'll try)
                    module_name = '<interactive>'
                obj.name = module_name   # internal name for identification
{'fraction_non_alphanumeric': 0.06495243757431629, 'fraction_numerical': 0.0005945303210463733, 'mean_word_length': 2.3594608087868196, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4280688', 'n_tokens_mistral': 1711, 'n_tokens_neox': 1644, 'n_words': 606}