<commit_before>import os
import sys
import time
import time
import stat
import types
import pickle
import signal
import logging
import importlib
import threading
import traceback
import multiprocessing
import os.path

from . import compiler, common, sim, endpoint as ep

api = common.api
deprecated = common.deprecated

DISTPY_SUFFIXES = [".da", ""]
PYTHON_SUFFIX = ".py"

log = logging.getLogger(__name__)
formatter = logging.Formatter(
    '[%(asctime)s]%(name)s:%(levelname)s: %(message)s')
log._formatter = formatter

PerformanceCounters = {}
CounterLock = threading.Lock()
RootProcess = None
RootLock = threading.Lock()
EndPointType = ep.UdpEndPoint
TotalUnits = None
ProcessIds = []

def find_file_on_paths(filename, paths):
    """Looks for a given 'filename' under a list of directories, in order.

    If found, returns a pair (path, mode), where 'path' is the full path to
    the file, and 'mode' is the result of calling 'os.stat' on the file.
    Otherwise, returns (None, None).

    """
    for path in paths:
        fullpath = os.path.join(path, filename)
        try:
            filemode = os.stat(fullpath)
            return fullpath, filemode
        except OSError:
            pass
    return None, None

@api
def daimport(filename, force_recompile=False, compiler_args=[], indir=None):
    dotidx = filename.rfind(".")
    paths = sys.path if indir is None else [indir]
    if dotidx == -1:
        # filename does not contain suffix, let's try each one
        purename = filename
        for suffix in DISTPY_SUFFIXES:
            fullpath, mode = find_file_on_paths(filename + suffix, paths)
            if fullpath is not None:
                break
    else:
        purename = filename[:dotidx]
        fullpath, mode = find_file_on_paths(filename, paths)
    if fullpath is None:
        raise ImportError("Module %s not found." % filename)
    dotidx = fullpath.rfind(".")
    pyname = (fullpath[:dotidx] + PYTHON_SUFFIX) \
             if dotidx != -1 else (fullpath + PYTHON_SUFFIX)
    try:
        pymode = os.stat(pyname)
    except OSError:
        pymode = None

    if (pymode is None or
            pymode[stat.ST_MTIME] < mode[stat.ST_MTIME] or
            force_recompile):
        oldargv = sys.argv
        try:
            argv = oldargv[0:0] + compiler_args + [fullpath]
            res = compiler.ui.main(argv)
        except Exception as err:
            raise RuntimeError("Compiler failure!", err)
        finally:
            sys.argv = oldargv

        if res != 0:
            raise ImportError("Unable to compile %s, errno: %d" %
                              (fullpath, res))

    return importlib.import_module(purename)

@api
def use_channel(endpoint):
    global EndPointType

    ept = None
    if endpoint == "udp":
        ept = ep.UdpEndPoint
    elif endpoint == "tcp":
        ept = ep.TcpEndPoint
    else:
        log.error("Unknown channel type %s", endpoint)
        return

    if RootProcess is not None:
        if EndPointType != ept:
            log.warn(
                "Can not change channel type after creating child processes.")
        return
    EndPointType = ept

def entrypoint():
    GlobalOptions = common.get_global_options()
    target = GlobalOptions.file
    source_dir = os.path.dirname(target)
    basename = os.path.basename(target)
    if not os.access(target, os.R_OK):
        die("Can not access source file %s" % target)

    sys.path.insert(0, source_dir)
    try:
        module = daimport(basename,
                          force_recompile=GlobalOptions.recompile,
                          compiler_args=GlobalOptions.compiler_flags.split(),
                          indir=source_dir)
    except ImportError as e:
        die("ImportError: " + str(e))
    if not (hasattr(module, 'main') and
            isinstance(module.main, types.FunctionType)):
        die("'main' function not defined!")
    GlobalOptions.this_module_name = module.__name__
    GlobalOptions.main_module_name = module.__name__
    if GlobalOptions.inc_module_name is None:
        GlobalOptions.inc_module_name = module.__name__ + "_inc"
    common.sysinit()

    # Start the background statistics thread:
    RootLock.acquire()
    stat_th = threading.Thread(target=collect_statistics,
                               name="Stat Thread")
    stat_th.daemon = True<commit_msg>daimport() takes a module name instead of path name.<commit_after>import os
import sys
import time
import time
import stat
import types
import pickle
import signal
import logging
import importlib
import threading
import traceback
import multiprocessing
import os.path

from . import compiler, common, sim, endpoint as ep

api = common.api
deprecated = common.deprecated

DISTPY_SUFFIXES = [".da", ""]
PYTHON_SUFFIX = ".py"

log = logging.getLogger(__name__)
formatter = logging.Formatter(
    '[%(asctime)s]%(name)s:%(levelname)s: %(message)s')
log._formatter = formatter

PerformanceCounters = {}
CounterLock = threading.Lock()
RootProcess = None
RootLock = threading.Lock()
EndPointType = ep.UdpEndPoint
TotalUnits = None
ProcessIds = []

def find_file_on_paths(filename, paths):
    """Looks for a given 'filename' under a list of directories, in order.

    If found, returns a pair (path, mode), where 'path' is the full path to
    the file, and 'mode' is the result of calling 'os.stat' on the file.
    Otherwise, returns (None, None).

    """
    for path in paths:
        fullpath = os.path.join(path, filename)
        try:
            filemode = os.stat(fullpath)
            return fullpath, filemode
        except OSError:
            pass
    return None, None

def strip_suffix(filename):
    """Returns a filename minus it's extension."""

    dotidx = filename.rfind(".")
    return filename[:dotidx] if dotidx != -1 else filename

@api
def daimport(module_name, force_recompile=False, compiler_args=[], indir=None):
    paths = sys.path if indir is None else [indir]
    pathname = module_name.replace(".", "/")
    for suffix in DISTPY_SUFFIXES:
        fullpath, mode = find_file_on_paths(pathname + suffix, paths)
        if fullpath is not None:
            break
    if fullpath is None:
        raise ImportError("Module %s not found." % module_name)
    pyname = strip_suffix(fullpath) + PYTHON_SUFFIX
    try:
        pymode = os.stat(pyname)
    except OSError:
        pymode = None

    if (pymode is None or
            pymode[stat.ST_MTIME] < mode[stat.ST_MTIME] or
            force_recompile):
        oldargv = sys.argv
        try:
            argv = oldargv[0:0] + compiler_args + [fullpath]
            res = compiler.ui.main(argv)
        except Exception as err:
            raise RuntimeError("Compiler failure!", err)
        finally:
            sys.argv = oldargv

        if res != 0:
            raise ImportError("Unable to compile %s, errno: %d" %
                              (fullpath, res))

    return importlib.import_module(module_name)

@api
def use_channel(endpoint):
    global EndPointType

    ept = None
    if endpoint == "udp":
        ept = ep.UdpEndPoint
    elif endpoint == "tcp":
        ept = ep.TcpEndPoint
    else:
        log.error("Unknown channel type %s", endpoint)
        return

    if RootProcess is not None:
        if EndPointType != ept:
            log.warn(
                "Can not change channel type after creating child processes.")
        return
    EndPointType = ept

def entrypoint():
    GlobalOptions = common.get_global_options()
    target = GlobalOptions.file
    source_dir = os.path.dirname(target)
    basename = strip_suffix(os.path.basename(target))
    if not os.access(target, os.R_OK):
        die("Can not access source file %s" % target)

    sys.path.insert(0, source_dir)
    try:
        module = daimport(basename,
                          force_recompile=GlobalOptions.recompile,
                          compiler_args=GlobalOptions.compiler_flags.split(),
                          indir=source_dir)
    except ImportError as e:
        die("ImportError: " + str(e))
    if not (hasattr(module, 'main') and
            isinstance(module.main, types.FunctionType)):
        die("'main' function not defined!")
    GlobalOptions.this_module_name = module.__name__
    GlobalOptions.main_module_name = module.__name__
    if GlobalOptions.inc_module_name is None:
        GlobalOptions.inc_module_name = module.__name__ + "_inc"
    common.sysinit()

    # Start the background statistics thread:
    RootLock.acquire()
    stat_th = threading.Thread(target=collect_statistics,
                               name="Stat Thread")
    stat_th.daemon = True
{'fraction_non_alphanumeric': 0.0945977277996754, 'fraction_numerical': 0.001275214467887781, 'mean_word_length': 2.5140529531568228, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2668612', 'n_tokens_mistral': 2584, 'n_tokens_neox': 2546, 'n_words': 729}