<commit_before>import os
import os.path
import sys
import time
import logging
import glob

import datetime
import numpy
try:
    import configparser as cp  # Python 3 only
except ImportError:
    import ConfigParser as cp  # Python 2 only
from fnmatch import fnmatch
from daemon import runner
from module_common import SimulationTask

__simon_dir__ = os.path.dirname(os.path.abspath(__file__))


class SiMon(object):
    """
    Main code of Simulation Monitor (SiMon).
    """

    def __init__(self, pidfile=None, stdin='/dev/tty', stdout='/dev/tty', stderr='/dev/tty',
                 mode='interactive', cwd=None, config_file='SiMon.conf'):
        """
        :param pidfile:
        """

        # Only needed in interactive mode
        self.config = self.parse_config_file(config_file)
        if self.config is None:
            print('Error: Configure file SiMon.conf does not exist.')
            sys.exit(-1)
        else:
            try:
                cwd = self.config.get('SiMon', 'Root_dir')
            except cp.NoOptionError:
                print('Item Root_dir is missing in configure file SiMon.conf. SiMon cannot start. Exiting...')
                sys.exit(-1)
        # make sure that cwd is the absolute path
        if not os.path.isabs(cwd):
            cwd = os.path.join(__simon_dir__, cwd)
        if not os.path.isdir(cwd):
            print('Simulation root directory does not exist. Existing...')
            sys.exit(-1)

        self.module_dict = self.register_modules()

        self.selected_inst = []  # A list of the IDs of selected simulation instances
        self.sim_inst_dict = dict()  # the container of all SimulationTask objects (ID to object mapping)
        self.sim_inst_parent_dict = dict()  # given the current path, find out the instance of the parent

        # TODO: create subclass instance according to the config file
        self.sim_tree = SimulationTask(0, 'root', cwd, SimulationTask.STATUS_NEW)
        self.stdin_path = stdin
        self.stdout_path = stdout
        self.stderr_path = stderr
        self.pidfile_path = pidfile
        self.pidfile_timeout = 5
        self.mode = mode
        self.cwd = cwd
        self.inst_id = 0
        self.tcrit = 100
        self.logger = None
        self.max_concurrent_jobs = 2

        if self.config.has_option('SiMon', 'Max_concurrent_jobs'):
            self.max_concurrent_jobs = self.config.getint('SiMon', 'Max_concurrent_jobs')

        os.chdir(cwd)

    @staticmethod
    def id_input(prompt):
        """
        Prompt to the user to input the simulation ID (in the interactive mode)
        """
        confirmed = False
        vec_index_selected = []
        while confirmed is False:
            response = raw_input(prompt)
            fragment = response.split(',')
            for token_i in fragment:
                if '-' in token_i:  # it is a range
                    limits = token_i.split('-')
                    if len(limits) == 2:
                        if int(limits[0].strip()) < int(limits[1].strip()):
                            subrange = range(int(limits[0].strip()), int(limits[1].strip())+1)
                            for j in subrange:
                                vec_index_selected.append(j)
                else:
                    vec_index_selected.append(token_i.strip())
            if raw_input('Your input is \n\t'+str(vec_index_selected)+', confirm? [Y/N] ').lower() == 'y':
                confirmed = True
                return map(int, vec_index_selected)

    @staticmethod
    def parse_config_file(config_file):
        """
        Parse the configure file (SiMon.conf) for starting SiMon. The basic information of Simulation root directory
        must exist in the configure file before SiMon can start. A minimum configure file of SiMon looks like:

        ==============================================
        [SiMon]
        Root_dir: <the_root_dir_of_the_simulation_data>
        ==============================================

        :return: return 0 if succeed, -1 if failed (file not exist, and cannot be created). If the file does not exist
        but a new file with default values is created, the method returns 1.
        """
        conf = cp.ConfigParser()
        if os.path.isfile(config_file):
            conf.read(config_file)
            return conf
        else:
            return None

    @staticmethod
    def register_modules():
        """
        Register modules
        :return: A dict-like mapping between the name of the code and the filename of the module.
        """
        mod_dict = dict()
        module_candidates = glob.glob('module_*.py')
        for mod_name in module_candidates:
            mod = __import__(mod_name.split('.')[0])
            if hasattr(mod, '__simulation__'):
                # it is a valid SiMon module
                mod_dict[mod.__simulation__] = mod_name.split('.')[0]
        return mod_dict

    def traverse_simulation_dir_tree(self, pattern, base_dir, files):
        """
        Traverse the simulation file structure tree (Breadth-first search), until the leaf (i.e. no restart directory)
        or the simulation is not restartable (directory with the 'STOP' file).
        """
        for filename in sorted(files):
            if fnmatch(filename, pattern):
                if os.path.isdir(os.path.join(base_dir, filename)):
                    fullpath = os.path.join(base_dir, filename)
                    self.inst_id += 1
                    id = self.inst_id

                    # Try to determine the simulation code type by reading the config file
                    sim_config = self.parse_config_file(os.path.join(fullpath, 'SiMon.conf'))
                    sim_inst = None
                    if sim_config is not None:
                        try:
                            code_name = sim_config.get('Simulation', 'Code_name')

                            if code_name in self.module_dict:
                                sim_inst_mod = __import__(self.module_dict[code_name])<commit_msg>fix import dir error<commit_after>import os
import os.path
import sys
import time
import logging
import glob

import datetime
import numpy
try:
    import configparser as cp  # Python 3 only
except ImportError:
    import ConfigParser as cp  # Python 2 only
from fnmatch import fnmatch
from daemon import runner
from module_common import SimulationTask

__simon_dir__ = os.path.dirname(os.path.abspath(__file__))


class SiMon(object):
    """
    Main code of Simulation Monitor (SiMon).
    """

    def __init__(self, pidfile=None, stdin='/dev/tty', stdout='/dev/tty', stderr='/dev/tty',
                 mode='interactive', cwd=None, config_file='SiMon.conf'):
        """
        :param pidfile:
        """

        # Only needed in interactive mode
        # self.config = self.parse_config_file(os.getcwd(), config_file)
        conf_path = os.path.join(os.getcwd(), config_file)
        self.config = self.parse_config_file(conf_path)
        
        # TODO: os.getcwd() gives dir of where the current console is, eg. type simon-interactive on /SiMon will give an error, but on SiMon/Simon dir it can work.
        if self.config is None:
            print('Error: Configure file SiMon.conf does not exist.', conf_path)
            sys.exit(-1)
        else:
            try:
                cwd = self.config.get('SiMon', 'Root_dir')
            except cp.NoOptionError:
                print('Item Root_dir is missing in configure file SiMon.conf. SiMon cannot start. Exiting...')
                sys.exit(-1)
        # make sure that cwd is the absolute path
        if not os.path.isabs(cwd):
            cwd = os.path.join(__simon_dir__, cwd)
        if not os.path.isdir(cwd):
            print('Simulation root directory does not exist. Existing...')
            sys.exit(-1)

        self.module_dict = self.register_modules()

        self.selected_inst = []  # A list of the IDs of selected simulation instances
        self.sim_inst_dict = dict()  # the container of all SimulationTask objects (ID to object mapping)
        self.sim_inst_parent_dict = dict()  # given the current path, find out the instance of the parent

        # TODO: create subclass instance according to the config file
        self.sim_tree = SimulationTask(0, 'root', cwd, SimulationTask.STATUS_NEW)
        self.stdin_path = stdin
        self.stdout_path = stdout
        self.stderr_path = stderr
        self.pidfile_path = pidfile
        self.pidfile_timeout = 5
        self.mode = mode
        self.cwd = cwd
        self.inst_id = 0
        self.tcrit = 100
        self.logger = None
        self.max_concurrent_jobs = 2

        if self.config.has_option('SiMon', 'Max_concurrent_jobs'):
            self.max_concurrent_jobs = self.config.getint('SiMon', 'Max_concurrent_jobs')

        os.chdir(cwd)

    @staticmethod
    def id_input(prompt):
        """
        Prompt to the user to input the simulation ID (in the interactive mode)
        """
        confirmed = False
        vec_index_selected = []
        while confirmed is False:
            response = raw_input(prompt)
            fragment = response.split(',')
            for token_i in fragment:
                if '-' in token_i:  # it is a range
                    limits = token_i.split('-')
                    if len(limits) == 2:
                        if int(limits[0].strip()) < int(limits[1].strip()):
                            subrange = range(int(limits[0].strip()), int(limits[1].strip())+1)
                            for j in subrange:
                                vec_index_selected.append(j)
                else:
                    vec_index_selected.append(token_i.strip())
            if raw_input('Your input is \n\t'+str(vec_index_selected)+', confirm? [Y/N] ').lower() == 'y':
                confirmed = True
                return map(int, vec_index_selected)

    @staticmethod
    def parse_config_file(config_file):
        """
        Parse the configure file (SiMon.conf) for starting SiMon. The basic information of Simulation root directory
        must exist in the configure file before SiMon can start. A minimum configure file of SiMon looks like:

        ==============================================
        [SiMon]
        Root_dir: <the_root_dir_of_the_simulation_data>
        ==============================================

        :return: return 0 if succeed, -1 if failed (file not exist, and cannot be created). If the file does not exist
        but a new file with default values is created, the method returns 1.
        """
        conf = cp.ConfigParser()
        if os.path.isfile(config_file):
            conf.read(config_file)
            return conf
        else:
            return None

    @staticmethod
    def register_modules():
        """
        Register modules
        :return: A dict-like mapping between the name of the code and the filename of the module.
        """
        mod_dict = dict()
        module_candidates = glob.glob('module_*.py')
        for mod_name in module_candidates:
            sys.path.append(os.getcwd())
            import importlib
            mod =  __import__(mod_name.split('.')[0])
            
            # mod = __import__(mod_name.split('.')[0])
            
            print('mod_dict', mod, mod_dict)
            # mod = __import__(module_path)
            if hasattr(mod, '__simulation__'):
                # it is a valid SiMon module
                mod_dict[mod.__simulation__] = mod_name.split('.')[0]
            
        return mod_dict

    def traverse_simulation_dir_tree(self, pattern, base_dir, files):
        """
        Traverse the simulation file structure tree (Breadth-first search), until the leaf (i.e. no restart directory)
        or the simulation is not restartable (directory with the 'STOP' file).
        """
        for filename in sorted(files):
            if fnmatch(filename, pattern):
                if os.path.isdir(os.path.join(base_dir, filename)):
                    fullpath = os.path.join(base_dir, filename)
                    self.inst_id += 1
                    id = self.inst_id

                    # Try to determine the simulation code type by reading the config file
                    sim_config = self.parse_config_file(os.path.join(fullpath, 'SiMon.conf'))
                    sim_inst = None
                    if sim_config is not None:
                        try:
                            code_name = sim_config.get('Simulation', 'Code_name')

                            if code_name in self.module_dict:
                                sim_inst_mod = __import__(self.module_dict[code_name])
{'fraction_non_alphanumeric': 0.10318271119842828, 'fraction_numerical': 0.003850687622789784, 'mean_word_length': 1.9437890353920888, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4282626', 'n_tokens_mistral': 3545, 'n_tokens_neox': 3405, 'n_words': 1023}