<commit_before># Copyright (C) 2018 <NAME>
# All rights reserved.
#
# This file is part of yambopy
#
#
from __future__ import print_function, absolute_import
import os
import subprocess
import json
from copy import deepcopy

class Scheduler(object):
    """
    Generic scheduler class

    Initialize the class, by default we look for a config file in `~/.yambopy/config.json`

    This file has the following information:

        .. code-block:: javascript

            {
                "default": "oar <or> bash <or> pbs",
                "oar <or> bash <or> pbs":
                {
                    "mpirun": "<mpirun command>",
                    "modules": {"<module tag>":"<module name>"},
                    "pre_run": ["line1 to execute before the run",
                                "line2 to execute before the run"],
                    "pos_run": "file:<file_in_config_folder>",
                    "<tag>":"<value>"
                }
            }

    The "default" tag chooses the scheduler to use when no scheduler is
    specified by the user.

    "<tag>" and "<value>" in are additional variables that are handled
    differently for each scheduler. They are stored as arguments in the variable kwargs.
    """

    #TODO: Add a way to get the home directory
    _config_path     = os.path.expanduser("~") + "/.yambopy"
    _config_filename = "%s/config.json"%_config_path

    def __init__(self, name=None, nodes=None, cores=None, walltime="1:00:00", **kwargs ):
        self.name = name
        self.nodes = nodes
        self.cores = cores
        self.walltime = walltime
        self.kwargs = kwargs

        self.pre_run = self.get_arg("pre_run",[])
        self.pos_run = self.get_arg("pos_run",[])

        self.modules_dict = self.get_arg("modules_dict",{})
        self.modules_list = []
        for mod in self.get_arg("modules_list",[]):
            self.add_module(mod)
        self.arguments = []
        self.commands  = []

        self.initialize()


    @classmethod    
    def factory(cls,scheduler=None,cores=None,nodes=None,walltime="1:00:00",**kwargs):
        """
        Initialize a schduler instance.

        Default arguments:
        cores    - Number of cores to use
        nodes    - Number of nodes to use
        walltime - Walltime limit
        **kwargs - Additional tags specific for each scheduler
        """
        from .oar import Oar
        from .pbs import Pbs
        from .bash import Bash
        from .slurm import Slurm

        schedulers = { "oar":   Oar,
                       "pbs":   Pbs,
                       "slurm": Slurm,
                       "bash":  Bash }

        #load configurations file
        config = cls.load_config()

        #set from scheduler
        schedulername = scheduler
        if not scheduler: schedulername = config.get('default',None)

        if schedulername is None:
            raise ValueError('scheduler not specified and "default" not set in configuration file %s.'%self._config_filename)

        #load the configurations
        schedulerconfig = config.get(schedulername,{})

        #determine the scheduler type
        if "type" in schedulerconfig:
            schedulertype = schedulerconfig["type"]
        elif "oar"   in schedulername: schedulertype = "oar"
        elif "pbs"   in schedulername: schedulertype = "pbs"
        elif "bash"  in schedulername: schedulertype = "bash"
        elif "slurm" in schedulername: schedulertype = "slurm"
        else:
            raise ValueError("Could not determine the scheduler type. "
                             "Please specify it in the name of the scheduler or using the 'type' tag.")

        #sanity check
        if schedulername not in list(schedulers.keys()):
            raise ValueError("Scheduler name %s is invalid"%schedulername)

        if "nodes" in schedulerconfig and nodes is None:
            nodes = int(schedulerconfig["nodes"])
            del schedulerconfig["nodes"]
        if "cores" in schedulerconfig and cores is None:
            cores = int(schedulerconfig["cores"])
            del schedulerconfig["cores"]

        #add scheduler config arguments
        kwargs.update(schedulerconfig)

        #create an instance of the scheduler to use
        return schedulers[schedulertype](cores=cores,nodes=nodes,walltime=walltime,**kwargs)
<commit_msg>Read the walltime from the configuration file.<commit_after># Copyright (C) 2018 <NAME>
# All rights reserved.
#
# This file is part of yambopy
#
#
from __future__ import print_function, absolute_import
import os
import subprocess
import json
from copy import deepcopy

class Scheduler(object):
    """
    Generic scheduler class

    Initialize the class, by default we look for a config file in `~/.yambopy/config.json`

    This file has the following information:

        .. code-block:: javascript

            {
                "default": "oar <or> bash <or> pbs",
                "oar <or> bash <or> pbs":
                {
                    "mpirun": "<mpirun command>",
                    "modules": {"<module tag>":"<module name>"},
                    "pre_run": ["line1 to execute before the run",
                                "line2 to execute before the run"],
                    "pos_run": "file:<file_in_config_folder>",
                    "<tag>":"<value>"
                }
            }

    The "default" tag chooses the scheduler to use when no scheduler is
    specified by the user.

    "<tag>" and "<value>" in are additional variables that are handled
    differently for each scheduler. They are stored as arguments in the variable kwargs.
    """

    #TODO: Add a way to get the home directory
    _config_path     = os.path.expanduser("~") + "/.yambopy"
    _config_filename = "%s/config.json"%_config_path

    def __init__(self, name=None, nodes=None, cores=None, walltime="1:00:00", **kwargs ):
        self.name = name
        self.nodes = nodes
        self.cores = cores
        self.walltime = walltime
        self.kwargs = kwargs

        self.pre_run = self.get_arg("pre_run",[])
        self.pos_run = self.get_arg("pos_run",[])

        self.modules_dict = self.get_arg("modules_dict",{})
        self.modules_list = []
        for mod in self.get_arg("modules_list",[]):
            self.add_module(mod)
        self.arguments = []
        self.commands  = []

        self.initialize()


    @classmethod    
    def factory(cls,scheduler=None,cores=None,nodes=None,walltime="1:00:00",**kwargs):
        """
        Initialize a schduler instance.

        Default arguments:
        cores    - Number of cores to use
        nodes    - Number of nodes to use
        walltime - Walltime limit
        **kwargs - Additional tags specific for each scheduler
        """
        from .oar import Oar
        from .pbs import Pbs
        from .bash import Bash
        from .slurm import Slurm

        schedulers = { "oar":   Oar,
                       "pbs":   Pbs,
                       "slurm": Slurm,
                       "bash":  Bash }

        #load configurations file
        config = cls.load_config()

        #set from scheduler
        schedulername = scheduler
        if not scheduler: schedulername = config.get('default',None)

        if schedulername is None:
            raise ValueError('scheduler not specified and "default" not set in configuration file %s.'%self._config_filename)

        #load the configurations
        schedulerconfig = config.get(schedulername,{})

        #determine the scheduler type
        if "type" in schedulerconfig:
            schedulertype = schedulerconfig["type"]
        elif "oar"   in schedulername: schedulertype = "oar"
        elif "pbs"   in schedulername: schedulertype = "pbs"
        elif "bash"  in schedulername: schedulertype = "bash"
        elif "slurm" in schedulername: schedulertype = "slurm"
        else:
            raise ValueError("Could not determine the scheduler type. "
                             "Please specify it in the name of the scheduler or using the 'type' tag.")

        #sanity check
        if schedulername not in list(schedulers.keys()):
            raise ValueError("Scheduler name %s is invalid"%schedulername)

        if "nodes" in schedulerconfig and nodes is None:
            nodes = int(schedulerconfig["nodes"])
            del schedulerconfig["nodes"]
        if "cores" in schedulerconfig and cores is None:
            cores = int(schedulerconfig["cores"])
            del schedulerconfig["cores"]

        walltime = schedulerconfig.pop("walltime",walltime)

        #add scheduler config arguments
        kwargs.update(schedulerconfig)

        #create an instance of the scheduler to use
        return schedulers[schedulertype](cores=cores,nodes=nodes,walltime=walltime,**kwargs)

{'fraction_non_alphanumeric': 0.09664246823956443, 'fraction_numerical': 0.003629764065335753, 'mean_word_length': 2.0969441517386724, 'pattern_counts': {'":': 24, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6167232', 'n_tokens_mistral': 2492, 'n_tokens_neox': 2392, 'n_words': 756}