<commit_before>"""Distributed Extension"""
import re
import os
import sys
import drmaa
import itertools

from cement.core import backend, handler, hook

from scilifelab.pm.core import command

LOG = backend.minimal_logger(__name__)

class DistributedCommandHandler(command.CommandHandler):
    """ 
    This class is an implementation of the :ref:`ICommand
    <scilifelab.pm.core.command>` interface.
    """    

    class Meta:
        """Handler meta-data"""
        
        interface = command.ICommand
        """The interface that this class implements."""

        label = 'distributed'
        """The string identifier of this handler."""

        n_submitted_jobs = 0
        """The number of submitted jobs"""

        jobid = None
        """The submitted jobid"""

        platform_args = None
        """Platform specific arguments"""

    def command(self, cmd_args, capture=True, ignore_error=False, cwd=None, **kw):
        ## Is there no easier way to get at --drmaa?!?
        if '--drmaa' in self.app._meta.argv:
            self.drmaa(cmd_args, capture, ignore_error, cwd, **kw)
        else:
            pass

    def _check_args(self, **kw):
        pargs = kw.get('platform_args', None)
        if not self.app.pargs.account and "-A" not in pargs and "--account" not in pargs:
            return False
        if not self.app.pargs.jobname and "-J" not in pargs and "--jobname" not in pargs:
            return False
        if not self.app.pargs.partition and "-p" not in pargs and "--partition" not in pargs:
            return False
        if not self.app.pargs.time and "-t" not in pargs and "--time" not in pargs:
            return False
        return True

    def drmaa(self, cmd_args, capture=True, ignore_error=False, cwd=None, **kw):
        if self.app.pargs.partition == "node" and self.app.pargs.max_node_jobs < self._meta.n_submitted_jobs:
            self.app.log.info("number of submitted jobs larger than maximum number of allowed node jobs; not submitting job")
            return
        self._meta.n_submitted_jobs = self._meta.n_submitted_jobs + 1
        if not self._check_args(**kw):
            self.app.log.warn("missing argument; cannot proceed with drmaa command. Make sure you provide time, account, partition, and jobname")
            return
        command = " ".join(cmd_args)
        def runpipe():
            s = drmaa.Session()
            s.initialize()
            jt = s.createJobTemplate()
            jt.remoteCommand = cmd_args[0]
            jt.args = cmd_args[1:]
            if kw['platform_args']:
                platform_args = opt_to_dict(kw['platform_args'])
            else:
                platform_args = opt_to_dict([])
            opt_d = make_job_template_args(platform_args, **vars(self.app.pargs))<commit_msg>Update default kw['platform_args'] so it returns []<commit_after>"""Distributed Extension"""
import re
import os
import sys
import drmaa
import itertools

from cement.core import backend, handler, hook

from scilifelab.pm.core import command

LOG = backend.minimal_logger(__name__)

class DistributedCommandHandler(command.CommandHandler):
    """ 
    This class is an implementation of the :ref:`ICommand
    <scilifelab.pm.core.command>` interface.
    """    

    class Meta:
        """Handler meta-data"""
        
        interface = command.ICommand
        """The interface that this class implements."""

        label = 'distributed'
        """The string identifier of this handler."""

        n_submitted_jobs = 0
        """The number of submitted jobs"""

        jobid = None
        """The submitted jobid"""

        platform_args = None
        """Platform specific arguments"""

    def command(self, cmd_args, capture=True, ignore_error=False, cwd=None, **kw):
        ## Is there no easier way to get at --drmaa?!?
        if '--drmaa' in self.app._meta.argv:
            self.drmaa(cmd_args, capture, ignore_error, cwd, **kw)
        else:
            pass

    def _check_args(self, **kw):
        pargs = kw.get('platform_args', [])
        if not self.app.pargs.account and "-A" not in pargs and "--account" not in pargs:
            return False
        if not self.app.pargs.jobname and "-J" not in pargs and "--jobname" not in pargs:
            return False
        if not self.app.pargs.partition and "-p" not in pargs and "--partition" not in pargs:
            return False
        if not self.app.pargs.time and "-t" not in pargs and "--time" not in pargs:
            return False
        return True

    def drmaa(self, cmd_args, capture=True, ignore_error=False, cwd=None, **kw):
        if self.app.pargs.partition == "node" and self.app.pargs.max_node_jobs < self._meta.n_submitted_jobs:
            self.app.log.info("number of submitted jobs larger than maximum number of allowed node jobs; not submitting job")
            return
        self._meta.n_submitted_jobs = self._meta.n_submitted_jobs + 1
        if not self._check_args(**kw):
            self.app.log.warn("missing argument; cannot proceed with drmaa command. Make sure you provide time, account, partition, and jobname")
            return
        command = " ".join(cmd_args)
        def runpipe():
            s = drmaa.Session()
            s.initialize()
            jt = s.createJobTemplate()
            jt.remoteCommand = cmd_args[0]
            jt.args = cmd_args[1:]
            if kw['platform_args']:
                platform_args = opt_to_dict(kw['platform_args'])
            else:
                platform_args = opt_to_dict([])
            opt_d = make_job_template_args(platform_args, **vars(self.app.pargs))
{'fraction_non_alphanumeric': 0.10692717584369449, 'fraction_numerical': 0.0014209591474245115, 'mean_word_length': 2.318208603417796, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7469920', 'n_tokens_mistral': 1658, 'n_tokens_neox': 1607, 'n_words': 492}