<commit_before>from __future__ import print_function

import glob
import os
import time
import stat

from PyAnalysisTools.base import _logger
from PyAnalysisTools.base.ShellUtils import remove_file, change_dir


class BatchJob(object):
    def __init__(self):
        pass


class BatchHandle(object):
    """
    Class to handle submission and monitoring to batch system
    """

    def __init__(self, job):
        if not hasattr(job, 'log_fname'):
            job.log_fname = 'log'
        self.system = 'qsub'
        self.queue = job.queue
        self.is_master = False
        self.log_level = job.log_level
        self.log_fname = job.log_fname
        self.local = job.local
        self.job = job
        if job.cluster_cfg_file is None:
            self.is_master = True

    def execute(self):
        _logger.debug("Executing batch job")
        if self.is_master:
            job_ids = self.job.prepare()
            self.output_dir = self.submit_childs(*self.job.submit_args, job_ids=job_ids)
            self.job.finish()
        else:
            self.job.run(self.job.cluster_cfg_file, self.job.job_id)

    def submit_childs(self, cfg_file_name, exec_script, n_jobs, output_dir, disable_wait=False, job_ids=None):
        base_path = os.path.join('/', *os.path.abspath(exec_script).split("/")[1:-2])
        _logger.debug('childs ids: {} {:d}'.format(str(job_ids), n_jobs))
        for job_id in range(n_jobs):
            if job_ids is not None and job_id not in job_ids:
                continue
            if self.local:
                log_fn = '{:s}_{:d}.txt'.format(self.log_fname, job_id)
                os.system('python {:s} -mtcf {:s} -id {:d} -log {:s} > {:s}'.format(exec_script, cfg_file_name, job_id,<commit_msg>implement option to inject extra submission arguments<commit_after>from __future__ import print_function

import glob
import os
import time
import stat

from PyAnalysisTools.base import _logger
from PyAnalysisTools.base.ShellUtils import remove_file, change_dir
import dill
dill._dill._reverse_typemap['ObjectType'] = object


class BatchJob(object):
    def __init__(self):
        pass


class BatchHandle(object):
    """
    Class to handle submission and monitoring to batch system
    """

    def __init__(self, job):
        if not hasattr(job, 'log_fname'):
            job.log_fname = 'log'
        self.system = 'qsub'
        if hasattr(job, 'extra_submission_args'):
            if job.extra_submission_args is not None:
                self.system += ' {:s}'.format(job.extra_submission_args)

        self.queue = job.queue
        self.is_master = False
        self.log_level = job.log_level
        self.log_fname = job.log_fname
        self.local = job.local
        self.job = job
        if job.cluster_cfg_file is None:
            self.is_master = True

    def execute(self):
        _logger.debug("Executing batch job")
        if self.is_master:
            job_ids = self.job.prepare()
            self.output_dir = self.submit_childs(*self.job.submit_args, job_ids=job_ids)
            self.job.finish()
        else:
            self.job.run(self.job.cluster_cfg_file, self.job.job_id)

    def submit_childs(self, cfg_file_name, exec_script, n_jobs, output_dir, disable_wait=False, job_ids=None):
        base_path = os.path.join('/', *os.path.abspath(exec_script).split("/")[1:-2])
        _logger.debug('childs ids: {} {:d}'.format(str(job_ids), n_jobs))
        for job_id in range(n_jobs):
            if job_ids is not None and job_id not in job_ids:
                continue
            if self.local:
                log_fn = '{:s}_{:d}.txt'.format(self.log_fname, job_id)
                os.system('python {:s} -mtcf {:s} -id {:d} -log {:s} > {:s}'.format(exec_script, cfg_file_name, job_id,
{'fraction_non_alphanumeric': 0.11299285525271235, 'fraction_numerical': 0.0010584810796507012, 'mean_word_length': 2.5097493036211698, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6594429', 'n_tokens_mistral': 1245, 'n_tokens_neox': 1208, 'n_words': 275}