<commit_before>import os
from time import sleep
from subprocess import Popen, PIPE


class Slurm(object):
    """
    A python interface for slurm using subprocesses
    """

    def __init__(self):
        """
        Check if the system has Slurm installed
        """
        if not any(os.access(os.path.join(path, 'sinfo'), os.X_OK) for path in os.environ["PATH"].split(os.pathsep)):
            raise Exception(
                'Unable to find slurm, is it installed on this sytem?')

    def batch(self, cmd, sargs=None):
        """
        Submit to the batch queue in non-interactive mode

        Parameters:
            cmd (str): The path to the run script that should be submitted
            sargs (str): The additional arguments to pass to slurm
        Returns:
            job id of the new job (int)
        """

        out, err = self._submit('sbatch', cmd, sargs)
        if err:
            raise Exception('SLURM ERROR: ' + err)
        out = out.split()
        if 'error' in out:
            return 0
        job_id = int(out[-1])
        return job_id

    # def run(self, cmd, sargs=False):
    #     """
    #     Submit to slurm controller in interactive mode

    #     NOTE: THIS IS A BLOCKING CALL. Control will not return until the command completes

    #     Parameters:
    #         cmd (str): the command to run
    #     Returns:
    #         the output of the job (str)
    #     """
    #     out, err = self._submit('srun', cmd, sargs)
    #     if 'error' in out:
    #         return False, err
    #     else:
    #         return True, out

    def _submit(self, subtype, cmd, sargs=None):

        cmd = [subtype, cmd, sargs] if sargs else [subtype, cmd]
        tries = 0
        while tries != 10:
            try:
                proc = Popen(cmd, shell=False, stderr=PIPE, stdout=PIPE)
                out, err = proc.communicate()
                if 'Transport endpoint is not connected' in err:
                    tries += 1
                    sleep(tries)
                else:
                    break
            except:
                sleep(1)
        if tries == 10:
            raise Exception('SLURM ERROR: Transport endpoint is not connected')
        if 'Invalid job id specified' in err:
            raise Exception('SLURM ERROR: ' + err)

        return out, err

    def showjob(self, jobid):
        """<commit_msg>Handles the case where slurm returns an error, but it was still able to submit the job<commit_after>import os
from time import sleep
from subprocess import Popen, PIPE


class Slurm(object):
    """
    A python interface for slurm using subprocesses
    """

    def __init__(self):
        """
        Check if the system has Slurm installed
        """
        if not any(os.access(os.path.join(path, 'sinfo'), os.X_OK) for path in os.environ["PATH"].split(os.pathsep)):
            raise Exception(
                'Unable to find slurm, is it installed on this sytem?')

    def batch(self, cmd, sargs=None):
        """
        Submit to the batch queue in non-interactive mode

        Parameters:
            cmd (str): The path to the run script that should be submitted
            sargs (str): The additional arguments to pass to slurm
        Returns:
            job id of the new job (int)
        """

        out, err = self._submit('sbatch', cmd, sargs)
        if err:
            raise Exception('SLURM ERROR: ' + err)
        out = out.split()
        if 'error' in out:
            return 0
        job_id = int(out[-1])
        return job_id

    def _submit(self, subtype, cmd, sargs=None):

        cmd = [subtype, cmd, sargs] if sargs else [subtype, cmd]
        tries = 0
        while tries != 10:
            try:
                proc = Popen(cmd, shell=False, stderr=PIPE, stdout=PIPE)
                out, err = proc.communicate()
                if 'Transport endpoint is not connected' in err:
                    print 'unable to submit job, trying again'
                    tries += 1
                    sleep(tries)
                    proc = Popen(['scontrol', 'show', 'jobs'],
                                shell=False, stderr=PIPE, stdout=PIPE)
                    out, err = proc.communicate()
                    all_jobs = list()
                    jobinfo = dict()
                    for item in out.split('\n'):
                        if item == '':
                            all_jobs.append(jobinfo)
                            jobinfo = dict()
                        for j in item.split(' '):
                            index = j.find('=')
                            if index <= 0:
                                continue
                            jobinfo[j[:index]] = j[index + 1:]
                    for job in all_jobs:
                        if job['Command'] == cmd[1]:
                            return 'Submitted batch job ' + job['JobId'], None
                else:
                    break
            except:
                proc = Popen(['scontrol', 'show', 'jobs'],
                            shell=False, stderr=PIPE, stdout=PIPE)
                out, err = proc.communicate()
                all_jobs = list()
                jobinfo = dict()
                for item in out.split('\n'):
                    if item == '':
                        all_jobs.append(jobinfo)
                        jobinfo = dict()
                    for j in item.split(' '):
                        index = j.find('=')
                        if index <= 0:
                            continue
                        jobinfo[j[:index]] = j[index + 1:]
                for job in all_jobs:
                    if job['Command'] == cmd[1]:
                        return ['Submitted', 'batch', 'job', job['JobId']], None
                sleep(1)
        if tries >= 10:
            raise Exception('SLURM ERROR: Transport endpoint is not connected')
        if 'Invalid job id specified' in err:
            raise Exception('SLURM ERROR: ' + err)

        return out, err

    def showjob(self, jobid):
        """
{'fraction_non_alphanumeric': 0.09793899422918384, 'fraction_numerical': 0.003957131079967024, 'mean_word_length': 1.352074447460256, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2426737', 'n_tokens_mistral': 1666, 'n_tokens_neox': 1579, 'n_words': 548}