<commit_before>'''Calculation manager.

This module provides classes for managing collections of calculations.
For persistant information it is best to interact via a JobCache instance.
'''

import copy
import os
import os.path
import pickle
import re
import time
import subprocess

### Custom exceptions ###

class UserError(Exception):
    '''Raised due to an error from user-input.'''
    pass


class LockException(Exception):
    '''Raised if a lock cannot be acquired.'''
    pass

### Cache classes ###

class JobStatus(object):
    '''enum-esque class for specifying the status of a job.

Defined statuses: held, queueing, running, finished, analysed.'''
    held = 'held'
    queueing = 'queueing'
    running = 'running'
    finished = 'finished'
    analysed = 'analysed'


class Job:
    '''Store of information regarding a calculation job.

job_id: job id (e.g. pid or from queueing system)
path: path to job directory
input_fname: input file name
output_fname: output file name
status: current status of job.  See JobStatus for defined statuses.  This must
be an attribute of JobStatus.
submit: submit script file name.
comment: further information regarding the job.

Only job_id, program and path are required.  All other attributes are optional.
Not all attributes are always applicable.
'''
    def __init__(self, job_id, program, path, input_fname=None, output_fname=None, status=None, submit=None, comment=None):
        self.job_id = job_id
        self.program = program
        self.path = path
        self.input_fname = input_fname
        self.output_fname = output_fname
        self.status = status
        self.submit = submit
        self.comment = comment
        # time since epoch job entry was modified.  useful for merging job caches.
        self._timestamp = time.gmtime()

    def __repr__(self):
        return (self.job_id, self.path, self.input_fname, self.output_fname, self.status, self.submit, self.comment).__repr__()

    def mtime(self):
        '''Return the last time (in seconds since the epoch) that the job was modified.'''
        return self._timestamp

    def auto_update(self):
        '''Update status of job automatically.

This inspects the output from ps and any queueing system to discover if the
status of the job has changed (e.g. if the job has started or has finished).
Note that this assumes the job is running on the local computer.  Warning: if
this condition is not met, then the job status will be incorrectly updated to
finished.

Currently only aware of the PBS queueing system.

Only jobs which are currently held, queueing or running are updated.
'''
        # Check the queueing systems.
        # To add a queueing system, add a dictionary to the list.
        queues = [
            dict(
                command=["ps", "aux"],  # command to list all jobs
                job_column=1,  # column of output which contains the job id field (0-indexed).
                status_column=7,  # column of output which contains the status field.
                held=None,  # string which indicates a held status (not used if None).
                queueing=None,  # string which indicates a queueing status (not used if None).
                running=None,  # string which indicates a running status (not used if None).
            ),
            dict(
                command=["qstat"],
                job_column=0,
                status_column=4,
                held='H',
                queueing='Q',
                running='R',
            ),
        ]

        if self.status in (JobStatus.held, JobStatus.queueing, JobStatus.running):

            for queue in queues:<commit_msg>auto-update for LoadLeveler queueing system implemented.<commit_after>'''Calculation manager.

This module provides classes for managing collections of calculations.
For persistant information it is best to interact via a JobCache instance.
'''

import copy
import os
import os.path
import pickle
import re
import time
import subprocess

### Custom exceptions ###

class UserError(Exception):
    '''Raised due to an error from user-input.'''
    pass


class LockException(Exception):
    '''Raised if a lock cannot be acquired.'''
    pass

### Cache classes ###

class JobStatus(object):
    '''enum-esque class for specifying the status of a job.

Defined statuses: held, queueing, running, finished, analysed.'''
    held = 'held'
    queueing = 'queueing'
    running = 'running'
    finished = 'finished'
    analysed = 'analysed'


class Job:
    '''Store of information regarding a calculation job.

job_id: job id (e.g. pid or from queueing system)
path: path to job directory
input_fname: input file name
output_fname: output file name
status: current status of job.  See JobStatus for defined statuses.  This must
be an attribute of JobStatus.
submit: submit script file name.
comment: further information regarding the job.

Only job_id, program and path are required.  All other attributes are optional.
Not all attributes are always applicable.
'''
    def __init__(self, job_id, program, path, input_fname=None, output_fname=None, status=None, submit=None, comment=None):
        self.job_id = job_id
        self.program = program
        self.path = path
        self.input_fname = input_fname
        self.output_fname = output_fname
        self.status = status
        self.submit = submit
        self.comment = comment
        # time since epoch job entry was modified.  useful for merging job caches.
        self._timestamp = time.gmtime()

    def __repr__(self):
        return (self.job_id, self.path, self.input_fname, self.output_fname, self.status, self.submit, self.comment).__repr__()

    def mtime(self):
        '''Return the last time (in seconds since the epoch) that the job was modified.'''
        return self._timestamp

    def auto_update(self):
        '''Update status of job automatically.

This inspects the output from ps and any queueing system to discover if the
status of the job has changed (e.g. if the job has started or has finished).
Note that this assumes the job is running on the local computer.  Warning: if
this condition is not met, then the job status will be incorrectly updated to
finished.

Currently only aware of the PBS and LoadLeveler queueing systems.

Only jobs which are currently held, queueing or running are updated.
'''
        # Check the queueing systems.
        # To add a queueing system, add a dictionary to the list.
        queues = [
            dict(
                command=["ps", "aux"],  # command to list all jobs
                job_column=1,  # column of output which contains the job id field (0-indexed).
                status_column=7,  # column of output which contains the status field.
                held=None,  # string which indicates a held status (not used if None).
                queueing=None,  # string which indicates a queueing status (not used if None).
                running=None,  # string which indicates a running status (not used if None).
            ),
            dict(
                command=["qstat"],
                job_column=0,
                status_column=4,
                held='H',
                queueing='Q',
                running='R',
            ),
            dict(
                command=["llq"],
                job_column=0,
                status_column=3,
                held='H|NQ|S',
                queueing='I',
                running='R',
            ),
        ]

        if self.status in (JobStatus.held, JobStatus.queueing, JobStatus.running):

            for queue in queues:
{'fraction_non_alphanumeric': 0.07935254638768259, 'fraction_numerical': 0.0015791551519936833, 'mean_word_length': 2.538175046554935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4500976', 'n_tokens_mistral': 2044, 'n_tokens_neox': 1983, 'n_words': 839}