<commit_before>import os
import time
from datetime import date

import cluster
from cluster import PBS
import XnatUtils

# Job Statuses
NEED_TO_RUN='NEED_TO_RUN' # assessor that is ready to be launch on the cluster (ACCRE). All the input data for the process to run are there.
NEED_INPUTS='NEED_INPUTS' # assessor where input data are missing from a scan, multiple scans or other assessor.
JOB_RUNNING='JOB_RUNNING' # the job has been submitted on the cluster and is running right now.
JOB_FAILED='JOB_FAILED' # the job failed on the cluster.
READY_TO_UPLOAD='READY_TO_UPLOAD' # Job done, waiting for the Spider to upload the results
UPLOADING='UPLOADING' # in the process of uploading the resources on XNAT.
COMPLETE='COMPLETE' # the assessors contains all the files. The upload and the job are done.  
READY_TO_COMPLETE='READY_TO_COMPLETE' # the job finished and upload is complete
DOES_NOT_EXIST='DOES_NOT_EXIST'
OPEN_STATUS_LIST = [NEED_TO_RUN, UPLOADING, JOB_RUNNING, READY_TO_COMPLETE, JOB_FAILED]

# QA Statuses
JOB_PENDING = 'Job Pending' # job is still running, not ready for QA yet
NEEDS_QA='Needs QA' # For FS, the complete status
PASSED_QA='Passed' # QA status set by the Image Analyst after looking at the results.
FAILED='Failed' # QA status set by the Image Analyst after looking at the results.
FAILED_NEEDS_REPROC='Failed-needs reprocessing'
PASSED_EDITED_QA='Passed with edits'
RERUN='Rerun' # will cause spider to delete results and rerun the processing
REPROC='Reproc' # will cause spider to zip the current results and put in OLD, and then processing
OPEN_QC_LIST = [RERUN, REPROC]

# Other Constants
DEFAULT_PBS_DIR=os.environ['UPLOAD_SPIDER_DIR']+'/PBS'
DEFAULT_OUT_DIR=os.environ['UPLOAD_SPIDER_DIR']+'/OUTLOG'
READY_TO_UPLOAD_FLAG_FILENAME = 'READY_TO_UPLOAD.txt'
OLD_RESOURCE = 'OLD'
EDITS_RESOURCE = 'EDITS'
REPROC_RES_SKIP_LIST = [OLD_RESOURCE, EDITS_RESOURCE]

class Task(object):
    def __init__(self, processor, assessor, upload_dir):
        self.processor = processor
        self.assessor = assessor
        self.upload_dir = upload_dir    
        self.atype = processor.xsitype.lower()

        # Create assessor if needed
        if not assessor.exists():
            assessor.create(assessors=self.atype)
            self.set_createdate_today()
            if self.atype == 'proc:genprocdata':
                assessor.attrs.set('proc:genprocdata/proctype', self.get_processor_name())
                assessor.attrs.set('proc:genprocdata/validation/status', JOB_PENDING)
                assessor.attrs.set('proc:genprocdata/procversion', self.get_processor_version())
            if processor.has_inputs(assessor):
                self.set_status(NEED_TO_RUN)
            else:
                self.set_status(NEED_INPUTS)
        
        # Cache for convenience
        self.assessor_id = assessor.id()
        self.assessor_label = assessor.label()
                 
    def get_processor_name(self):<commit_msg>RESULTS_DIR<commit_after>import os
import time
from datetime import date

import cluster
from cluster import PBS
import XnatUtils

from constant import RESULTS_DIR

# Job Statuses
NEED_TO_RUN='NEED_TO_RUN' # assessor that is ready to be launch on the cluster (ACCRE). All the input data for the process to run are there.
NEED_INPUTS='NEED_INPUTS' # assessor where input data are missing from a scan, multiple scans or other assessor.
JOB_RUNNING='JOB_RUNNING' # the job has been submitted on the cluster and is running right now.
JOB_FAILED='JOB_FAILED' # the job failed on the cluster.
READY_TO_UPLOAD='READY_TO_UPLOAD' # Job done, waiting for the Spider to upload the results
UPLOADING='UPLOADING' # in the process of uploading the resources on XNAT.
COMPLETE='COMPLETE' # the assessors contains all the files. The upload and the job are done.  
READY_TO_COMPLETE='READY_TO_COMPLETE' # the job finished and upload is complete
DOES_NOT_EXIST='DOES_NOT_EXIST'
OPEN_STATUS_LIST = [NEED_TO_RUN, UPLOADING, JOB_RUNNING, READY_TO_COMPLETE, JOB_FAILED]

# QA Statuses
JOB_PENDING = 'Job Pending' # job is still running, not ready for QA yet
NEEDS_QA='Needs QA' # For FS, the complete status
PASSED_QA='Passed' # QA status set by the Image Analyst after looking at the results.
FAILED='Failed' # QA status set by the Image Analyst after looking at the results.
FAILED_NEEDS_REPROC='Failed-needs reprocessing'
PASSED_EDITED_QA='Passed with edits'
RERUN='Rerun' # will cause spider to delete results and rerun the processing
REPROC='Reproc' # will cause spider to zip the current results and put in OLD, and then processing
OPEN_QC_LIST = [RERUN, REPROC]

# Other Constants
DEFAULT_PBS_DIR=os.path.join(RESULTS_DIR,'PBS')
DEFAULT_OUT_DIR=os.path.join(RESULTS_DIR,'OUTLOG')
READY_TO_UPLOAD_FLAG_FILENAME = 'READY_TO_UPLOAD.txt'
OLD_RESOURCE = 'OLD'
EDITS_RESOURCE = 'EDITS'
REPROC_RES_SKIP_LIST = [OLD_RESOURCE, EDITS_RESOURCE]

class Task(object):
    def __init__(self, processor, assessor, upload_dir):
        self.processor = processor
        self.assessor = assessor
        self.upload_dir = upload_dir    
        self.atype = processor.xsitype.lower()

        # Create assessor if needed
        if not assessor.exists():
            assessor.create(assessors=self.atype)
            self.set_createdate_today()
            if self.atype == 'proc:genprocdata':
                assessor.attrs.set('proc:genprocdata/proctype', self.get_processor_name())
                assessor.attrs.set('proc:genprocdata/validation/status', JOB_PENDING)
                assessor.attrs.set('proc:genprocdata/procversion', self.get_processor_version())
            if processor.has_inputs(assessor):
                self.set_status(NEED_TO_RUN)
            else:
                self.set_status(NEED_INPUTS)
        
        # Cache for convenience
        self.assessor_id = assessor.id()
        self.assessor_label = assessor.label()
                 
    def get_processor_name(self):
{'fraction_non_alphanumeric': 0.07829902126223422, 'fraction_numerical': 0.0, 'mean_word_length': 3.8661740558292284, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3768518', 'n_tokens_mistral': 1910, 'n_tokens_neox': 1921, 'n_words': 541}