<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - <NAME>, <EMAIL>, 2018
# - <NAME>, <EMAIL>, 2018

"""
The implementation of data structure to host Job definition.

The main reasons for such incapsulation are to
 - apply in one place all data validation actions (for attributes and values)
 - introduce internal information schema (names of attribues) to remove dependency
 with data structrure, formats, names from external source (PanDA)

:author: <NAME>
:contact: <EMAIL>
:date: February 2018
"""

import os
import re
import ast
import shlex
import pipes

from .basedata import BaseData
from .filespec import FileSpec
from pilot.util.filehandling import get_guid

import logging
logger = logging.getLogger(__name__)


class JobData(BaseData):
    """
        High-level object to host Job definition/settings
    """

    # ## put explicit list of all the attributes with comments for better inline-documentation by Sphinx
    # ## FIX ME LATER: use proper doc format
    # ## incomplete list of attributes .. to be extended once becomes used

    jobid = None                # Unique Job  identifier (forced to be a string)
    taskid = None               # Unique Task identifier, the task that this job belongs to (forced to be a string)

    jobparams = ""         # Job parameters defining the execution of the job
    transformation = ""    # Script execution name

    state = ""            # Current job state
    status = ""           # Current job status
    workdir = ""          # Working directoty for this job

    corecount = 1   # Number of cores as requested by the task
    platform = ""   # cmtconfig value from the task definition

    is_eventservice = False        # True for event service jobs

    transfertype = ""  # direct access instruction from server
    accessmode = ""  # direct access instruction from jobparams
    processingtype = ""  # e.g. nightlies

    # set by the pilot (not from job definition)
    workdirsizes = []  # time ordered list of work dir sizes
    fileinfo = {}
    piloterrorcode = 0  # current pilot error code
    piloterrorcodes = []  # ordered list of stored pilot error codes
    piloterrordiag = ""  # current pilot error diagnostics
    piloterrordiags = []  # ordered list of stored pilot error diagnostics
    transexitcode = 0
    exeerrorcode = 0
    exeerrordiag = ""
    exitcode = 0
    exitmsg = ""
    state = ""  # internal pilot state; running, failed, finished, holding
    stageout = ""  # stage-out identifier, e.g. log
    metadata = {}  # payload metadata (job report)
    cpuconsumptionunit = ""
    cpuconsumptiontime = ""
    cpuconversionfactor = 1
    nevents = 0  # number of events
    neventsw = 0  # number of events written
    dbtime = None
    dbdata = None
    payload = ""  # payload name
    utilities = {}  # utility processes { <name>: [<process handle>, number of launches, command string], .. }
    pid = None  # payload pid
    pgrp = None  # process group

    # time variable used for on-the-fly cpu consumption time measurements done by job monitoring
    t0 = None  # payload startup time

    overwrite_queuedata = {}  # Custom settings extracted from job parameters (--overwriteQueueData) to be used as master values for `QueueData`
    zipmap = ""               # ZIP MAP values extracted from jobparameters
    imagename = ""            # user defined container image name extracted from job parameters

    # from job definition
    attemptnr = 0  # job attempt number
    #ddmendpointin = ""  # comma-separated list (string) of ddm endpoints for input    ## TO BE DEPRECATED: moved to FileSpec (job.indata)
    #ddmendpointout = ""  # comma-separated list (string) of ddm endpoints for output  ## TO BE DEPRECATED: moved to FileSpec (job.outdata)
    destinationdblock = ""  ## to be moved to FileSpec (job.outdata)
    datasetin = ""  ## TO BE DEPRECATED: moved to FileSpec (job.indata)
    #datasetout = ""
    debug = False
    produserid = ""  # the user DN (added to trace report)
    jobdefinitionid = ""  # the job definition id (added to trace report)

    # infiles = ""  # comma-separated list (string) of input files  ## TO BE DEPRECATED: moved to FileSpec (use job.indata instead)
    infilesguids = ""

    indata = []   # list of `FileSpec` objects for input files (aggregated inFiles, ddmEndPointIn, scopeIn, filesizeIn, etc)
    outdata = []  # list of `FileSpec` objects for output files
    logdata = []  # list of `FileSpec` objects for log file(s)

    # home package string with additional payload release information; does not need to be added to
    # the conversion function since it's already lower case
    homepackage = ""

    jobsetid = ""  # job set id
    #logfile = ""  #  file name for log                                   ## TO BE DEPRECATED: moved to FileSpec (use job.logdata instead)
    #logguid = ""  # unique guid for log file                             ## TO BE DEPRECATED: moved to FileSpec (use job.logdata instead)
    noexecstrcnv = None  # server instruction to the pilot if it should take payload setup from job parameters
    #outfiles = ""  # comma-separated list (string) of output files       ## TO BE DEPRECATED: moved to FileSpec (job.outdata)
    #scopein = ""  # comma-separated list (string) of input file scopes   ## TO BE DEPRECATED: moved to FileSpec (job.indata)
    #scopelog = ""  # scope for log file                                  ## TO BE DEPRECATED: moved to FileSpec (use job.logdata instead)
    #scopeout = ""  # comma-separated list (string) of output file scopes ## TO BE DEPRECATED: moved to FileSpec (use job.logdata instead)
    swrelease = ""  # software release string

    # RAW data to keep backward compatible behavior for a while ## TO BE REMOVED once all job attributes will be covered
    _rawdata = {}

    # specify the type of attributes for proper data validation and casting
    _keys = {int: ['corecount', 'piloterrorcode', 'transexitcode', 'exitcode', 'cpuconversionfactor', 'exeerrorcode',
                   'attemptnr', 'nevents', 'neventsw', 'pid'],
             str: ['jobid', 'taskid', 'jobparams', 'transformation', 'destinationdblock', 'exeerrordiag'
                   'state', 'status', 'workdir', 'stageout',
                   'platform', 'piloterrordiag', 'exitmsg', 'produserid', 'jobdefinitionid',
                   #'infiles',         ## TO BE DEPRECATED: moved to FileSpec (job.indata)
                   #'scopein',        ## TO BE DEPRECATED: moved to FileSpec (job.indata)
                   #'outfiles', 'ddmendpointin',   ## TO BE DEPRECATED: moved to FileSpec (job.indata)
                   #'scopeout', 'ddmendpointout',    ## TO BE DEPRECATED: moved to FileSpec (job.outdata)
                   #'scopelog', 'logfile', 'logguid',            ## TO BE DEPRECATED: moved to FileSpec (job.logdata)
                   'cpuconsumptionunit', 'cpuconsumptiontime', 'homepackage', 'jobsetid', 'payload', 'processingtype',
                   'swrelease', 'zipmap', 'imagename', 'transfertype',
                   'datasetin',    ## TO BE DEPRECATED: moved to FileSpec (job.indata)
                   #'datasetout',  ## TO BE DEPRECATED: moved to FileSpec (job.outdata)
                   'infilesguids'],
             list: ['piloterrorcodes', 'piloterrordiags', 'workdirsizes'],
             dict: ['fileinfo', 'metadata', 'utilities', 'overwrite_queuedata'],
             bool: ['is_eventservice', 'noexecstrcnv', 'debug']
             }

    def __init__(self, data):
        """
            :param data: input dictionary of data settings
        """

        self.infosys = None  # reference to Job specific InfoService instace
        self._rawdata = data  ###  TEMPORARY CACHE -- REMOVE ME LATER once all fields moved to Job object attributes<commit_msg>Added accessmode to _keys dictionary<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - <NAME>, <EMAIL>, 2018
# - <NAME>, <EMAIL>, 2018

"""
The implementation of data structure to host Job definition.

The main reasons for such incapsulation are to
 - apply in one place all data validation actions (for attributes and values)
 - introduce internal information schema (names of attribues) to remove dependency
 with data structrure, formats, names from external source (PanDA)

:author: <NAME>
:contact: <EMAIL>
:date: February 2018
"""

import os
import re
import ast
import shlex
import pipes

from .basedata import BaseData
from .filespec import FileSpec
from pilot.util.filehandling import get_guid

import logging
logger = logging.getLogger(__name__)


class JobData(BaseData):
    """
        High-level object to host Job definition/settings
    """

    # ## put explicit list of all the attributes with comments for better inline-documentation by Sphinx
    # ## FIX ME LATER: use proper doc format
    # ## incomplete list of attributes .. to be extended once becomes used

    jobid = None                # Unique Job  identifier (forced to be a string)
    taskid = None               # Unique Task identifier, the task that this job belongs to (forced to be a string)

    jobparams = ""         # Job parameters defining the execution of the job
    transformation = ""    # Script execution name

    state = ""            # Current job state
    status = ""           # Current job status
    workdir = ""          # Working directoty for this job

    corecount = 1   # Number of cores as requested by the task
    platform = ""   # cmtconfig value from the task definition

    is_eventservice = False        # True for event service jobs

    transfertype = ""  # direct access instruction from server
    accessmode = ""  # direct access instruction from jobparams
    processingtype = ""  # e.g. nightlies

    # set by the pilot (not from job definition)
    workdirsizes = []  # time ordered list of work dir sizes
    fileinfo = {}
    piloterrorcode = 0  # current pilot error code
    piloterrorcodes = []  # ordered list of stored pilot error codes
    piloterrordiag = ""  # current pilot error diagnostics
    piloterrordiags = []  # ordered list of stored pilot error diagnostics
    transexitcode = 0
    exeerrorcode = 0
    exeerrordiag = ""
    exitcode = 0
    exitmsg = ""
    state = ""  # internal pilot state; running, failed, finished, holding
    stageout = ""  # stage-out identifier, e.g. log
    metadata = {}  # payload metadata (job report)
    cpuconsumptionunit = ""
    cpuconsumptiontime = ""
    cpuconversionfactor = 1
    nevents = 0  # number of events
    neventsw = 0  # number of events written
    dbtime = None
    dbdata = None
    payload = ""  # payload name
    utilities = {}  # utility processes { <name>: [<process handle>, number of launches, command string], .. }
    pid = None  # payload pid
    pgrp = None  # process group

    # time variable used for on-the-fly cpu consumption time measurements done by job monitoring
    t0 = None  # payload startup time

    overwrite_queuedata = {}  # Custom settings extracted from job parameters (--overwriteQueueData) to be used as master values for `QueueData`
    zipmap = ""               # ZIP MAP values extracted from jobparameters
    imagename = ""            # user defined container image name extracted from job parameters

    # from job definition
    attemptnr = 0  # job attempt number
    #ddmendpointin = ""  # comma-separated list (string) of ddm endpoints for input    ## TO BE DEPRECATED: moved to FileSpec (job.indata)
    #ddmendpointout = ""  # comma-separated list (string) of ddm endpoints for output  ## TO BE DEPRECATED: moved to FileSpec (job.outdata)
    destinationdblock = ""  ## to be moved to FileSpec (job.outdata)
    datasetin = ""  ## TO BE DEPRECATED: moved to FileSpec (job.indata)
    #datasetout = ""
    debug = False
    produserid = ""  # the user DN (added to trace report)
    jobdefinitionid = ""  # the job definition id (added to trace report)

    # infiles = ""  # comma-separated list (string) of input files  ## TO BE DEPRECATED: moved to FileSpec (use job.indata instead)
    infilesguids = ""

    indata = []   # list of `FileSpec` objects for input files (aggregated inFiles, ddmEndPointIn, scopeIn, filesizeIn, etc)
    outdata = []  # list of `FileSpec` objects for output files
    logdata = []  # list of `FileSpec` objects for log file(s)

    # home package string with additional payload release information; does not need to be added to
    # the conversion function since it's already lower case
    homepackage = ""

    jobsetid = ""  # job set id
    #logfile = ""  #  file name for log                                   ## TO BE DEPRECATED: moved to FileSpec (use job.logdata instead)
    #logguid = ""  # unique guid for log file                             ## TO BE DEPRECATED: moved to FileSpec (use job.logdata instead)
    noexecstrcnv = None  # server instruction to the pilot if it should take payload setup from job parameters
    #outfiles = ""  # comma-separated list (string) of output files       ## TO BE DEPRECATED: moved to FileSpec (job.outdata)
    #scopein = ""  # comma-separated list (string) of input file scopes   ## TO BE DEPRECATED: moved to FileSpec (job.indata)
    #scopelog = ""  # scope for log file                                  ## TO BE DEPRECATED: moved to FileSpec (use job.logdata instead)
    #scopeout = ""  # comma-separated list (string) of output file scopes ## TO BE DEPRECATED: moved to FileSpec (use job.logdata instead)
    swrelease = ""  # software release string

    # RAW data to keep backward compatible behavior for a while ## TO BE REMOVED once all job attributes will be covered
    _rawdata = {}

    # specify the type of attributes for proper data validation and casting
    _keys = {int: ['corecount', 'piloterrorcode', 'transexitcode', 'exitcode', 'cpuconversionfactor', 'exeerrorcode',
                   'attemptnr', 'nevents', 'neventsw', 'pid'],
             str: ['jobid', 'taskid', 'jobparams', 'transformation', 'destinationdblock', 'exeerrordiag'
                   'state', 'status', 'workdir', 'stageout',
                   'platform', 'piloterrordiag', 'exitmsg', 'produserid', 'jobdefinitionid',
                   #'infiles',         ## TO BE DEPRECATED: moved to FileSpec (job.indata)
                   #'scopein',        ## TO BE DEPRECATED: moved to FileSpec (job.indata)
                   #'outfiles', 'ddmendpointin',   ## TO BE DEPRECATED: moved to FileSpec (job.indata)
                   #'scopeout', 'ddmendpointout',    ## TO BE DEPRECATED: moved to FileSpec (job.outdata)
                   #'scopelog', 'logfile', 'logguid',            ## TO BE DEPRECATED: moved to FileSpec (job.logdata)
                   'cpuconsumptionunit', 'cpuconsumptiontime', 'homepackage', 'jobsetid', 'payload', 'processingtype',
                   'swrelease', 'zipmap', 'imagename', 'accessmode', 'transfertype',
                   'datasetin',    ## TO BE DEPRECATED: moved to FileSpec (job.indata)
                   #'datasetout',  ## TO BE DEPRECATED: moved to FileSpec (job.outdata)
                   'infilesguids'],
             list: ['piloterrorcodes', 'piloterrordiags', 'workdirsizes'],
             dict: ['fileinfo', 'metadata', 'utilities', 'overwrite_queuedata'],
             bool: ['is_eventservice', 'noexecstrcnv', 'debug']
             }

    def __init__(self, data):
        """
            :param data: input dictionary of data settings
        """

        self.infosys = None  # reference to Job specific InfoService instace
        self._rawdata = data  ###  TEMPORARY CACHE -- REMOVE ME LATER once all fields moved to Job object attributes
{'fraction_non_alphanumeric': 0.09192026495032181, 'fraction_numerical': 0.003249390739236393, 'mean_word_length': 2.7149489322191274, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1032799', 'n_tokens_mistral': 4511, 'n_tokens_neox': 4300, 'n_words': 1704}