<commit_before>"""
This file implements the Job class for pegasus-monitord.
"""

##
#  Copyright 2007-2012 University Of Southern California
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
##

import collections
import json
import logging
import os
import re
from io import StringIO

from Pegasus.tools import utils

logger = logging.getLogger(__name__)

# Global variables
good_rsl = {"maxcputime": 1, "maxtime": 1, "maxwalltime": 1}
MAX_OUTPUT_LENGTH = 2 ** 16 - 1  # Only keep stdout to 64K

# some constants
NOOP_JOB_PREFIX = (
    "noop_"  # prefix for noop jobs for which .out and err files are not created
)

MONITORING_EVENT_START_MARKER = "@@@MONITORING_PAYLOAD - START@@@"
MONITORING_EVENT_END_MARKER = "@@@MONITORING_PAYLOAD - END@@@"

# Used in parse_sub_file
re_rsl_string = re.compile(r"^\s*globusrsl\W", re.IGNORECASE)
re_rsl_clean = re.compile(r"([-_])")
re_site_parse_gvds = re.compile(
    r"^\s*\+(pegasus|wf)_(site|resource)\s*=\s*([\'\"])?(\S+)\3"
)
re_parse_transformation = re.compile(r"^\s*\+pegasus_wf_xformation\s*=\s*(\S+)")
re_parse_derivation = re.compile(r"^\s*\+pegasus_wf_dax_job_id\s*=\s*(\S+)")
re_parse_multiplier_factor = re.compile(r"^\s*\+pegasus_cores\s=\s(\S+)")
re_parse_executable = re.compile(r"^\s*executable\s*=\s*(\S+)")
re_parse_arguments = re.compile(r'^\s*arguments\s*=\s*"([^"\r\n]*)"')
re_parse_environment = re.compile(r"^\s*environment\s*=\s*(.*)")
re_site_parse_euryale = re.compile(r"^\#!\s+site=(\S+)")
re_parse_property = re.compile(r"([^:= \t]+)\s*[:=]?\s*(.*)")
re_parse_input = re.compile(r"^\s*intput\s*=\s*(\S+)")
re_parse_output = re.compile(r"^\s*output\s*=\s*(\S+)")
re_parse_error = re.compile(r"^\s*error\s*=\s*(\S+)")
re_parse_job_class = re.compile(r"^\s*\+pegasus_job_class\s*=\s*(\S+)")
re_parse_pegasuslite_hostname = re.compile(
    r"^.*Executing on host\s*(\S+)(?:\s*IP=(\S+))*$", re.MULTILINE
)


TaskOutput = collections.namedtuple("TaskOutput", ["user_data", "events"])


class IntegrityMetric:<commit_msg>PM-1785 the regex to parse the hostname and ip address needs to account for other text/ environment variables that might follow the ip when a job runs on OSG node For example 2021-06-30 00:05:51: Executing on host lnxfarm274.colorado.edu IP=192.168.3.174 OSG_SITE_NAME=UColorado_HEP GLIDEIN_Site=Colorado GLIDEIN_ResourceName=UColorado_HEP<commit_after>"""
This file implements the Job class for pegasus-monitord.
"""

##
#  Copyright 2007-2012 University Of Southern California
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
##

import collections
import json
import logging
import os
import re
from io import StringIO

from Pegasus.tools import utils

logger = logging.getLogger(__name__)

# Global variables
good_rsl = {"maxcputime": 1, "maxtime": 1, "maxwalltime": 1}
MAX_OUTPUT_LENGTH = 2 ** 16 - 1  # Only keep stdout to 64K

# some constants
NOOP_JOB_PREFIX = (
    "noop_"  # prefix for noop jobs for which .out and err files are not created
)

MONITORING_EVENT_START_MARKER = "@@@MONITORING_PAYLOAD - START@@@"
MONITORING_EVENT_END_MARKER = "@@@MONITORING_PAYLOAD - END@@@"

# Used in parse_sub_file
re_rsl_string = re.compile(r"^\s*globusrsl\W", re.IGNORECASE)
re_rsl_clean = re.compile(r"([-_])")
re_site_parse_gvds = re.compile(
    r"^\s*\+(pegasus|wf)_(site|resource)\s*=\s*([\'\"])?(\S+)\3"
)
re_parse_transformation = re.compile(r"^\s*\+pegasus_wf_xformation\s*=\s*(\S+)")
re_parse_derivation = re.compile(r"^\s*\+pegasus_wf_dax_job_id\s*=\s*(\S+)")
re_parse_multiplier_factor = re.compile(r"^\s*\+pegasus_cores\s=\s(\S+)")
re_parse_executable = re.compile(r"^\s*executable\s*=\s*(\S+)")
re_parse_arguments = re.compile(r'^\s*arguments\s*=\s*"([^"\r\n]*)"')
re_parse_environment = re.compile(r"^\s*environment\s*=\s*(.*)")
re_site_parse_euryale = re.compile(r"^\#!\s+site=(\S+)")
re_parse_property = re.compile(r"([^:= \t]+)\s*[:=]?\s*(.*)")
re_parse_input = re.compile(r"^\s*intput\s*=\s*(\S+)")
re_parse_output = re.compile(r"^\s*output\s*=\s*(\S+)")
re_parse_error = re.compile(r"^\s*error\s*=\s*(\S+)")
re_parse_job_class = re.compile(r"^\s*\+pegasus_job_class\s*=\s*(\S+)")
re_parse_pegasuslite_hostname = re.compile(
    r"^.*Executing on host\s*(\S+)(?:\s*IP=(\S+))*.*$", re.MULTILINE
)


TaskOutput = collections.namedtuple("TaskOutput", ["user_data", "events"])


class IntegrityMetric:
{'fraction_non_alphanumeric': 0.17276499434175782, 'fraction_numerical': 0.014145605431912485, 'mean_word_length': 5.268321513002364, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '411493', 'n_tokens_mistral': 2092, 'n_tokens_neox': 1980, 'n_words': 444}