<commit_before>#!/usr/bin/python3
import os
import sys
import argparse
from subprocess import check_output, CalledProcessError
import time
import configparser
from pathlib import Path


CLI = argparse.ArgumentParser(
    description="Generate APEL accounting records for an HTCondor CE and LRMS",
    epilog="""
HTCondor configuration values:

  condor_config_val:
    PER_JOB_HISTORY_DIR  path to which per-job history files are written

  condor_ce_config_val:
    APEL_OUTPUT_DIR      path to which APEL record files should be written
    APEL_SCALE_DEFAULT   default scale when no job attribute applies, such as 1.0 or UNDEFINED
    APEL_CE_HOST         hostname of the CE
    APEL_CE_ID           APEL identifier for the CE
    APEL_SCALING_ATTR    job attribute for optional performance factor
    APEL_SPEC_ATTR       job attribute for optional absolute performance
""".strip(),
    formatter_class=argparse.RawDescriptionHelpFormatter,
)
CLI.add_argument(
    "--apel-config",
    help="path to apel client configuration file [default: %(default)s]",
    default="/etc/apel/client.cfg",
    type=Path,
)
CLI.add_argument(
    "--dry-run",
    help="do not perform destructive actions, write data to stdout",
    action="store_true",
)


def read_lrms_config_val(key: str) -> str:
    return check_output(["condor_config_val", key], universal_newlines=True).strip()


def read_ce_config_val(key: str) -> str:
    return check_output(["condor_ce_config_val", key], universal_newlines=True).strip()


def read_apel_specs(apel_config: Path, ce_id: str) -> "dict[str, float]":
    """Read APEL manual_specX entries for a given CE identifier"""
    config = configparser.ConfigParser()
    with apel_config.open() as config_stream:
        config.read_file(config_stream)
    specs = {}
    for spec_id in range(1, len(config["spec_updater"]) + 1):
        try:
            spec = config["spec_updater"][f"manual_spec{spec_id}"]
        except KeyError:
            break
        spec_ce_id, spec_type, spec_value, *_ = spec.split(",")
        if ce_id == spec_ce_id:
            specs[spec_type] = float(spec_value)
    return specs


def format_apel_scaling(apel_config: Path, ce_id: str) -> str:
    """Build a ClassAd expression for the factor of used vs average performance"""
    try:
        scale_query = read_ce_config_val("APEL_SCALE_DEFAULT")
    except CalledProcessError:
        # assume average performance by default
        scale_query = "1.0"
    try:
        scaling_attr = read_ce_config_val("APEL_SCALING_ATTR")
    except CalledProcessError:
        pass
    else:
        scale_query = f"({scaling_attr} ?: {scale_query})"
    try:
        spec_attr = read_ce_config_val("APEL_SPEC_ATTR")
    except CalledProcessError:
        return scale_query
    else:
        specs = read_apel_specs(apel_config, ce_id)
        for spec_type, spec_value in reversed(list(specs.items())):
            scale_query = (
                f"(({spec_attr}.'{spec_type}' / {spec_value}) ?: {scale_query})"
            )
        return scale_query


def condor_q_format(job_history: Path, *formats: str) -> str:
    """Run `condor_q` with several `-format` fields for a given `job_history` file"""
    assert len(formats) % 2 == 0
    format_fields = ["-format"] * (len(formats) // 2 * 3)
    format_fields[1::3] = formats[::2]
    format_fields[2::3] = formats[1::2]
    return check_output(
            ["condor_q", "-job", str(job_history)] + format_fields,
            universal_newlines=True, env={**os.environ, "TZ": "GMT"},
        )


options = CLI.parse_args()
dry_run: bool = options.dry_run<commit_msg>documented logic of APEL scaling expression<commit_after>#!/usr/bin/python3
import os
import sys
import argparse
from subprocess import check_output, CalledProcessError
import time
import configparser
from pathlib import Path


CLI = argparse.ArgumentParser(
    description="Generate APEL accounting records for an HTCondor CE and LRMS",
    epilog="""
HTCondor configuration values:

  condor_config_val:
    PER_JOB_HISTORY_DIR  path to which per-job history files are written

  condor_ce_config_val:
    APEL_OUTPUT_DIR      path to which APEL record files should be written
    APEL_SCALE_DEFAULT   default scale when no job attribute applies, such as 1.0 or UNDEFINED
    APEL_CE_HOST         hostname of the CE
    APEL_CE_ID           APEL identifier for the CE
    APEL_SCALING_ATTR    job attribute for optional performance factor
    APEL_SPEC_ATTR       job attribute for optional absolute performance
""".strip(),
    formatter_class=argparse.RawDescriptionHelpFormatter,
)
CLI.add_argument(
    "--apel-config",
    help="path to apel client configuration file [default: %(default)s]",
    default="/etc/apel/client.cfg",
    type=Path,
)
CLI.add_argument(
    "--dry-run",
    help="do not perform destructive actions, write data to stdout",
    action="store_true",
)


def read_lrms_config_val(key: str) -> str:
    return check_output(["condor_config_val", key], universal_newlines=True).strip()


def read_ce_config_val(key: str) -> str:
    return check_output(["condor_ce_config_val", key], universal_newlines=True).strip()


def read_apel_specs(apel_config: Path, ce_id: str) -> "dict[str, float]":
    """Read APEL manual_specX entries for a given CE identifier"""
    config = configparser.ConfigParser()
    with apel_config.open() as config_stream:
        config.read_file(config_stream)
    specs = {}
    for spec_id in range(1, len(config["spec_updater"]) + 1):
        try:
            spec = config["spec_updater"][f"manual_spec{spec_id}"]
        except KeyError:
            break
        spec_ce_id, spec_type, spec_value, *_ = spec.split(",")
        if ce_id == spec_ce_id:
            specs[spec_type] = float(spec_value)
    return specs


def format_apel_scaling(apel_config: Path, ce_id: str) -> str:
    """Build a ClassAd expression to compute the factor of used vs average performance"""
    # The expression chains several cases via `?:` to fall-through to the first not-UNDEFINED value.
    # This means we build the expression in reverse order, starting at the default and adding
    # higher-priority cases around it.
    try:
        scale_query = read_ce_config_val("APEL_SCALE_DEFAULT")
    except CalledProcessError:
        # assume average performance by default
        scale_query = "1.0"
    try:
        scaling_attr = read_ce_config_val("APEL_SCALING_ATTR")
    except CalledProcessError:
        pass
    else:
        scale_query = f"({scaling_attr} ?: {scale_query})"
    try:
        spec_attr = read_ce_config_val("APEL_SPEC_ATTR")
    except CalledProcessError:
        return scale_query
    else:
        # Match the specs defined by APEL on the CE to those of the machine/job:
        # The result looks like `((MachineAttrApelSpecs0.'HEPSPEC' / 13.45) ?: (...))` where
        # `MachineAttrApelSpecs0` is a ClassAd like `[HEPSPEC=14.37; SI2K=2793]` or UNDEFINED.
        # If the current spec_type is UNDEFINED for the job, we fall-through to the next type.
        specs = read_apel_specs(apel_config, ce_id)
        for spec_type, spec_value in reversed(list(specs.items())):
            scale_query = (
                f"(({spec_attr}.'{spec_type}' / {spec_value}) ?: {scale_query})"
            )
        return scale_query


def condor_q_format(job_history: Path, *formats: str) -> str:
    """Run `condor_q` with several `-format` fields for a given `job_history` file"""
    assert len(formats) % 2 == 0
    format_fields = ["-format"] * (len(formats) // 2 * 3)
    format_fields[1::3] = formats[::2]
    format_fields[2::3] = formats[1::2]
    return check_output(
            ["condor_q", "-job", str(job_history)] + format_fields,
            universal_newlines=True, env={**os.environ, "TZ": "GMT"},
        )


options = CLI.parse_args()
dry_run: bool = options.dry_run
{'fraction_non_alphanumeric': 0.10331681280975982, 'fraction_numerical': 0.006481128478841022, 'mean_word_length': 3.3504698728579325, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4998457', 'n_tokens_mistral': 2595, 'n_tokens_neox': 2492, 'n_words': 664}