<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - <NAME>, <EMAIL>

import os
import re

from pilot.util.information import get_container_options, get_container_type, get_catchall
from pilot.user.atlas.setup import get_file_system_root_path

import logging
logger = logging.getLogger(__name__)


def wrapper(executable, **kwargs):
    """
    Wrapper function for any container specific usage.
    This function will be called by pilot.util.container.execute() and prepends the executable with a container command.

    :param executable: command to be executed (string).
    :param kwargs:
    :return: executable wrapped with container command (string).
    """

    platform = kwargs.get('platform', '')
    workdir = kwargs.get('workdir', '.')
    pilot_home = os.environ.get('PILOT_HOME', '')
    if workdir == '.' and pilot_home != '':
        workdir = pilot_home

    return singularity_wrapper(executable, platform, workdir)


def use_payload_container(job):
    pass


def use_middleware_container():
    """
    Should middleware from container be used?
    In case middleware, i.e. the copy command for stage-in/out, should be taken from a container this function should
    return True.

    :return: True if middleware should be taken from container. False otherwise.
    """

    if get_middleware_type() == 'container':
        return True
    else:
        return False


def get_middleware_container():
    pass


def extract_container_options():
    """ Extract any singularity options from catchall """

    # e.g. catchall = "somestuff singularity_options=\'-B /etc/grid-security/certificates,/var/spool/slurmd,/cvmfs,/ceph/grid,/data0,/sys/fs/cgroup\'"
    # catchall = "singularity_options=\'-B /etc/grid-security/certificates,/cvmfs,${workdir} --contain\'" #readpar("catchall")

    # ${workdir} should be there, otherwise the pilot cannot add the current workdir
    # if not there, add it

    # First try with reading new parameters from schedconfig
    container_options = get_container_options()
    if container_options == "":
        logger.warning("container_options either does not exist in queuedata or is empty, trying with catchall instead")
        catchall = get_catchall()
        # E.g. catchall = "singularity_options=\'-B /etc/grid-security/certificates,/cvmfs,${workdir} --contain\'"

        pattern = re.compile(r"singularity\_options\=\'?\"?(.+)\'?\"?")
        found = re.findall(pattern, catchall)
        if len(found) > 0:
            container_options = found[0]
            logger.info('extracted from catchall: %s' % str(container_options))

    if container_options and container_options != "":
        if container_options.endswith("'") or container_options.endswith('"'):<commit_msg>More debug log messages<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - <NAME>, <EMAIL>

import os
import re

from pilot.util.information import get_container_options, get_container_type, get_catchall
from pilot.user.atlas.setup import get_file_system_root_path

import logging
logger = logging.getLogger(__name__)


def wrapper(executable, **kwargs):
    """
    Wrapper function for any container specific usage.
    This function will be called by pilot.util.container.execute() and prepends the executable with a container command.

    :param executable: command to be executed (string).
    :param kwargs:
    :return: executable wrapped with container command (string).
    """

    platform = kwargs.get('platform', '')
    workdir = kwargs.get('workdir', '.')
    pilot_home = os.environ.get('PILOT_HOME', '')
    if workdir == '.' and pilot_home != '':
        workdir = pilot_home

    return singularity_wrapper(executable, platform, workdir)


def use_payload_container(job):
    pass


def use_middleware_container():
    """
    Should middleware from container be used?
    In case middleware, i.e. the copy command for stage-in/out, should be taken from a container this function should
    return True.

    :return: True if middleware should be taken from container. False otherwise.
    """

    if get_middleware_type() == 'container':
        return True
    else:
        return False


def get_middleware_container():
    pass


def extract_container_options():
    """ Extract any singularity options from catchall """

    # e.g. catchall = "somestuff singularity_options=\'-B /etc/grid-security/certificates,/var/spool/slurmd,/cvmfs,/ceph/grid,/data0,/sys/fs/cgroup\'"
    # catchall = "singularity_options=\'-B /etc/grid-security/certificates,/cvmfs,${workdir} --contain\'" #readpar("catchall")

    # ${workdir} should be there, otherwise the pilot cannot add the current workdir
    # if not there, add it

    # First try with reading new parameters from schedconfig
    container_options = get_container_options()
    if container_options == "":
        logger.warning("container_options either does not exist in queuedata or is empty, trying with catchall instead")
        catchall = get_catchall()
        logger.info('catchall=%s'%str(catchall))
        # E.g. catchall = "singularity_options=\'-B /etc/grid-security/certificates,/cvmfs,${workdir} --contain\'"

        pattern = re.compile(r"singularity\_options\=\'?\"?(.+)\'?\"?")
        found = re.findall(pattern, catchall)
        logger.info('found=%s'%str(found))
        if len(found) > 0:
            container_options = found[0]
            logger.info('extracted from catchall: %s' % str(container_options))

    logger.info('2 container_options=%s'%str(container_options))
    if container_options and container_options != "":
        if container_options.endswith("'") or container_options.endswith('"'):
{'fraction_non_alphanumeric': 0.11689377770481038, 'fraction_numerical': 0.002462649811196848, 'mean_word_length': 3.6046863189720333, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3803686', 'n_tokens_mistral': 1816, 'n_tokens_neox': 1755, 'n_words': 518}