<commit_before>#!/usr/bin/env python
from __future__ import print_function
import sys
import subprocess
sys.path.append('../lib/')
from argo_communicator import ArgoCommunicator
import logging, os

if os.path.split(os.getcwd())[-1] == 'lib' or os.path.split(os.getcwd())[-1] == 'cron_scripts':
    sys.path.append('../include')
else:
    sys.path.append('include')

import py_functions

py_functions.setLogging()

if __name__ == "__main__":
    argo = ArgoCommunicator()
    secant_conf_path = os.environ.get('SECANT_CONFIG_DIR', '/etc/secant') + '/' + 'secant.conf'
    url = py_functions.getSettingsFromBashConfFile(secant_conf_path, "IMAGE_LIST_URL")
    dir = py_functions.getSettingsFromBashConfFile(secant_conf_path, "IMAGE_LIST_DIR")
    state_dir = py_functions.getSettingsFromBashConfFile(secant_conf_path, "STATE_DIR")
    registered_dir = state_dir + "/registered"
    if not os.path.isdir(registered_dir):
        os.mkdir(registered_dir, 755)

    images = argo.get_templates_for_assessment(dir)
    logging.info("Secant consumer: Obtained %d image list(s) for assessment" % (len(images)))

    for img_list in images:
        #sudo -u cloudkeeper /opt/cloudkeeper/bin/cloudkeeper --image-lists=https://vmcaster.appdb.egi.eu/store/vappliance/demo.va.public/image.list --debug
        #img_list = "https://vmcaster.appdb.egi.eu/store/vappliance/demo.va.public/image.list"
        logging.debug("Secant consumer: Trying to register image list %s" % (img_list))
        img_url = "%s/%s" % (url, img_list)
        cmd = (["sudo", "-u", "cloudkeeper", "/opt/cloudkeeper/bin/cloudkeeper", "--image-lists=" + img_url, "--debug"])
        try:
            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            logging.error("Secant consumer: Registering image list %s failed: %s" % (img_list, e.output))
            print("Failed to register image %s, check the log." % img_list, file=sys.stderr)
            continue
        logging.debug("Secant consumer: Image list %s has been registered" % (img_list))
        os.rename("%s/%s" % (dir, img_list), "%s/%s" % (registered_dir, img_list))
<commit_msg>Use LOG_DIR to keep the output of cloudkeeper<commit_after>#!/usr/bin/env python
from __future__ import print_function
import sys
import subprocess
sys.path.append('../lib/')
from argo_communicator import ArgoCommunicator
import logging, os

if os.path.split(os.getcwd())[-1] == 'lib' or os.path.split(os.getcwd())[-1] == 'cron_scripts':
    sys.path.append('../include')
else:
    sys.path.append('include')

import py_functions

py_functions.setLogging()

if __name__ == "__main__":
    argo = ArgoCommunicator()
    secant_conf_path = os.environ.get('SECANT_CONFIG_DIR', '/etc/secant') + '/' + 'secant.conf'
    url = py_functions.getSettingsFromBashConfFile(secant_conf_path, "IMAGE_LIST_URL")
    dir = py_functions.getSettingsFromBashConfFile(secant_conf_path, "IMAGE_LIST_DIR")
    state_dir = py_functions.getSettingsFromBashConfFile(secant_conf_path, "STATE_DIR")
    log_dir = py_functions.getSettingsFromBashConfFile(secant_conf_path, "LOG_DIR")

    registered_dir = state_dir + "/registered"
    if not os.path.isdir(registered_dir):
        os.mkdir(registered_dir, 755)

    images = argo.get_templates_for_assessment(dir)
    logging.info("Secant consumer: Obtained %d image list(s) for assessment" % (len(images)))

    cloudkeeper_log = open(log_dir + "/cloudkeeper.log", mode="a");

    for img_list in images:
        #sudo -u cloudkeeper /opt/cloudkeeper/bin/cloudkeeper --image-lists=https://vmcaster.appdb.egi.eu/store/vappliance/demo.va.public/image.list --debug
        #img_list = "https://vmcaster.appdb.egi.eu/store/vappliance/demo.va.public/image.list"
        logging.debug("Secant consumer: Trying to register image list %s" % (img_list))
        img_url = "%s/%s" % (url, img_list)
        cmd = (["sudo", "-u", "cloudkeeper", "/opt/cloudkeeper/bin/cloudkeeper", "--image-lists=" + img_url, "--debug"])
        try:
            subprocess.check_call(cmd, stdout=cloudkeeper_log, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            logging.error("Secant consumer: Registering image list %s failed: %s" % (img_list, e.output))
            print("Failed to register image %s, check the log." % img_list, file=sys.stderr)
            continue
        logging.debug("Secant consumer: Image list %s has been registered" % (img_list))
        os.rename("%s/%s" % (dir, img_list), "%s/%s" % (registered_dir, img_list))

    cloudkeeper_log.close()

{'fraction_non_alphanumeric': 0.1412355380921196, 'fraction_numerical': 0.0021829294913774284, 'mean_word_length': 3.9005347593582886, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1526209', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1520, 'n_words': 302}