<commit_before># -*- coding: utf-8 -*-

# This file is part of Cockpit.
#
# Copyright (C) 2013 Red Hat, Inc.
#
# Cockpit is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Cockpit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.

from contextlib import contextmanager
import os
import sys
import subprocess
import shutil
import stat
import tempfile
import time
from common import testinfra

BASE = testinfra.TEST_DIR
IMAGES = os.path.join(BASE, "images")
DATA = os.path.join(os.environ.get("TEST_DATA", BASE), "images")
# this is the path git uses within the repo, not on the filesystem (which would be IMAGES)
GIT_IMAGE_PATH = "test/images"

# threshold in G below which unreferenced qcow2 images will be pruned, even if they aren't old
PRUNE_THRESHOLD_G = float(os.environ.get("PRUNE_THRESHOLD_G", 15))
DEVNULL = open("/dev/null", "r+")

CONFIG = "~/.config/image-stores"
DEFAULT = [
    "https://192.168.3.11:8493/",
    "https://192.168.127.12:8493/",
    "https://fedorapeople.org/groups/cockpit/images"
]

def download(link, force, stores):
    if not os.path.exists(DATA):
        os.makedirs(DATA)

    dest = os.readlink(link)
    relative_dir = os.path.dirname(os.path.abspath(link))
    full_dest = os.path.join(relative_dir, dest)
    while not ".qcow2" in dest and os.path.islink(full_dest):
        link = full_dest
        dest = os.readlink(link)
        relative_dir = os.path.dirname(os.path.abspath(link))
        full_dest = os.path.join(relative_dir, dest)

    dest = os.path.join(DATA, dest)

    # we have the file but there is not valid link
    if os.path.exists(dest) and not os.path.exists(link):
        os.symlink(dest, os.path.join(IMAGES, os.readlink(link)))

    # The image file in the images directory, may be same as dest
    image_file = os.path.join(IMAGES, os.readlink(link))

    # file already exists, double check that symlink in place
    if not force and os.path.exists(dest):
        if not os.path.exists(image_file):
            os.symlink(os.path.abspath(dest), image_file)
        return

    if not stores:
        config = os.path.expanduser(CONFIG)
        if os.path.exists(config):
            with open(config, 'r') as fp:
                stores = fp.read().strip().split("\n")
        else:
            stores = []
        stores += DEFAULT

    ca = os.path.join(BASE, "common", "ca.pem")
    for store in stores:
        source = os.path.join(store, os.path.basename(dest)) + ".xz"
        try:
            cmd = ["curl", "--head", "--silent", "--fail", "--cacert", ca, source]
            subprocess.check_call(cmd, stdout=DEVNULL)
            break<commit_msg>test: When including CA assume cert is for cockpit-tests<commit_after># -*- coding: utf-8 -*-

# This file is part of Cockpit.
#
# Copyright (C) 2013 Red Hat, Inc.
#
# Cockpit is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Cockpit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.

from contextlib import contextmanager
import os
import sys
import subprocess
import shutil
import stat
import tempfile
import time
import urlparse

from common import testinfra

BASE = testinfra.TEST_DIR
IMAGES = os.path.join(BASE, "images")
DATA = os.path.join(os.environ.get("TEST_DATA", BASE), "images")
# this is the path git uses within the repo, not on the filesystem (which would be IMAGES)
GIT_IMAGE_PATH = "test/images"

# threshold in G below which unreferenced qcow2 images will be pruned, even if they aren't old
PRUNE_THRESHOLD_G = float(os.environ.get("PRUNE_THRESHOLD_G", 15))
DEVNULL = open("/dev/null", "r+")

CONFIG = "~/.config/image-stores"
DEFAULT = [
    "https://192.168.3.11:8493/",
    "https://192.168.127.12:8493/",
    "https://fedorapeople.org/groups/cockpit/images"
]

def download(link, force, stores):
    if not os.path.exists(DATA):
        os.makedirs(DATA)

    dest = os.readlink(link)
    relative_dir = os.path.dirname(os.path.abspath(link))
    full_dest = os.path.join(relative_dir, dest)
    while not ".qcow2" in dest and os.path.islink(full_dest):
        link = full_dest
        dest = os.readlink(link)
        relative_dir = os.path.dirname(os.path.abspath(link))
        full_dest = os.path.join(relative_dir, dest)

    dest = os.path.join(DATA, dest)

    # we have the file but there is not valid link
    if os.path.exists(dest) and not os.path.exists(link):
        os.symlink(dest, os.path.join(IMAGES, os.readlink(link)))

    # The image file in the images directory, may be same as dest
    image_file = os.path.join(IMAGES, os.readlink(link))

    # file already exists, double check that symlink in place
    if not force and os.path.exists(dest):
        if not os.path.exists(image_file):
            os.symlink(os.path.abspath(dest), image_file)
        return

    if not stores:
        config = os.path.expanduser(CONFIG)
        if os.path.exists(config):
            with open(config, 'r') as fp:
                stores = fp.read().strip().split("\n")
        else:
            stores = []
        stores += DEFAULT

    ca = os.path.join(BASE, "common", "ca.pem")
    for store in stores:
        url = urlparse.urlparse(store)
        if url.port:
            resolve = "cockpit-tests:{0}:{1}".format(url.port, url.hostname)
            url = url._replace(netloc="cockpit-tests:{}".format(url.port))
        else:
            resolve = "cockpit-tests:{1}".format(0, url.hostname)
            url = url._replace(netloc="cockpit-tests")

        name = os.path.basename(dest) + ".xz"
        ca_source = urlparse.urljoin(url.geturl(), name)
        source = urlparse.urljoin(store, name)

        try:
            cmd = ["curl", "--head", "--silent",  "--resolve", resolve, "--fail", "--cacert", ca, ca_source]
            subprocess.check_call(cmd, stdout=DEVNULL)
            break
{'fraction_non_alphanumeric': 0.12, 'fraction_numerical': 0.012058823529411764, 'mean_word_length': 3.0029429075927014, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7285184', 'n_tokens_mistral': 2211, 'n_tokens_neox': 2135, 'n_words': 673}