<commit_before># Copyright 2017 the Isard-vdi project authors:
#      <NAME>
#      <NAME>
# License: AGPLv3
# coding=utf-8

import os
import pprint
import signal
import subprocess
import threading
from os.path import dirname
from time import sleep

from engine.config import CONFIG_DICT
from engine.services.db import (
    delete_domain,
    get_config_branch,
    get_domain,
    get_hyp_hostname_user_port_from_id,
    update_domain_dict_create_dict,
    update_table_field,
)
from engine.services.db.db import (
    get_media_with_status,
    new_rethink_connection,
    remove_media,
)
from engine.services.db.domains import get_domains_with_status, update_domain_status
from engine.services.db.downloads import (
    get_downloads_in_progress,
    get_media,
    update_download_percent,
    update_status_table,
)
from engine.services.db.hypervisors import get_hypers_in_pool
from engine.services.lib.download import test_url_for_download
from engine.services.lib.functions import get_tid
from engine.services.lib.qcow import (
    create_cmds_delete_disk,
    get_host_disk_operations_from_path,
    get_path_to_disk,
)
from engine.services.log import logs
from rethinkdb import r

URL_DOWNLOAD_INSECURE_SSL = True
TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD = 10


class DownloadThread(threading.Thread, object):
    def __init__(
        self,
        url,
        path,
        path_selected,
        table,
        id_down,
        dict_header,
        finalished_threads,
        threads_disk_operations,
        pool_id,
        type_path_selected,
    ):
        threading.Thread.__init__(self)
        self.name = "_".join([table, id_down])
        self.table = table
        self.path = path
        self.path_selected = path_selected
        self.id = id_down
        self.url = url
        self.dict_header = dict_header
        self.stop = False
        self.finalished_threads = finalished_threads

        self.threads_disk_operations = threads_disk_operations
        self.hostname = None
        self.user = None
        self.port = None
        self.pool_id = pool_id
        self.type_path_selected = type_path_selected

    def run(self):

        # if self.table == 'domains':
        #     type_path_selected = 'groups'
        # elif self.table in ['isos']:
        #     type_path_selected = 'isos'
        # else:
        #     type_path_selected = 'media'
        #
        # new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected)
        # logs.downloads.debug("PATHS ___________________________________________________________________")
        # logs.downloads.debug(new_file)
        # logs.downloads.debug(path_selected)
        # logs.downloads.debug(pprint.pformat(self.__dict__))
        #
        # hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool,
        #                                                                 type_path=type_path_selected)

        # hypervisor to launch download command
        # wait to threads disk_operations are alive
        time_elapsed = 0
        path_selected = self.path_selected
        while True:
            if len(self.threads_disk_operations) > 0:

                hyp_to_disk_create = get_host_disk_operations_from_path(
                    path_selected, pool=self.pool_id, type_path=self.type_path_selected
                )
                logs.downloads.debug(
                    f"Thread download started to in hypervisor: {hyp_to_disk_create}"
                )
                if (
                    self.threads_disk_operations.get(hyp_to_disk_create, False)
                    is not False
                ):
                    if self.threads_disk_operations[hyp_to_disk_create].is_alive():
                        d = get_hyp_hostname_user_port_from_id(hyp_to_disk_create)
                        self.hostname = d["hostname"]
                        self.user = d["user"]
                        self.port = d["port"]
                        break
            sleep(0.2)
            time_elapsed += 0.2
            if time_elapsed > TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD:
                logs.downloads.info(
                    f"Timeout ({TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD} sec) waiting hypervisor online to download. Maybe there is no disk_operations selected in Hypervisors???"
                )
                if self.table == "domains":
                    update_domain_status(
                        "Failed",
                        self.id,
                        detail="No hypervisor could do the disk_operations.",
                    )
                else:
                    update_status_table(self.table, "Failed", self.id)
                self.finalished_threads.append(self.path)
                return False

        header_template = "--header '{header_key}: {header_value}' "
        headers = ""

        if URL_DOWNLOAD_INSECURE_SSL == True:
            insecure_option = "--insecure"
        else:
            insecure_option = ""

        dict_header = {}
        for k, v in self.dict_header.items():
            headers += header_template.format(header_key=k, header_value=v)
            dict_header[k] = v

        # TEST IF url return an stream of data
        ok, error_msg = test_url_for_download(
            self.url,<commit_msg>fix(engine): set DownloadFailed if no hypervisor<commit_after># Copyright 2017 the Isard-vdi project authors:
#      <NAME>
#      <NAME>
# License: AGPLv3
# coding=utf-8

import os
import pprint
import signal
import subprocess
import threading
from os.path import dirname
from time import sleep

from engine.config import CONFIG_DICT
from engine.services.db import (
    delete_domain,
    get_config_branch,
    get_domain,
    get_hyp_hostname_user_port_from_id,
    update_domain_dict_create_dict,
    update_table_field,
)
from engine.services.db.db import (
    get_media_with_status,
    new_rethink_connection,
    remove_media,
)
from engine.services.db.domains import get_domains_with_status, update_domain_status
from engine.services.db.downloads import (
    get_downloads_in_progress,
    get_media,
    update_download_percent,
    update_status_table,
)
from engine.services.db.hypervisors import get_hypers_in_pool
from engine.services.lib.download import test_url_for_download
from engine.services.lib.functions import get_tid
from engine.services.lib.qcow import (
    create_cmds_delete_disk,
    get_host_disk_operations_from_path,
    get_path_to_disk,
)
from engine.services.log import logs
from rethinkdb import r

URL_DOWNLOAD_INSECURE_SSL = True
TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD = 10


class DownloadThread(threading.Thread, object):
    def __init__(
        self,
        url,
        path,
        path_selected,
        table,
        id_down,
        dict_header,
        finalished_threads,
        threads_disk_operations,
        pool_id,
        type_path_selected,
    ):
        threading.Thread.__init__(self)
        self.name = "_".join([table, id_down])
        self.table = table
        self.path = path
        self.path_selected = path_selected
        self.id = id_down
        self.url = url
        self.dict_header = dict_header
        self.stop = False
        self.finalished_threads = finalished_threads

        self.threads_disk_operations = threads_disk_operations
        self.hostname = None
        self.user = None
        self.port = None
        self.pool_id = pool_id
        self.type_path_selected = type_path_selected

    def run(self):

        # if self.table == 'domains':
        #     type_path_selected = 'groups'
        # elif self.table in ['isos']:
        #     type_path_selected = 'isos'
        # else:
        #     type_path_selected = 'media'
        #
        # new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected)
        # logs.downloads.debug("PATHS ___________________________________________________________________")
        # logs.downloads.debug(new_file)
        # logs.downloads.debug(path_selected)
        # logs.downloads.debug(pprint.pformat(self.__dict__))
        #
        # hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool,
        #                                                                 type_path=type_path_selected)

        # hypervisor to launch download command
        # wait to threads disk_operations are alive
        time_elapsed = 0
        path_selected = self.path_selected
        while True:
            if len(self.threads_disk_operations) > 0:

                hyp_to_disk_create = get_host_disk_operations_from_path(
                    path_selected, pool=self.pool_id, type_path=self.type_path_selected
                )
                logs.downloads.debug(
                    f"Thread download started to in hypervisor: {hyp_to_disk_create}"
                )
                if (
                    self.threads_disk_operations.get(hyp_to_disk_create, False)
                    is not False
                ):
                    if self.threads_disk_operations[hyp_to_disk_create].is_alive():
                        d = get_hyp_hostname_user_port_from_id(hyp_to_disk_create)
                        self.hostname = d["hostname"]
                        self.user = d["user"]
                        self.port = d["port"]
                        break
            sleep(0.2)
            time_elapsed += 0.2
            if time_elapsed > TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD:
                logs.downloads.info(
                    f"Timeout ({TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD} sec) waiting hypervisor online to download. Maybe there is no disk_operations selected in Hypervisors???"
                )
                if self.table == "domains":
                    update_domain_status(
                        "DownloadFailed",
                        self.id,
                        detail="No hypervisor could do the disk_operations.",
                    )
                else:
                    update_status_table(self.table, "DownloadFailed", self.id)
                self.finalished_threads.append(self.path)
                return False

        header_template = "--header '{header_key}: {header_value}' "
        headers = ""

        if URL_DOWNLOAD_INSECURE_SSL == True:
            insecure_option = "--insecure"
        else:
            insecure_option = ""

        dict_header = {}
        for k, v in self.dict_header.items():
            headers += header_template.format(header_key=k, header_value=v)
            dict_header[k] = v

        # TEST IF url return an stream of data
        ok, error_msg = test_url_for_download(
            self.url,
{'fraction_non_alphanumeric': 0.07158109739114288, 'fraction_numerical': 0.0025995729273048, 'mean_word_length': 2.048104131295982, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 41, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6610893', 'n_tokens_mistral': 3101, 'n_tokens_neox': 2990, 'n_words': 594}