<commit_before># coding=utf-8
import csv
import os
import tempfile
from multiprocessing.pool import ThreadPool
from threading import Lock

from sandbox_scripts.QualiEnvironmentUtils.ConfigFileManager import ConfigFileManager
from sandbox_scripts.QualiEnvironmentUtils.ConfigPoolManager import ConfigPoolManager
from sandbox_scripts.helpers.Networking.base_save_restore import *
from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError


class NetworkingSaveRestore(object):
    def __init__(self, sandbox):
        """
        Get the root directory for the config files on the storage server
        :param SandboxBase sandbox:  The sandbox save & restore will be done in
        """
        self.sandbox = sandbox
        storage_server_resource = self.sandbox.get_storage_server_resource()
        self.storage_mgr = None
        if storage_server_resource is not None:
            self.storage_mgr = StorageManager(sandbox)
            self.config_files_root = self.storage_mgr.get_configs_root()
        else:
            if self.is_resources_in_reservation_to_restore(ignore_models=None):
                self.sandbox.report_info("Failed to find a storage server resource (e.g. ftp) in the sandbox. ",
                                         write_to_output_window=False)

    # ----------------------------------
    # load_network_config(ResourceName,config_type, RestoreMethod=Override)
    # ResourceName - The name of the resource we would like to load the config onto
    # ConfigPath – the path to the configuration file, including the configuration file name.
    # The path should include the protocol type. This input is mandatory.
    # the configuration file name doesnt include “StartUp” or “Running”
    # Restore Method – optional, if empty the default value will be taken.
    # Possible values – Append or Override
    # Default value – Override
    # The command should fail if the configuration file name doesnt include “StartUp” or “Running”
    # config_stage - Gold, Base, Snapshot...
    # A path to a config file will look like ROOT_DIR/CONFIG_STAGE/BlueprintX/resourceY_ModelZ.cfg
    # e.g. tftp://configs/Gold/Large_Office/svl290-gg07-sw1_c3850.cfg
    # Base config is an exception. The blueprint's name is not in the path
    # e.g. tftp://configs/Base/svl290-gg07-sw1_c3850.cfg
    # ----------------------------------
    def load_config(self, config_stage, config_type, restore_method="Override", config_set_name='', ignore_models=None,
                    write_to_output=True, remove_temp_files=False, use_Config_file_path_attr=False):
        """
        Load the configuration from config files on the Blueprint's devices
        :param str config_stage:  The stage of the config e.g Gold, Base
        :param str config_type:  Possible values - StartUp or Running
        :param str restore_method: Optional. Restore method. Can be Append or Override
        :param str config_set_name: Optional. If we have multiple configuration sets for the same blueprint.
        the nam of the set selected by the user
        :param list[str] ignore_models: Optional. Models that should be ignored and not load config on the device
        :param bool write_to_output: Optional. should messages be sent to the command output.
        """
        root_path = ''
        if config_stage.lower() == 'gold' or config_stage.lower() == 'snapshots':
            root_path = self.config_files_root + '/' + config_stage + '/' + self.sandbox.Blueprint_name + '/'
        elif config_stage.lower() == 'base':
            root_path = self.config_files_root + '/' + config_stage + '/'
        if config_set_name != '':
            root_path = root_path + config_set_name.strip() + '/'

        root_path = root_path.replace(' ', '_')
        self.sandbox.report_info("RootPath: " + root_path, write_to_output_window=True)
        images_path_dict = self._get_images_path_dict(root_path)
        self.sandbox.report_info("\nLoading image and configuration on the devices. This action may take some time",
                                 write_to_output_window=True)
        root_resources = self.sandbox.get_root_networking_resources()
        """:type : list[ResourceBase]"""
        if len(root_resources) > 0:
            pool = ThreadPool(len(root_resources))
            lock = Lock()
            async_results = [pool.apply_async(self._run_asynch_load,
                                              (resource, images_path_dict, root_path, ignore_models, config_stage, lock,
                                               use_Config_file_path_attr))
                             for resource in root_resources]

            pool.close()
            pool.join()
            for async_result in async_results:
                res = async_result.get()
                """:type : load_result_struct"""
                if not res.run_result:
                    err = "Failed to load configuration on device " + res.resource_name
                    self.sandbox.report_error(err, write_to_output_window=write_to_output, raise_error=False)
                    self.sandbox.report_error(res.message, raise_error=False)
                    self.sandbox.api_session.SetResourceLiveStatus(res.resource_name, 'Error')
                elif res.message != '':
                    self.sandbox.report_info(res.resource_name + "\n" + res.message, write_to_output_window=True)
            if remove_temp_files:
                self._remove_temp_config_files()
        else:
            self.sandbox.report_info("No networking resources found to process.")

    # ----------------------------------
    # ----------------------------------
    def _remove_temp_config_files(self):
        root_resources = self.sandbox.get_root_resources()
        """:type : list[ResourceBase]"""
        for resource in root_resources:
            if resource.attribute_exist('Config file path'):
                tmp_config_file_path = resource.get_attribute('Config file path')
                if tmp_config_file_path != '' and tmp_config_file_path.find('/temp/') > -1:
                    try:
                        self.storage_mgr.delete(tmp_config_file_path)
                        # TODO - clean the attribute
                    except QualiError as qe:
                        self.sandbox.report_info("Failed to delete temp config file " + tmp_config_file_path +
                                                 "Error is: " + str(qe))

    # ----------------------------------
    # ----------------------------------
    def _run_asynch_load(self, resource, images_path_dict, root_path, ignore_models, config_stage, lock,
                         use_Config_file_path_attr):
        message = ""
        # run_status = True
        saved_artifact_info = None
        additionalinfo = ''
        load_result = load_result_struct(resource.name)
        # Check if needs to load the config to the device
        load_config_to_device = self._is_load_config_to_device(resource, ignore_models=ignore_models)
        if load_config_to_device:

            self.sandbox.report_info(resource.name + " starting health check", write_to_output_window=True)
            health_check_result = resource.health_check(self.sandbox.id)
            if health_check_result == "":
                self.sandbox.report_info(resource.name + " -- Initial Health Check Passed.")
                try:
                    config_path = ''
                    with lock:
                        config_path = self._get_concrete_config_file_path(root_path, resource, config_stage,
                                                                          write_to_output=True)
                    if use_Config_file_path_attr:
                        resource.set_attribute_value('Config file path', config_path)
                    # TODO - Snapshots currently only restore configuration. We need to restore firmware as well
                    if config_stage.lower() == 'snapshots':
                        if resource.has_command('orchestration_restore'):
                            dest_name = resource.name + '_' + resource.model + '_artifact.txt'
                            dest_name = dest_name.replace(' ', '-')
                            saved_artifact_info = self.storage_mgr.download_artifact_info(config_path, dest_name)
                            resource.orchestration_restore(self.sandbox.id, config_path, saved_artifact_info)
                        else:
                            resource.load_network_config(self.sandbox.id, config_path, 'Running', 'Override')
                    else:
                        if len(images_path_dict) > 0:
                            # check what the device FW version is currently.
                            version = resource.get_version(self.sandbox.id)
                            self.sandbox.report_info(resource.name + ": current version: " + version,
                                                     write_to_output_window=False)
                            # First try with an firmware image key of concrete resource name!!
                            dict_img_version = ''
                            image_key = ''

                            if resource.name in images_path_dict:
                                #  Try using concrete resource name to find entry
                                image_key = resource.name
                            elif (resource.alias + '_' + resource.model).replace(' ', '_') in images_path_dict:
                                # Try using alias_model
                                image_key = (resource.alias + '_' + resource.model).replace(' ', '_')
                            elif resource.model.replace(' ', '_') in images_path_dict:
                                # Try using just model from attribute xxxxxx.Model
                                image_key = resource.model.replace(' ', '_')

                            if image_key:
                                dict_img_version = images_path_dict[image_key].version
                            else:
                                # Getting here means no firmware specified in FirmwareData.csv
                                message += "\n" + resource.name + ": NO firmware version specified in Base FirmwareData.csv"

                            # same image version - Only load config (running override)
                            message += "\n" + resource.name + ": loading configuration from: " + config_path
                            if dict_img_version.lower() == version.lower():
                                resource.load_network_config(self.sandbox.id, config_path=config_path,
                                                             config_type='Running',
                                                             restore_method='Override')
                            # Different image - Load config to the RUNNING ALSO and load the image
                            else:
                                message += "\n" + resource.name + ": loading configuration from:" + config_path
                                resource.load_network_config(self.sandbox.id, config_path,
                                                             config_type='Running',
                                                             restore_method='Override')

                                if dict_img_version != '':
                                    resource_image_path = images_path_dict[image_key].path
                                    message += "\n" + resource.name + ": changing firmware from: " + version + \
                                               " to: " + dict_img_version + " at \n    " + resource_image_path
                                    additionalinfo = " (firmware) "
                                    self.sandbox.report_info(resource.name + " changing firmware from: " + version +
                                                             " to: " + dict_img_version + " at \n    " +
                                                             resource_image_path,
                                                             write_to_output_window=True)
                                    resource.load_firmware(self.sandbox.id, resource_image_path)<commit_msg>turned on one write to output and added a couple periods to msg to op window<commit_after># coding=utf-8
import csv
import os
import tempfile
from multiprocessing.pool import ThreadPool
from threading import Lock

from sandbox_scripts.QualiEnvironmentUtils.ConfigFileManager import ConfigFileManager
from sandbox_scripts.QualiEnvironmentUtils.ConfigPoolManager import ConfigPoolManager
from sandbox_scripts.helpers.Networking.base_save_restore import *
from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError


class NetworkingSaveRestore(object):
    def __init__(self, sandbox):
        """
        Get the root directory for the config files on the storage server
        :param SandboxBase sandbox:  The sandbox save & restore will be done in
        """
        self.sandbox = sandbox
        storage_server_resource = self.sandbox.get_storage_server_resource()
        self.storage_mgr = None
        if storage_server_resource is not None:
            self.storage_mgr = StorageManager(sandbox)
            self.config_files_root = self.storage_mgr.get_configs_root()
        else:
            if self.is_resources_in_reservation_to_restore(ignore_models=None):
                self.sandbox.report_info("Failed to find a storage server resource (e.g. ftp) in the sandbox. ",
                                         write_to_output_window=False)

    # ----------------------------------
    # load_network_config(ResourceName,config_type, RestoreMethod=Override)
    # ResourceName - The name of the resource we would like to load the config onto
    # ConfigPath – the path to the configuration file, including the configuration file name.
    # The path should include the protocol type. This input is mandatory.
    # the configuration file name doesnt include “StartUp” or “Running”
    # Restore Method – optional, if empty the default value will be taken.
    # Possible values – Append or Override
    # Default value – Override
    # The command should fail if the configuration file name doesnt include “StartUp” or “Running”
    # config_stage - Gold, Base, Snapshot...
    # A path to a config file will look like ROOT_DIR/CONFIG_STAGE/BlueprintX/resourceY_ModelZ.cfg
    # e.g. tftp://configs/Gold/Large_Office/svl290-gg07-sw1_c3850.cfg
    # Base config is an exception. The blueprint's name is not in the path
    # e.g. tftp://configs/Base/svl290-gg07-sw1_c3850.cfg
    # ----------------------------------
    def load_config(self, config_stage, config_type, restore_method="Override", config_set_name='', ignore_models=None,
                    write_to_output=True, remove_temp_files=False, use_Config_file_path_attr=False):
        """
        Load the configuration from config files on the Blueprint's devices
        :param str config_stage:  The stage of the config e.g Gold, Base
        :param str config_type:  Possible values - StartUp or Running
        :param str restore_method: Optional. Restore method. Can be Append or Override
        :param str config_set_name: Optional. If we have multiple configuration sets for the same blueprint.
        the nam of the set selected by the user
        :param list[str] ignore_models: Optional. Models that should be ignored and not load config on the device
        :param bool write_to_output: Optional. should messages be sent to the command output.
        """
        root_path = ''
        if config_stage.lower() == 'gold' or config_stage.lower() == 'snapshots':
            root_path = self.config_files_root + '/' + config_stage + '/' + self.sandbox.Blueprint_name + '/'
        elif config_stage.lower() == 'base':
            root_path = self.config_files_root + '/' + config_stage + '/'
        if config_set_name != '':
            root_path = root_path + config_set_name.strip() + '/'

        root_path = root_path.replace(' ', '_')
        self.sandbox.report_info("RootPath: " + root_path, write_to_output_window=True)
        images_path_dict = self._get_images_path_dict(root_path)
        self.sandbox.report_info("\nLoading image and configuration on the devices. This action may take some time.",
                                 write_to_output_window=True)
        root_resources = self.sandbox.get_root_networking_resources()
        """:type : list[ResourceBase]"""
        if len(root_resources) > 0:
            pool = ThreadPool(len(root_resources))
            lock = Lock()
            async_results = [pool.apply_async(self._run_asynch_load,
                                              (resource, images_path_dict, root_path, ignore_models, config_stage, lock,
                                               use_Config_file_path_attr))
                             for resource in root_resources]

            pool.close()
            pool.join()
            for async_result in async_results:
                res = async_result.get()
                """:type : load_result_struct"""
                if not res.run_result:
                    err = "Failed to load configuration on device " + res.resource_name
                    self.sandbox.report_error(err, write_to_output_window=write_to_output, raise_error=False)
                    self.sandbox.report_error(res.message, raise_error=False)
                    self.sandbox.api_session.SetResourceLiveStatus(res.resource_name, 'Error')
                elif res.message != '':
                    self.sandbox.report_info(res.resource_name + "\n" + res.message, write_to_output_window=True)
            if remove_temp_files:
                self._remove_temp_config_files()
        else:
            self.sandbox.report_info("No networking resources found to process.")

    # ----------------------------------
    # ----------------------------------
    def _remove_temp_config_files(self):
        root_resources = self.sandbox.get_root_resources()
        """:type : list[ResourceBase]"""
        for resource in root_resources:
            if resource.attribute_exist('Config file path'):
                tmp_config_file_path = resource.get_attribute('Config file path')
                if tmp_config_file_path != '' and tmp_config_file_path.find('/temp/') > -1:
                    try:
                        self.storage_mgr.delete(tmp_config_file_path)
                        # TODO - clean the attribute
                    except QualiError as qe:
                        self.sandbox.report_info("Failed to delete temp config file " + tmp_config_file_path +
                                                 "Error is: " + str(qe))

    # ----------------------------------
    # ----------------------------------
    def _run_asynch_load(self, resource, images_path_dict, root_path, ignore_models, config_stage, lock,
                         use_Config_file_path_attr):
        message = ""
        # run_status = True
        saved_artifact_info = None
        additionalinfo = ''
        load_result = load_result_struct(resource.name)
        # Check if needs to load the config to the device
        load_config_to_device = self._is_load_config_to_device(resource, ignore_models=ignore_models)
        if load_config_to_device:

            self.sandbox.report_info(resource.name + " starting health check", write_to_output_window=True)
            health_check_result = resource.health_check(self.sandbox.id)
            if health_check_result == "":
                self.sandbox.report_info(resource.name + " -- Initial Health Check Passed.")
                try:
                    config_path = ''
                    with lock:
                        config_path = self._get_concrete_config_file_path(root_path, resource, config_stage,
                                                                          write_to_output=True)
                    if use_Config_file_path_attr:
                        resource.set_attribute_value('Config file path', config_path)
                    # TODO - Snapshots currently only restore configuration. We need to restore firmware as well
                    if config_stage.lower() == 'snapshots':
                        if resource.has_command('orchestration_restore'):
                            dest_name = resource.name + '_' + resource.model + '_artifact.txt'
                            dest_name = dest_name.replace(' ', '-')
                            saved_artifact_info = self.storage_mgr.download_artifact_info(config_path, dest_name)
                            resource.orchestration_restore(self.sandbox.id, config_path, saved_artifact_info)
                        else:
                            resource.load_network_config(self.sandbox.id, config_path, 'Running', 'Override')
                    else:
                        if len(images_path_dict) > 0:
                            # check what the device FW version is currently.
                            version = resource.get_version(self.sandbox.id)
                            self.sandbox.report_info(resource.name + " current version: " + version,
                                                     write_to_output_window=True)
                            # First try with an firmware image key of concrete resource name!!
                            dict_img_version = ''
                            image_key = ''

                            if resource.name in images_path_dict:
                                #  Try using concrete resource name to find entry
                                image_key = resource.name
                            elif (resource.alias + '_' + resource.model).replace(' ', '_') in images_path_dict:
                                # Try using alias_model
                                image_key = (resource.alias + '_' + resource.model).replace(' ', '_')
                            elif resource.model.replace(' ', '_') in images_path_dict:
                                # Try using just model from attribute xxxxxx.Model
                                image_key = resource.model.replace(' ', '_')

                            if image_key:
                                dict_img_version = images_path_dict[image_key].version
                            else:
                                # Getting here means no firmware specified in FirmwareData.csv
                                message += "\n" + resource.name + ": NO firmware version specified in Base FirmwareData.csv"

                            # same image version - Only load config (running override)
                            message += "\n" + resource.name + ": loading configuration from: " + config_path
                            if dict_img_version.lower() == version.lower():
                                resource.load_network_config(self.sandbox.id, config_path=config_path,
                                                             config_type='Running',
                                                             restore_method='Override')
                            # Different image - Load config to the RUNNING ALSO and load the image
                            else:
                                resource.load_network_config(self.sandbox.id, config_path,
                                                             config_type='Running',
                                                             restore_method='Override')

                                if dict_img_version != '':
                                    resource_image_path = images_path_dict[image_key].path
                                    message += "\n" + resource.name + ": changing firmware from: " + version + \
                                               " to: " + dict_img_version + " at \n    " + resource_image_path
                                    additionalinfo = " (firmware) "
                                    self.sandbox.report_info(resource.name + " changing firmware from: " + version +
                                                             " to: " + dict_img_version + " at \n    " +
                                                             resource_image_path,
                                                             write_to_output_window=True)
                                    resource.load_firmware(self.sandbox.id, resource_image_path)
{'fraction_non_alphanumeric': 0.08902379400260756, 'fraction_numerical': 0.001955671447196871, 'mean_word_length': 1.6064564086227036, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 86, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '136076', 'n_tokens_mistral': 5992, 'n_tokens_neox': 5756, 'n_words': 1461}