<commit_before>"""
Copyright (c) 2015 SONATA-NFV and Paderborn University
ALL RIGHTS RESERVED.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
nor the names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.

This work has been performed in the framework of the SONATA project,
funded by the European Commission under Grant number 671517 through
the Horizon 2020 and 5G-PPP programmes. The authors would like to
acknowledge the contributions of their colleagues of the SONATA
partner consortium (www.sonata-nfv.eu).
"""
"""

"""
import paramiko
import json
import logging
import mininet.clean
import requests
import time
import threading
import yaml
import os
import stat

# define some constants for easy changing
# will likely be removed as default values dont make sense past testing
DEFAULT_KEY_LOC = "~/.ssh/id_rsa"
PATH_COMMAND = "cd ~/son-emu"
EXEC_COMMAND = "sudo python src/emuvim/examples/profiling.py"
DEFAULT_SSH_USER_NAME = "ssh_user"

# create a Logger
logging.basicConfig()
LOG = logging.getLogger("SON-Profile Emulator")
LOG.setLevel(logging.DEBUG)
logging.getLogger("werkzeug").setLevel(logging.WARNING)

"""
 A class which provides methods to do experiments with service packages
"""
class Emulator:

    """
     Initialize with a list of descriptors of workers to run experiments on
     :tpd_loc: target platforms descriptor. A YAML file describing the emulator nodes available.
    """
    def __init__(self, tpd):
        # if the whole config dictionary has been given, extract only the target platforms
        # a descriptor version should only be in the top level of the file
        if "descriptor_version" in tpd:
            tpd = tpd.get("target_platforms")
        # save the emulator nodes
        self.emulator_nodes = tpd
        # check for empty emulator node lists
        if not len(self.emulator_nodes):
            raise Exception("Need at least one emulator to be specified in the target platforms descriptor.")
        # all nodes are available at the start
        self.available_nodes = self.emulator_nodes.keys()
        LOG.info("%r nodes found."%len(self.emulator_nodes))
        LOG.debug("List of emulator nodes: %r"%self.emulator_nodes.keys())

    """
     Conduct multiple experiments using the do_experiment method
     All experiments are started in a separate thread
     The order in which the experiments are run is not fixed!
     :experiments: a dictionary mapping from run_id to package path
     :runtime: the time an experiment runs for
    """
    def do_experiment_series(self, experiments, runtime=10):
        # start the experiments in separate threads
        for i in experiments.keys():
            t = threading.Thread(target=self.do_experiment, kwargs={"run_id":i, "path_to_pkg":experiments[i], "runtime":runtime})
            t.start()

    """
     Conduct a single experiment with given values
     One experiment consists of:
     1) starting the topology remotely on a server
     2) uploading the service package
     3) starting the service
     4) wait a specified amount of time
     5) stop the service
     6) gather log files
     :path_to_pkg: the path to the service package which is to be tested
     :runtime: the service will run for the specified amoutn of seconds
     :node_name: the name of the emulator node to be used for the experiment. If not specified, the first available node will be used.
    """
    def do_experiment(self,
            path_to_pkg,
            run_id,
            runtime=10,
            node_name=None):
        # if the package path contains a tilde, expand it to full directory path
        path_to_pkg = os.path.expanduser(path_to_pkg)

        # determine which worker to use (specified or first available)
        if not node_name:
            # choose an emulator node
            # the first idle node would be best
            while not self.available_nodes:
                time.sleep(1)
            node_name = self.available_nodes.pop()
        node=self.emulator_nodes[node_name]

        # get neccessary information from (un-)specified worker
        LOG.info("Running package for %r seconds on emulator node %r"%(runtime, node_name))
        address = node["address"]
        # get the port to upload the packages to, if not specified, default to 5000
        package_port = node.get("package_port", 5000)
        # get the ssh port, if not specified, default to 22
        ssh_port = node.get("ssh_port", 22)
        username = node["ssh_user"]
        key_loc = os.path.expanduser(node["ssh_key_loc"])


        # ensure a clean mininet instance
        #mininet.clean.cleanUp().cleanup()

        # connect to the client per ssh
        ssh = paramiko.client.SSHClient()

        # set policy for unknown hosts or import the keys
        # for now, we just add all new keys instead of adding certain ones
        ssh.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())

        # import the RSA key
        pkey = paramiko.RSAKey.from_private_key_file(key_loc)

        # connect to the remote host via ssh<commit_msg>cleaned up and changed some documentation<commit_after>"""
Copyright (c) 2015 SONATA-NFV and Paderborn University
ALL RIGHTS RESERVED.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
nor the names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.

This work has been performed in the framework of the SONATA project,
funded by the European Commission under Grant number 671517 through
the Horizon 2020 and 5G-PPP programmes. The authors would like to
acknowledge the contributions of their colleagues of the SONATA
partner consortium (www.sonata-nfv.eu).
"""
"""

"""
import paramiko
import json
import logging
import requests
import time
import threading
import yaml
import os
import stat

# define some constants for easy changing
# will likely be removed as default values dont make sense past testing
DEFAULT_KEY_LOC = "~/.ssh/id_rsa"
PATH_COMMAND = "cd ~/son-emu"
EXEC_COMMAND = "sudo python src/emuvim/examples/profiling.py"
DEFAULT_SSH_USER_NAME = "ssh_user"

# create a Logger
logging.basicConfig()
LOG = logging.getLogger("SON-Profile Emulator")
LOG.setLevel(logging.DEBUG)
logging.getLogger("werkzeug").setLevel(logging.WARNING)

"""
 A class which provides methods to do experiments with service packages
"""
class Emulator:

    """
     Initialize with a list of descriptors of workers to run experiments on
     :tpd: target platforms descriptor. A dictionary describing the remote hosts on which service can be tested or a dictionary containing a key which value is the descriptor
    """
    def __init__(self, tpd):
        # if the whole config dictionary has been given, extract only the target platforms
        # a descriptor version should only be in the top level of the file
        if "descriptor_version" in tpd:
            tpd = tpd.get("target_platforms")
        # save the emulator nodes
        self.emulator_nodes = tpd
        # check for empty emulator node lists
        if not len(self.emulator_nodes):
            raise Exception("Need at least one emulator to be specified in the target platforms descriptor.")
        # all nodes are available at the start
        self.available_nodes = self.emulator_nodes.keys()
        LOG.info("%r nodes found."%len(self.emulator_nodes))
        LOG.debug("List of emulator nodes: %r"%self.emulator_nodes.keys())

    """
     Conduct multiple experiments using the do_experiment method
     All experiments are started in a separate thread
     The order in which the experiments are run is not fixed!
     :experiments: a dictionary mapping from run_id to package path
     :runtime: the time an experiment runs for
    """
    def do_experiment_series(self, experiments, runtime=10):
        # start the experiments in separate threads
        for i in experiments.keys():
            t = threading.Thread(target=self.do_experiment, kwargs={"run_id":i, "path_to_pkg":experiments[i], "runtime":runtime})
            t.start()

    """
     Conduct a single experiment with given values
     One experiment consists of:
     1) starting the topology remotely on a server
     2) uploading the service package
     3) starting the service
     4) wait a specified amount of time
     5) stop the service
     6) gather log files
     7) clean up log files from remote server
     8) close the connection
     :path_to_pkg: the path to the service package which is to be tested
     :run_id: the experiment ID for this run. Log files will be found under result/run_id
     :runtime: the service will run for the specified amoutn of seconds
     :node_name: the name of the emulator node to be used for the experiment. If not specified, the first available node will be used.
    """
    def do_experiment(self,
            path_to_pkg,
            run_id,
            runtime=10,
            node_name=None):
        # if the package path contains a tilde, expand it to full directory path
        path_to_pkg = os.path.expanduser(path_to_pkg)

        # determine which worker to use (specified or first available)
        if not node_name:
            # choose an emulator node
            # the first idle node would be best
            while not self.available_nodes:
                time.sleep(1)
            node_name = self.available_nodes.pop()
        node=self.emulator_nodes[node_name]

        # get neccessary information from (un-)specified worker
        LOG.info("Running package for %r seconds on emulator node %r"%(runtime, node_name))
        address = node["address"]
        # get the port to upload the packages to, if not specified, default to 5000
        package_port = node.get("package_port", 5000)
        # get the ssh port, if not specified, default to 22
        ssh_port = node.get("ssh_port", 22)
        username = node["ssh_user"]
        key_loc = os.path.expanduser(node["ssh_key_loc"])

        # connect to the client per ssh
        ssh = paramiko.client.SSHClient()

        # set policy for unknown hosts or import the keys
        # for now, we just add all new keys instead of adding certain ones
        ssh.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())

        # import the RSA key
        pkey = paramiko.RSAKey.from_private_key_file(key_loc)

        # connect to the remote host via ssh
{'fraction_non_alphanumeric': 0.06451892266389539, 'fraction_numerical': 0.0074478219450939635, 'mean_word_length': 3.1125356125356127, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2856431', 'n_tokens_mistral': 3139, 'n_tokens_neox': 2999, 'n_words': 1329}