<commit_before>import atexit
from ConfigParser import RawConfigParser
from mininet.topo import Topo
import logging
import os
import random
import re
import subprocess
import tempfile
import threading
import time

import Pyro4

if hasattr(Pyro4.config, 'SERIALIZERS_ACCEPTED'):
    # From Pyro 4.25, pickle is not supported by default due to security.
    # However, it is required to serialise some objects used by maxinet.
    Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
Pyro4.config.SERIALIZER = 'pickle'

class MaxiNetConfig(RawConfigParser):

    def __init__(self, file=None, register=False, **args):
        RawConfigParser.__init__(self, **args)
        self.logger = logging.getLogger(__name__)
        self.daemon = None
        if(file is None):
            self.read(["MaxiNet.cfg", os.path.expanduser("~/.MaxiNet.cfg"),
                       "/etc/MaxiNet.cfg"])
        self.set_loglevel()
        if(register):
            self.register()

    def set_loglevel(self, level=None):
        if(level is None):
            level = self.get_loglevel()
        logging.basicConfig(level=level)

    def get_nameserver_port(self):
        return self.getint("all", "port_ns")

    def get_sshd_port(self):
        return self.getint("all", "port_sshd")

    def get_frontend_ip(self):
        return self.get("FrontendServer", "ip")

    def get_controller(self):
        return self.get("all", "controller")

    def get_worker_ip(self, hostname, classifier=None):
        if(not self.has_section(hostname)):
            self.logger.warn("Unknown hostname: %s" % hostname)
            return None
        else:
            if(classifier is None):
                return self.get(hostname, "ip")
            else:
                if(not self.has_option(hostname, "ip_%s" % classifier)):
                    return self.get_worker_ip(hostname)
                else:
                    return self.get(hostname, "ip_%s" % classifier)

    def run_with_1500_mtu(self):
        if(self.has_option("all","runWith1500MTU")):
            return self.getboolean("all","runWith1500MTU")
        return False

    def get_nameserver_ip(self):
        return self.get_frontend_ip()

    def get_nameserver_password(self):
        return self.get("all", "password")

    def get_loglevel(self):
        lvl = self.get("all", "logLevel")
        lvls = {"CRITICAL": logging.CRITICAL,
                "ERROR": logging.ERROR,
                "WARNING": logging.WARNING,
                "INFO": logging.INFO,
                "DEBUG": logging.DEBUG}
        return lvls[lvl]

    def register(self):
        self.nameserver = Pyro4.locateNS(self.get_nameserver_ip(), self.get_nameserver_port(), hmac_key=self.get_nameserver_password())
        self.daemon = Pyro4.Daemon(host=self.get_nameserver_ip())
        self.daemon._pyroHmacKey = self.get_nameserver_password()
        uri = self.daemon.register(self)
        self.daemon_thread = threading.Thread(target=self.daemon.requestLoop)
        self.daemon_thread.daemon = True
        self.daemon_thread.start()
        self.nameserver.register("config", uri)
        atexit.register(self.unregister)

    def unregister(self):
        if(self.daemon):
            self.nameserver.remove("config")
            self.daemon.shutdown()
            self.daemon = None
            self.daemon_thread.join()
            self.daemon_thread = None


class SSH_Tool(object):

    def __init__(self, config):
        self.config = config
        (self.key_priv, self.key_pub) = self._generate_ssh_key()
        self.known_hosts = tempfile.mkstemp()[1]

    def _generate_ssh_key(self):
        folder = tempfile.mkdtemp()
        subprocess.call(["ssh-keygen", "-t", "rsa", "-q", "-N", "", "-f",
                         os.path.join(folder, "sshkey")])
        return (os.path.join(folder, "sshkey"), os.path.join(folder, "sshkey.pub"))

    def get_pub_ssh_key(self):
        with open(self.key_pub) as fn:
            return fn.read().strip()

    def get_ssh_cmd(self, targethostname, cmd, opts=None):
        rip = self.config.get_worker_ip(targethostname)
        user = self.config.get("all", "sshuser")
        if(rip is None):
            return None
        cm = ["ssh", "-p", str(self.config.get_sshd_port()), "-o",
              "UserKnownHostsFile=%s" % self.known_hosts,
              "-q", "-i", self.key_priv]
        if(opts):
            cm.extend(opts)
        cm.append("%s@%s" % (user, rip))
        if(type(cmd) == str):
            if(self.config.getboolean("all", "usesudo")):
                cmd = "sudo "+cmd
            cm.append(cmd)
        else:
            if(self.config.getboolean("all", "usesudo")):
                cmd = ["sudo"] + cmd
            cm.extend(cmd)
        return cm

    def get_scp_put_cmd(self, targethostname, local, remote, opts=None):
        rip = self.config.get_worker_ip(targethostname)
        user = self.config.get("all", "sshuser")
        if(rip is None):
            return None
        cmd = ["scp", "-P", str(self.config.get_sshd_port()), "-o",
               "UserKnownHostsFile=%s" % self.known_hosts,
               "-r", "-i", self.key_priv]
        if(opts):
            cmd.extend(opts)
        cmd.extend([local, "%s@%s:%s" % (user, rip, remote)])
        return cmd

    def get_scp_get_cmd(self, targethostname, remote, local, opts=None):
        rip = self.config.get_worker_ip(targethostname)
        user = self.config.get("all", "sshuser")
        if(rip is None):
            return None
        cmd = ["scp", "-P", str(self.config.get_sshd_port()), "-o",
               "UserKnownHostsFile=%s" % self.known_hosts,
               "-r", "-i", self.key_priv]
        if(opts):
            cmd.extend(opts)
        cmd.extend(["%s@%s:%s" % (user, rip, remote), local])
        return cmd

    def add_known_host(self, ip):
        with open(self.known_hosts, "a") as kh:
            fp = subprocess.check_output(["ssh-keyscan", "-p",
                                          str(self.config.get_sshd_port()), ip])
            kh.write(fp)

#
# Fat-tree topology implemention for mininet
#
class FatTree(Topo):
    def randByte(self):
        return hex(random.randint(0, 255))[2:]

    def makeMAC(self, i):
        return self.randByte() + ":" + self.randByte() + ":" + \
               self.randByte() + ":00:00:" + hex(i)[2:]

    def makeDPID(self, i):
        a = self.makeMAC(i)
        dp = "".join(re.findall(r'[a-f0-9]+', a))
        return "0" * (12 - len(dp)) + dp

    # args is a string defining the arguments of the topology!
    # has be to format: "x,y,z" to have x hosts and a bw limit of y for
    # those hosts each and a latency of z (in ms) per hop
    def __init__(self, hosts=2, bwlimit=10, lat=0.1, **opts):
        Topo.__init__(self, **opts)<commit_msg>remove ssh keyfile after usage<commit_after>import atexit
from ConfigParser import RawConfigParser
from mininet.topo import Topo
import logging
import os
import random
import re
import subprocess
import tempfile
import threading
import time

import Pyro4

if hasattr(Pyro4.config, 'SERIALIZERS_ACCEPTED'):
    # From Pyro 4.25, pickle is not supported by default due to security.
    # However, it is required to serialise some objects used by maxinet.
    Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
Pyro4.config.SERIALIZER = 'pickle'

class MaxiNetConfig(RawConfigParser):

    def __init__(self, file=None, register=False, **args):
        RawConfigParser.__init__(self, **args)
        self.logger = logging.getLogger(__name__)
        self.daemon = None
        if(file is None):
            self.read(["MaxiNet.cfg", os.path.expanduser("~/.MaxiNet.cfg"),
                       "/etc/MaxiNet.cfg"])
        self.set_loglevel()
        if(register):
            self.register()

    def set_loglevel(self, level=None):
        if(level is None):
            level = self.get_loglevel()
        logging.basicConfig(level=level)

    def get_nameserver_port(self):
        return self.getint("all", "port_ns")

    def get_sshd_port(self):
        return self.getint("all", "port_sshd")

    def get_frontend_ip(self):
        return self.get("FrontendServer", "ip")

    def get_controller(self):
        return self.get("all", "controller")

    def get_worker_ip(self, hostname, classifier=None):
        if(not self.has_section(hostname)):
            self.logger.warn("Unknown hostname: %s" % hostname)
            return None
        else:
            if(classifier is None):
                return self.get(hostname, "ip")
            else:
                if(not self.has_option(hostname, "ip_%s" % classifier)):
                    return self.get_worker_ip(hostname)
                else:
                    return self.get(hostname, "ip_%s" % classifier)

    def run_with_1500_mtu(self):
        if(self.has_option("all","runWith1500MTU")):
            return self.getboolean("all","runWith1500MTU")
        return False

    def get_nameserver_ip(self):
        return self.get_frontend_ip()

    def get_nameserver_password(self):
        return self.get("all", "password")

    def get_loglevel(self):
        lvl = self.get("all", "logLevel")
        lvls = {"CRITICAL": logging.CRITICAL,
                "ERROR": logging.ERROR,
                "WARNING": logging.WARNING,
                "INFO": logging.INFO,
                "DEBUG": logging.DEBUG}
        return lvls[lvl]

    def register(self):
        self.nameserver = Pyro4.locateNS(self.get_nameserver_ip(), self.get_nameserver_port(), hmac_key=self.get_nameserver_password())
        self.daemon = Pyro4.Daemon(host=self.get_nameserver_ip())
        self.daemon._pyroHmacKey = self.get_nameserver_password()
        uri = self.daemon.register(self)
        self.daemon_thread = threading.Thread(target=self.daemon.requestLoop)
        self.daemon_thread.daemon = True
        self.daemon_thread.start()
        self.nameserver.register("config", uri)
        atexit.register(self.unregister)

    def unregister(self):
        if(self.daemon):
            self.nameserver.remove("config")
            self.daemon.shutdown()
            self.daemon = None
            self.daemon_thread.join()
            self.daemon_thread = None


class SSH_Tool(object):

    def __init__(self, config):
        self.config = config
        (self.key_priv, self.key_pub) = self._generate_ssh_key()
        self.known_hosts = tempfile.mkstemp()[1]

    def _generate_ssh_key(self):
        folder = tempfile.mkdtemp()
        subprocess.call(["ssh-keygen", "-t", "rsa", "-q", "-N", "", "-f",
                         os.path.join(folder, "sshkey")])
        atexit.register(self._cleanup)
        return (os.path.join(folder, "sshkey"), os.path.join(folder, "sshkey.pub"))

    def get_pub_ssh_key(self):
        with open(self.key_pub) as fn:
            return fn.read().strip()

    def get_ssh_cmd(self, targethostname, cmd, opts=None):
        rip = self.config.get_worker_ip(targethostname)
        user = self.config.get("all", "sshuser")
        if(rip is None):
            return None
        cm = ["ssh", "-p", str(self.config.get_sshd_port()), "-o",
              "UserKnownHostsFile=%s" % self.known_hosts,
              "-q", "-i", self.key_priv]
        if(opts):
            cm.extend(opts)
        cm.append("%s@%s" % (user, rip))
        if(type(cmd) == str):
            if(self.config.getboolean("all", "usesudo")):
                cmd = "sudo "+cmd
            cm.append(cmd)
        else:
            if(self.config.getboolean("all", "usesudo")):
                cmd = ["sudo"] + cmd
            cm.extend(cmd)
        return cm

    def get_scp_put_cmd(self, targethostname, local, remote, opts=None):
        rip = self.config.get_worker_ip(targethostname)
        user = self.config.get("all", "sshuser")
        if(rip is None):
            return None
        cmd = ["scp", "-P", str(self.config.get_sshd_port()), "-o",
               "UserKnownHostsFile=%s" % self.known_hosts,
               "-r", "-i", self.key_priv]
        if(opts):
            cmd.extend(opts)
        cmd.extend([local, "%s@%s:%s" % (user, rip, remote)])
        return cmd

    def get_scp_get_cmd(self, targethostname, remote, local, opts=None):
        rip = self.config.get_worker_ip(targethostname)
        user = self.config.get("all", "sshuser")
        if(rip is None):
            return None
        cmd = ["scp", "-P", str(self.config.get_sshd_port()), "-o",
               "UserKnownHostsFile=%s" % self.known_hosts,
               "-r", "-i", self.key_priv]
        if(opts):
            cmd.extend(opts)
        cmd.extend(["%s@%s:%s" % (user, rip, remote), local])
        return cmd

    def add_known_host(self, ip):
        with open(self.known_hosts, "a") as kh:
            fp = subprocess.check_output(["ssh-keyscan", "-p",
                                          str(self.config.get_sshd_port()), ip])
            kh.write(fp)

    def _cleanup(self):
        subprocess.call(["rm", self.key_priv])
        subprocess.call(["rm", self.key_pub])
        subprocess.call(["rmdir", os.path.dirname(self.key_priv)])

#
# Fat-tree topology implemention for mininet
#
class FatTree(Topo):
    def randByte(self):
        return hex(random.randint(0, 255))[2:]

    def makeMAC(self, i):
        return self.randByte() + ":" + self.randByte() + ":" + \
               self.randByte() + ":00:00:" + hex(i)[2:]

    def makeDPID(self, i):
        a = self.makeMAC(i)
        dp = "".join(re.findall(r'[a-f0-9]+', a))
        return "0" * (12 - len(dp)) + dp

    # args is a string defining the arguments of the topology!
    # has be to format: "x,y,z" to have x hosts and a bw limit of y for
    # those hosts each and a latency of z (in ms) per hop
    def __init__(self, hosts=2, bwlimit=10, lat=0.1, **opts):
        Topo.__init__(self, **opts)
{'fraction_non_alphanumeric': 0.13809558238739064, 'fraction_numerical': 0.0060733135709637774, 'mean_word_length': 2.2909826314537236, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2079002', 'n_tokens_mistral': 4590, 'n_tokens_neox': 4415, 'n_words': 914}