<commit_before>#!/usr/bin/env python3

import os
import re
import time
import ipaddress
import pprint
from functools import partial
import yaml  # pytype: disable=pyi-error

from clib.mininet_test_util import timeout_cmd
from clib.mininet_test_base import FaucetTestBase, IPV4_ETH
from clib.mininet_test_topo_generator import FaucetTopoGenerator


class FaucetTopoTestBase(FaucetTestBase):
    """
    Extension to the base test for the integration test suite to help set up arbitrary topologies
    This is supposed to be called with clib.mininet_topo_generator to take networkx graph
        generators and produce a set of switch-switch links and host-switch links
    """

    NETPREFIX = 24
    IPV = 4
    GROUP_TABLE = False

    NUM_DPS = 2
    NUM_HOSTS = 4
    LINKS_PER_HOST = 1

    dpids = None
    port_maps = None
    n_vlans = 0
    dp_links = None
    host_links = None
    host_vlans = None
    stack_roots = None
    routers = None
    dp_options = None
    host_options = None
    vlan_options = None
    host_information = None
    faucet_vips = None

    def non_host_links(self, dpid):
        """Return dpid peer links from topo"""
        return self.topo.dpid_peer_links(dpid)

    @staticmethod
    def get_config_header(_config_global, _debug_log, _dpid, _hardware):
        """Don't generate standard config file header."""
        return ''

    @staticmethod
    def acls():
        """Dictionary of ACLs"""
        return {}

    @staticmethod
    def acl_in_dp():
        """Dictionary of DP & port map to ACL"""
        return {}

    @staticmethod
    def dp_name(i):
        """DP config name"""
        return 'faucet-%u' % (i + 1)

    def faucet_vip(self, i):
        """Faucet VLAN VIP"""
        return '10.%u.0.254/%u' % (i+1, self.NETPREFIX)

    @staticmethod
    def faucet_mac(i):
        """Faucet VLAN MAC"""
        return '00:00:00:00:00:%u%u' % (i+1, i+1)

    def host_ip_address(self, host_index, vlan_index):
        """Create a string of the host IP address"""
        if isinstance(vlan_index, tuple):
            vlan_index = vlan_index[0]
        return '10.%u.0.%u/%u' % (vlan_index+1, host_index+1, self.NETPREFIX)

    @staticmethod
    def vlan_name(i):
        """VLAN name"""
        return 'vlan-%i' % (i+1)

    @staticmethod
    def vlan_vid(i):
        """VLAN VID value"""
        return (i+1) * 100

    def host_ping(self, src_host, dst_ip):
        """Default method to ping from one host to an IP address"""
        self.one_ipv4_ping(
            src_host, dst_ip, require_host_learned=False, retries=5, timeout=1000)

    def set_host_ip(self, host, host_ip):
        """Default method for setting a hosts IP address"""
        host.setIP(str(host_ip.ip), prefixLen=self.NETPREFIX)

    def build_net(self, n_dps=1, n_vlans=1,
                  dp_links=None, host_links=None, host_vlans=None,
                  vlan_options=None, dp_options=None, host_options=None,
                  routers=None, stack_roots=None,
                  include=None, include_optional=None,
                  hw_dpid=None, lacp=False):
        """
        Use the TopologyGenerator to generate the YAML configuration and create the network
        Args:
            n_dps: Number of DPs
            n_vlans: Number of VLANs
            dp_links (dict): dp index to dp index
            host_links (dict): host index to list of dp index
            host_vlans (dict): host index to vlan index
            vlan_options (dict): vlan_index to key, value dp options
            dp_options (dict): dp index to key, value dp options
            host_options (dict): Host index to host option key, values
            routers (dict): router index to list of vlan index
            stack_roots (dict): dp index to priority value (leave none for tagged links)
            include:
            include_optional:
            hw_dpid: DPID of hardware switch
            lacp: Use LACP trunk ports
        """
        if include is None:
            include = []
        if include_optional is None:
            include_optional = []
        self.NUM_DPS = n_dps
        self.dpids = [str(self.rand_dpid()) for _ in range(n_dps)]
        self.dpids[0] = self.dpid
        vlan_vids = {vlan: self.vlan_vid(vlan) for vlan in range(n_vlans)}
        self.topo = FaucetTopoGenerator(
            self.OVS_TYPE,
            self.ports_sock,
            self._test_name(),
            self.dpids,
            dp_links,
            host_links,
            host_vlans,
            vlan_vids,
            hw_dpid=self.hw_dpid,
            switch_map=self.switch_map,
            port_order=self.port_order,
        )
        self.port_maps = {dpid: self.create_port_map(dpid) for dpid in self.dpids}
        self.port_map = self.port_maps[self.dpid]
        self.CONFIG = self.get_config(
            dpids=self.dpids,
            hw_dpid=hw_dpid,
            hardware=self.hardware,
            ofchannel_log=self.debug_log_path,
            n_vlans=n_vlans,
            host_links=host_links,
            host_vlans=host_vlans,
            stack_roots=stack_roots,
            include=include,
            include_optional=include_optional,
            acls=self.acls(),
            acl_in_dp=self.acl_in_dp(),
            lacp=lacp,
            vlan_options=vlan_options,
            dp_options=dp_options,
            routers=routers,
            host_options=host_options
        )
        self.n_vlans = n_vlans
        self.dp_links = dp_links
        self.host_links = host_links
        self.host_vlans = host_vlans
        self.stack_roots = stack_roots
        self.routers = routers
        self.dp_options = dp_options
        self.host_options = host_options
        self.vlan_options = vlan_options

    def start_net(self):
        """
        Override start_net to create the faucet vips, the host information and set up the
            host routes for routed hosts
        """
        super(FaucetTopoTestBase, self).start_net()
        self.host_information = {}
        for host_id, host_name in self.topo.hosts_by_id.items():
            host_obj = self.net.get(host_name)
            vlan = self.host_vlans[host_id]
            ip_interface = ipaddress.ip_interface(self.host_ip_address(host_id, vlan))
            self.set_host_ip(host_obj, ip_interface)
            self.host_information[host_id] = {
                'host': host_obj,
                'ip': ip_interface,
                'vlan': vlan
            }

        self.faucet_vips = {}
        for vlan in range(self.n_vlans):
            self.faucet_vips[vlan] = ipaddress.ip_interface(self.faucet_vip(vlan))

        if self.routers:
            for src in self.host_information:
                src_host = self.host_information[src]['host']
                src_vlan = self.host_information[src]['vlan']
                src_ip = self.host_information[src]['ip']
                for dst in self.host_information:
                    if src != dst:
                        dst_host = self.host_information[dst]['host']
                        dst_vlan = self.host_information[dst]['vlan']
                        dst_ip = self.host_information[dst]['ip']
                        if src_vlan != dst_vlan and self.is_routed_vlans(src_vlan, dst_vlan):
                            src_faucet_vip = self.faucet_vips[src_vlan]
                            dst_faucet_vip = self.faucet_vips[dst_vlan]
                            self.add_host_route(src_host, dst_ip, src_faucet_vip.ip)
                            self.add_host_route(dst_host, src_ip, dst_faucet_vip.ip)

    def get_config(self, dpids=None, hw_dpid=None, hardware=None, ofchannel_log=None,
                   n_vlans=1, host_links=None, host_vlans=None, stack_roots=None,
                   include=None, include_optional=None, acls=None, acl_in_dp=None,
                   lacp=False, vlan_options=None, dp_options=None,
                   routers=None, host_options=None):
        """
        Args:
            dpids: List of DPIDs the dp indices in the configuration dictionaries refer to
            hw_dpid: DPID for connected hardware switch
            hardware:
            ofchannel_log: Debug log path
            n_vlans: Number of VLANs
            host_links (dict): host index to dp index
            host_vlans (dict): host index to vlan index
            stack_roots (dict): dp index to priority value (leave none for tagged links)
            include:
            include_optional:
            hw_dpid: DPID of hardware switch
            lacp: Use LACP trunk ports
            vlan_options (dict): vlan_index to key, value dp options
            dp_options (dict): dp index to key, value dp options
            routers (dict): router index to list of vlan index
            host_options (dict): Host index to host option key, values
        """
        if dpids is None:
            dpids = []
        if include is None:
            include = []
        if include_optional is None:
            include_optional = []
        if acls is None:
            acls = {}
        if acl_in_dp is None:
            acl_in_dp = {}

        def add_vlans(n_vlans, host_vlans, vlan_options):
            vlans_config = {}
            for vlan in range(n_vlans):
                n_tagged = 0
                n_untagged = 0
                for vlans in host_vlans.values():
                    if isinstance(vlans, int) and vlan == vlans:
                        n_untagged += 1
                    elif isinstance(vlans, tuple) and vlan in vlans:
                        n_tagged += 1
                vlans_config[self.vlan_name(vlan)] = {
                    'description': '%s tagged, %s untagged' % (n_tagged, n_untagged),
                    'vid': self.vlan_vid(vlan)
                }
            if vlan_options:
                for vlan, options in vlan_options.items():
                    for key, value in options.items():
                        vlans_config[self.vlan_name(vlan)][key] = value
            return vlans_config

        def add_routers(routers):
            router_config = {}
            for i, vlans in routers.items():
                router_config['router-%s' % i] = {
                    'vlans': [self.vlan_name(vlan) for vlan in vlans]
                }
            return router_config

        def add_acl_to_port(i, port, interfaces_config):
            if i in acl_in_dp and port in acl_in_dp[i]:
                interfaces_config[port]['acl_in'] = acl_in_dp[i][port]

        def add_dp(i, dpid, hw_dpid, ofchannel_log, group_table,
                   n_vlans, host_vlans, stack_roots, host_links, dpid_peer_links, port_maps):
            dp_config = {
                'dp_id': int(dpid),
                'hardware': hardware if dpid == hw_dpid else 'Open vSwitch',
                'ofchannel_log': ofchannel_log + str(i) if ofchannel_log else None,
                'interfaces': {},
                'group_table': group_table,
            }

            if dp_options and i in dp_options:
                for key, value in dp_options[i].items():
                    dp_config[key] = value

            if stack_roots and i in stack_roots:
                dp_config['stack'] = {}
                dp_config['stack']['priority'] = stack_roots[i]

            interfaces_config = {}
            # Generate host links
            index = 1
            for host_id, links in host_links.items():
                if i in links:
                    n_links = links.count(i)
                    vlan = host_vlans[host_id]
                    if isinstance(vlan, int):
                        key = 'native_vlan'
                        value = self.vlan_name(vlan)
                    else:
                        key = 'tagged_vlans'
                        value = [self.vlan_name(vlan) for vlan in vlan]
                    for _ in range(n_links):
                        port = port_maps[dpid]['port_%d' % index]
                        interfaces_config[port] = {
                            key: value
                        }
                        if host_options and host_id in host_options:
                            for option_key, option_value in host_options[host_id].items():
                                interfaces_config[port][option_key] = option_value
                        index += 1
                        add_acl_to_port(i, port, interfaces_config)

            # Generate switch-switch links
            for link in dpid_peer_links:
                port, peer_dpid, peer_port = link.port, link.peer_dpid, link.peer_port
                interfaces_config[port] = {}
                if stack_roots:
                    interfaces_config[port].update({
                        'stack': {
                            'dp': self.dp_name(dpids.index(peer_dpid)),
                            'port': peer_port
                        }})
                else:
                    tagged_vlans = [self.vlan_name(vlan) for vlan in range(n_vlans)]
                    interfaces_config[port].update({'tagged_vlans': tagged_vlans})
                    if lacp:
                        interfaces_config[port].update({
                            'lacp': 1,
                            'lacp_active': True
                        })
                        dp_config['lacp_timeout'] = 10
                add_acl_to_port(i, port, interfaces_config)

            dp_config['interfaces'] = interfaces_config
            return dp_config

        config = {'version': 2}
        if include:
            config['include'] = list(include)
        if include_optional:
            config['include_optional'] = list(include_optional)
        config['acls'] = acls.copy()
        config['vlans'] = add_vlans(n_vlans, host_vlans, vlan_options)

        if routers:
            config['routers'] = add_routers(routers)

        dpid_names = {dpids[i]: self.dp_name(i) for i in range(len(dpids))}
        self.set_dpid_names(dpid_names)
        config['dps'] = {}
        for i, dpid in enumerate(dpids):
            config['dps'][self.dp_name(i)] = add_dp(
                i, dpid, hw_dpid, ofchannel_log, self.GROUP_TABLE, n_vlans, host_vlans,
                stack_roots, host_links, self.topo.dpid_peer_links(dpid), self.port_maps)

        return yaml.dump(config, default_flow_style=False)

    def debug(self):
        """Print host information when debugging"""
        try:
            super(FaucetTopoTestBase, self).debug()
        except:
            pprint.pprint(self.host_information)
            raise

    def verify_no_cable_errors(self):
        """Check that prometheus does not detect any stack cabling errors on all DPs"""
        i = 0
        for dpid in self.dpids:
            i += 1
            labels = {'dp_id': '0x%x' % int(dpid), 'dp_name': 'faucet-%u' % i}
            self.assertEqual(
                0, self.scrape_prometheus_var(
                    var='stack_cabling_errors_total', labels=labels, default=None))
            self.assertGreater(
                self.scrape_prometheus_var(
                    var='stack_probes_received_total', labels=labels), 0)<commit_msg>stickler<commit_after>#!/usr/bin/env python3

import os
import re
import time
import ipaddress
import pprint
from functools import partial
import yaml  # pytype: disable=pyi-error

from clib.mininet_test_util import timeout_cmd
from clib.mininet_test_base import FaucetTestBase, IPV4_ETH
from clib.mininet_test_topo_generator import FaucetTopoGenerator


class FaucetTopoTestBase(FaucetTestBase):
    """
    Extension to the base test for the integration test suite to help set up arbitrary topologies
    This is supposed to be called with clib.mininet_topo_generator to take networkx graph
        generators and produce a set of switch-switch links and host-switch links
    """

    NETPREFIX = 24
    IPV = 4
    GROUP_TABLE = False

    NUM_DPS = 2
    NUM_HOSTS = 4
    LINKS_PER_HOST = 1

    dpids = None
    port_maps = None
    n_vlans = 0
    dp_links = None
    host_links = None
    host_vlans = None
    stack_roots = None
    routers = None
    dp_options = None
    host_options = None
    vlan_options = None
    host_information = None
    faucet_vips = None

    def non_host_links(self, dpid):
        """Return dpid peer links from topo"""
        return self.topo.dpid_peer_links(dpid)

    @staticmethod
    def get_config_header(_config_global, _debug_log, _dpid, _hardware):
        """Don't generate standard config file header."""
        return ''

    @staticmethod
    def acls():
        """Dictionary of ACLs"""
        return {}

    @staticmethod
    def acl_in_dp():
        """Dictionary of DP & port map to ACL"""
        return {}

    @staticmethod
    def dp_name(i):
        """DP config name"""
        return 'faucet-%u' % (i + 1)

    def faucet_vip(self, i):
        """Faucet VLAN VIP"""
        return '10.%u.0.254/%u' % (i+1, self.NETPREFIX)

    @staticmethod
    def faucet_mac(i):
        """Faucet VLAN MAC"""
        return '00:00:00:00:00:%u%u' % (i+1, i+1)

    def host_ip_address(self, host_index, vlan_index):
        """Create a string of the host IP address"""
        if isinstance(vlan_index, tuple):
            vlan_index = vlan_index[0]
        return '10.%u.0.%u/%u' % (vlan_index+1, host_index+1, self.NETPREFIX)

    @staticmethod
    def vlan_name(i):
        """VLAN name"""
        return 'vlan-%i' % (i+1)

    @staticmethod
    def vlan_vid(i):
        """VLAN VID value"""
        return (i+1) * 100

    def host_ping(self, src_host, dst_ip):
        """Default method to ping from one host to an IP address"""
        self.one_ipv4_ping(
            src_host, dst_ip, require_host_learned=False, retries=5, timeout=1000)

    def set_host_ip(self, host, host_ip):
        """Default method for setting a hosts IP address"""
        host.setIP(str(host_ip.ip), prefixLen=self.NETPREFIX)

    def build_net(self, n_dps=1, n_vlans=1,
                  dp_links=None, host_links=None, host_vlans=None,
                  vlan_options=None, dp_options=None, host_options=None,
                  routers=None, stack_roots=None,
                  include=None, include_optional=None,
                  hw_dpid=None, lacp=False):
        """
        Use the TopologyGenerator to generate the YAML configuration and create the network
        Args:
            n_dps: Number of DPs
            n_vlans: Number of VLANs
            dp_links (dict): dp index to dp index
            host_links (dict): host index to list of dp index
            host_vlans (dict): host index to vlan index
            vlan_options (dict): vlan_index to key, value dp options
            dp_options (dict): dp index to key, value dp options
            host_options (dict): Host index to host option key, values
            routers (dict): router index to list of vlan index
            stack_roots (dict): dp index to priority value (leave none for tagged links)
            include:
            include_optional:
            hw_dpid: DPID of hardware switch
            lacp: Use LACP trunk ports
        """
        if include is None:
            include = []
        if include_optional is None:
            include_optional = []
        self.NUM_DPS = n_dps
        self.dpids = [str(self.rand_dpid()) for _ in range(n_dps)]
        self.dpids[0] = self.dpid
        vlan_vids = {vlan: self.vlan_vid(vlan) for vlan in range(n_vlans)}
        self.topo = FaucetTopoGenerator(
            self.OVS_TYPE,
            self.ports_sock,
            self._test_name(),
            self.dpids,
            dp_links,
            host_links,
            host_vlans,
            vlan_vids,
            hw_dpid=self.hw_dpid,
            switch_map=self.switch_map,
            port_order=self.port_order,
        )
        self.port_maps = {dpid: self.create_port_map(dpid) for dpid in self.dpids}
        self.port_map = self.port_maps[self.dpid]
        self.CONFIG = self.get_config(
            dpids=self.dpids,
            hw_dpid=hw_dpid,
            hardware=self.hardware,
            ofchannel_log=self.debug_log_path,
            n_vlans=n_vlans,
            host_links=host_links,
            host_vlans=host_vlans,
            stack_roots=stack_roots,
            include=include,
            include_optional=include_optional,
            acls=self.acls(),
            acl_in_dp=self.acl_in_dp(),
            lacp=lacp,
            vlan_options=vlan_options,
            dp_options=dp_options,
            routers=routers,
            host_options=host_options
        )
        self.n_vlans = n_vlans
        self.dp_links = dp_links
        self.host_links = host_links
        self.host_vlans = host_vlans
        self.stack_roots = stack_roots
        self.routers = routers
        self.dp_options = dp_options
        self.host_options = host_options
        self.vlan_options = vlan_options

    def start_net(self):
        """
        Override start_net to create the faucet vips, the host information and set up the
            host routes for routed hosts
        """
        super(FaucetTopoTestBase, self).start_net()
        self.host_information = {}
        for host_id, host_name in self.topo.hosts_by_id.items():
            host_obj = self.net.get(host_name)
            vlan = self.host_vlans[host_id]
            ip_interface = ipaddress.ip_interface(self.host_ip_address(host_id, vlan))
            self.set_host_ip(host_obj, ip_interface)
            self.host_information[host_id] = {
                'host': host_obj,
                'ip': ip_interface,
                'vlan': vlan
            }

        self.faucet_vips = {}
        for vlan in range(self.n_vlans):
            self.faucet_vips[vlan] = ipaddress.ip_interface(self.faucet_vip(vlan))

        if self.routers:
            for src in self.host_information:
                src_host = self.host_information[src]['host']
                src_vlan = self.host_information[src]['vlan']
                src_ip = self.host_information[src]['ip']
                for dst in self.host_information:
                    if src != dst:
                        dst_host = self.host_information[dst]['host']
                        dst_vlan = self.host_information[dst]['vlan']
                        dst_ip = self.host_information[dst]['ip']
                        if src_vlan != dst_vlan and self.is_routed_vlans(src_vlan, dst_vlan):
                            src_faucet_vip = self.faucet_vips[src_vlan]
                            dst_faucet_vip = self.faucet_vips[dst_vlan]
                            self.add_host_route(src_host, dst_ip, src_faucet_vip.ip)
                            self.add_host_route(dst_host, src_ip, dst_faucet_vip.ip)

    def get_config(self, dpids=None, hw_dpid=None, hardware=None, ofchannel_log=None,
                   n_vlans=1, host_links=None, host_vlans=None, stack_roots=None,
                   include=None, include_optional=None, acls=None, acl_in_dp=None,
                   lacp=False, vlan_options=None, dp_options=None,
                   routers=None, host_options=None):
        """
        Args:
            dpids: List of DPIDs the dp indices in the configuration dictionaries refer to
            hw_dpid: DPID for connected hardware switch
            hardware:
            ofchannel_log: Debug log path
            n_vlans: Number of VLANs
            host_links (dict): host index to dp index
            host_vlans (dict): host index to vlan index
            stack_roots (dict): dp index to priority value (leave none for tagged links)
            include:
            include_optional:
            hw_dpid: DPID of hardware switch
            lacp: Use LACP trunk ports
            vlan_options (dict): vlan_index to key, value dp options
            dp_options (dict): dp index to key, value dp options
            routers (dict): router index to list of vlan index
            host_options (dict): Host index to host option key, values
        """
        if dpids is None:
            dpids = []
        if include is None:
            include = []
        if include_optional is None:
            include_optional = []
        if acls is None:
            acls = {}
        if acl_in_dp is None:
            acl_in_dp = {}

        def add_vlans(n_vlans, host_vlans, vlan_options):
            vlans_config = {}
            for vlan in range(n_vlans):
                n_tagged = 0
                n_untagged = 0
                for vlans in host_vlans.values():
                    if isinstance(vlans, int) and vlan == vlans:
                        n_untagged += 1
                    elif isinstance(vlans, tuple) and vlan in vlans:
                        n_tagged += 1
                vlans_config[self.vlan_name(vlan)] = {
                    'description': '%s tagged, %s untagged' % (n_tagged, n_untagged),
                    'vid': self.vlan_vid(vlan)
                }
            if vlan_options:
                for vlan, options in vlan_options.items():
                    for key, value in options.items():
                        vlans_config[self.vlan_name(vlan)][key] = value
            return vlans_config

        def add_routers(routers):
            router_config = {}
            for i, vlans in routers.items():
                router_config['router-%s' % i] = {
                    'vlans': [self.vlan_name(vlan) for vlan in vlans]
                }
            return router_config

        def add_acl_to_port(i, port, interfaces_config):
            if i in acl_in_dp and port in acl_in_dp[i]:
                interfaces_config[port]['acl_in'] = acl_in_dp[i][port]

        def add_dp(i, dpid, hw_dpid, ofchannel_log, group_table,
                   n_vlans, host_vlans, stack_roots, host_links, dpid_peer_links, port_maps):
            dp_config = {
                'dp_id': int(dpid),
                'hardware': hardware if dpid == hw_dpid else 'Open vSwitch',
                'ofchannel_log': ofchannel_log + str(i) if ofchannel_log else None,
                'interfaces': {},
                'group_table': group_table,
            }

            if dp_options and i in dp_options:
                for key, value in dp_options[i].items():
                    dp_config[key] = value

            if stack_roots and i in stack_roots:
                dp_config['stack'] = {}
                dp_config['stack']['priority'] = stack_roots[i]

            interfaces_config = {}
            # Generate host links
            index = 1
            for host_id, links in host_links.items():
                if i in links:
                    n_links = links.count(i)
                    vlan = host_vlans[host_id]
                    if isinstance(vlan, int):
                        key = 'native_vlan'
                        value = self.vlan_name(vlan)
                    else:
                        key = 'tagged_vlans'
                        value = [self.vlan_name(vlan) for vlan in vlan]
                    for _ in range(n_links):
                        port = port_maps[dpid]['port_%d' % index]
                        interfaces_config[port] = {
                            key: value
                        }
                        if host_options and host_id in host_options:
                            for option_key, option_value in host_options[host_id].items():
                                interfaces_config[port][option_key] = option_value
                        index += 1
                        add_acl_to_port(i, port, interfaces_config)

            # Generate switch-switch links
            for link in dpid_peer_links:
                port, peer_dpid, peer_port = link.port, link.peer_dpid, link.peer_port
                interfaces_config[port] = {}
                if stack_roots:
                    interfaces_config[port].update({
                        'stack': {
                            'dp': self.dp_name(dpids.index(peer_dpid)),
                            'port': peer_port
                        }})
                else:
                    tagged_vlans = [self.vlan_name(vlan) for vlan in range(n_vlans)]
                    interfaces_config[port].update({'tagged_vlans': tagged_vlans})
                    if lacp:
                        interfaces_config[port].update({
                            'lacp': 1,
                            'lacp_active': True
                        })
                        dp_config['lacp_timeout'] = 10
                add_acl_to_port(i, port, interfaces_config)

            dp_config['interfaces'] = interfaces_config
            return dp_config

        config = {'version': 2}
        if include:
            config['include'] = list(include)
        if include_optional:
            config['include_optional'] = list(include_optional)
        config['acls'] = acls.copy()
        config['vlans'] = add_vlans(n_vlans, host_vlans, vlan_options)

        if routers:
            config['routers'] = add_routers(routers)

        dpid_names = {dpids[i]: self.dp_name(i) for i in range(len(dpids))}
        self.set_dpid_names(dpid_names)
        config['dps'] = {}
        for i, dpid in enumerate(dpids):
            config['dps'][self.dp_name(i)] = add_dp(
                i, dpid, hw_dpid, ofchannel_log, self.GROUP_TABLE, n_vlans, host_vlans,
                stack_roots, host_links, self.topo.dpid_peer_links(dpid), self.port_maps)

        return yaml.dump(config, default_flow_style=False)

    def debug(self):
        """Print host information when debugging"""
        try:
            super(FaucetTopoTestBase, self).debug()
        except Exception:
            pprint.pprint(self.host_information)
            raise

    def verify_no_cable_errors(self):
        """Check that prometheus does not detect any stack cabling errors on all DPs"""
        i = 0
        for dpid in self.dpids:
            i += 1
            labels = {'dp_id': '0x%x' % int(dpid), 'dp_name': 'faucet-%u' % i}
            self.assertEqual(
                0, self.scrape_prometheus_var(
                    var='stack_cabling_errors_total', labels=labels, default=None))
            self.assertGreater(
                self.scrape_prometheus_var(
                    var='stack_probes_received_total', labels=labels), 0)
{'fraction_non_alphanumeric': 0.08694790393445873, 'fraction_numerical': 0.00429453932806977, 'mean_word_length': 1.7306512718744362, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '792551', 'n_tokens_mistral': 9313, 'n_tokens_neox': 8867, 'n_words': 2124}