<commit_before>#! /usr/bin/python
# hhaim 
import sys
import os
python_ver = 'python%s' % sys.version_info[0]
yaml_path = os.path.join('external_libs', 'pyyaml-3.11', python_ver)
if yaml_path not in sys.path:
    sys.path.append(yaml_path)
import yaml
import dpdk_nic_bind
import re
import argparse
import copy
import shlex
import traceback
from collections import defaultdict, OrderedDict
from distutils.util import strtobool
import subprocess
import platform

# exit code is Important should be
# -1 : don't continue 
# 0  : no errors - no need to load mlx share object
# 32  : no errors - mlx share object should be loaded 
MLX_EXIT_CODE = 32

class VFIOBindErr(Exception): pass

PATH_ARR = os.getenv('PATH', '').split(':')
for path in ['/usr/local/sbin', '/usr/sbin', '/sbin']:
    if path not in PATH_ARR:
        PATH_ARR.append(path)
os.environ['PATH'] = ':'.join(PATH_ARR)

def if_list_remove_sub_if(if_list):
    return map(lambda x:x.split('/')[0],if_list)

class ConfigCreator(object):
    mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']
    _2hex_re = '[\da-fA-F]{2}'
    mac_re = re.compile('^({0}:){{5}}{0}$'.format(_2hex_re))
    MAC_LCORE_NUM = 63 # current bitmask is 64 bit

    # cpu_topology - dict: physical processor -> physical core -> logical processing unit (thread)
    # interfaces - array of dicts per interface, should include "mandatory_interface_fields" values
    def __init__(self, cpu_topology, interfaces, include_lcores = [], exclude_lcores = [], only_first_thread = False, zmq_rpc_port = None, zmq_pub_port = None, prefix = None, ignore_numa = False):
        self.cpu_topology = copy.deepcopy(cpu_topology)
        self.interfaces   = copy.deepcopy(interfaces)
        del cpu_topology
        del interfaces
        assert isinstance(self.cpu_topology, dict), 'Type of cpu_topology should be dict, got: %s' % type(self.cpu_topology)
        assert len(self.cpu_topology.keys()) > 0, 'cpu_topology should contain at least one processor'
        assert isinstance(self.interfaces, list), 'Type of interfaces should be list, got: %s' % type(list)
        assert len(self.interfaces) % 2 == 0, 'Should be even number of interfaces, got: %s' % len(self.interfaces)
        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
        assert isinstance(include_lcores, list), 'include_lcores should be list, got: %s' % type(include_lcores)
        assert isinstance(exclude_lcores, list), 'exclude_lcores should be list, got: %s' % type(exclude_lcores)
        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
        if only_first_thread:
            for cores in self.cpu_topology.values():
                for core in cores.keys():
                    cores[core] = cores[core][:1]
        include_lcores = [int(x) for x in include_lcores]
        exclude_lcores = [int(x) for x in exclude_lcores]

        self.has_zero_lcore = False
        self.lcores_per_numa = {}
        total_lcores = 0
        for numa, cores in self.cpu_topology.items():
            self.lcores_per_numa[numa] = {'main': [], 'siblings': [], 'all': []}
            for core, lcores in cores.items():
                total_lcores += len(lcores)
                for lcore in list(lcores):
                    if include_lcores and lcore not in include_lcores:
                        cores[core].remove(lcore)
                    if exclude_lcores and lcore in exclude_lcores:
                        cores[core].remove(lcore)
                    if lcore > self.MAC_LCORE_NUM:
                        cores[core].remove(lcore)
                if 0 in lcores:
                    self.has_zero_lcore = True
                    lcores.remove(0)
                    self.lcores_per_numa[numa]['siblings'].extend(lcores)
                else:
                    self.lcores_per_numa[numa]['main'].extend(lcores[:1])
                    self.lcores_per_numa[numa]['siblings'].extend(lcores[1:])
                self.lcores_per_numa[numa]['all'].extend(lcores)

        for interface in self.interfaces:
            for mandatory_interface_field in ConfigCreator.mandatory_interface_fields:
                if mandatory_interface_field not in interface:
                    raise DpdkSetup("Expected '%s' field in interface dictionary, got: %s" % (mandatory_interface_field, interface))

        Device_str = self._verify_devices_same_type(self.interfaces)
        if '40Gb' in Device_str:
            self.speed = 40
        else:
            self.speed = 10

        minimum_required_lcores = len(self.interfaces) // 2 + 2
        if total_lcores < minimum_required_lcores:
            raise DpdkSetup('Your system should have at least %s cores for %s interfaces, and it has: %s.' %
                    (minimum_required_lcores, len(self.interfaces), total_lcores))
        interfaces_per_numa = defaultdict(int)

        for i in range(0, len(self.interfaces), 2):
            numa = self.interfaces[i]['NUMA']
            if numa != self.interfaces[i+1]['NUMA'] and not ignore_numa:
                raise DpdkSetup('NUMA of each pair of interfaces should be the same. Got NUMA %s for client interface %s, NUMA %s for server interface %s' %
                        (numa, self.interfaces[i]['Slot_str'], self.interfaces[i+1]['NUMA'], self.interfaces[i+1]['Slot_str']))
            interfaces_per_numa[numa] += 2

        self.interfaces_per_numa = interfaces_per_numa
        self.prefix              = prefix
        self.zmq_pub_port        = zmq_pub_port
        self.zmq_rpc_port        = zmq_rpc_port
        self.ignore_numa         = ignore_numa

    @staticmethod
    def verify_mac(mac_string):
        if not ConfigCreator.mac_re.match(mac_string):
            raise DpdkSetup('MAC address should be in format of 12:34:56:78:9a:bc, got: %s' % mac_string)
        return mac_string.lower()

    @staticmethod
    def _exit_if_bad_ip(ip):
        if not ConfigCreator._verify_ip(ip):
            raise DpdkSetup("Got bad IP %s" % ip)

    @staticmethod
    def _verify_ip(ip):
        a = ip.split('.')
        if len(a) != 4:
            return False
        for x in a:
            if not x.isdigit():
                return False
            i = int(x)
            if i < 0 or i > 255:
                return False
        return True

    @staticmethod
    def _verify_devices_same_type(interfaces_list):
        Device_str = interfaces_list[0]['Device_str']
        for interface in interfaces_list:
            if Device_str != interface['Device_str']:
                raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (Device_str, interface['Device_str']))
        return Device_str

    def create_config(self, filename = None, print_config = False):
        config_str = '### Config file generated by dpdk_setup_ports.py ###\n\n'
        config_str += '- port_limit: %s\n' % len(self.interfaces)
        config_str += '  version: 2\n'
        config_str += "  interfaces: ['%s']\n" % "', '".join([interface['Slot_str'] for interface in self.interfaces])
        if self.speed > 10:
            config_str += '  port_bandwidth_gb: %s\n' % self.speed
        if self.prefix:
            config_str += '  prefix: %s\n' % self.prefix
        if self.zmq_pub_port:
            config_str += '  zmq_pub_port: %s\n' % self.zmq_pub_port
        if self.zmq_rpc_port:
            config_str += '  zmq_rpc_port: %s\n' % self.zmq_rpc_port
        config_str += '  port_info:\n'
        for index, interface in enumerate(self.interfaces):
            if 'ip' in interface:
                self._exit_if_bad_ip(interface['ip'])
                self._exit_if_bad_ip(interface['def_gw'])
                config_str += ' '*6 + '- ip: %s\n' % interface['ip']
                config_str += ' '*8 + 'default_gw: %s\n' % interface['def_gw']
            else:
                config_str += ' '*6 + '- dest_mac: %s' % self.verify_mac(interface['dest_mac'])
                if interface.get('loopback_dest'):
                    config_str += " # MAC OF LOOPBACK TO IT'S DUAL INTERFACE\n"
                else:
                    config_str += '\n'
                config_str += ' '*8 + 'src_mac:  %s\n' % self.verify_mac(interface['src_mac'])
            if index % 2:
                config_str += '\n' # dual if barrier

        if not self.ignore_numa:
            config_str += '  platform:\n'
            if len(self.interfaces_per_numa.keys()) == 1 and -1 in self.interfaces_per_numa: # VM, use any cores
                lcores_pool = sorted([lcore for lcores in self.lcores_per_numa.values() for lcore in lcores['all']])
                config_str += ' '*6 + 'master_thread_id: %s\n' % (0 if self.has_zero_lcore else lcores_pool.pop(0))
                config_str += ' '*6 + 'latency_thread_id: %s\n' % lcores_pool.pop(0)
                lcores_per_dual_if = int(len(lcores_pool) * 2 / len(self.interfaces))
                config_str += ' '*6 + 'dual_if:\n'
                for i in range(0, len(self.interfaces), 2):
                    lcores_for_this_dual_if = list(map(str, sorted(lcores_pool[:lcores_per_dual_if])))
                    lcores_pool = lcores_pool[lcores_per_dual_if:]
                    if not lcores_for_this_dual_if:
                        raise DpdkSetup('lcores_for_this_dual_if is empty (internal bug, please report with details of setup)')
                    config_str += ' '*8 + '- socket: 0\n'
                    config_str += ' '*10 + 'threads: [%s]\n\n' % ','.join(lcores_for_this_dual_if)
            else:
                # we will take common minimum among all NUMAs, to satisfy all
                lcores_per_dual_if = 99
                extra_lcores = 1 if self.has_zero_lcore else 2
                # worst case 3 iterations, to ensure master and "rx" have cores left
                while (lcores_per_dual_if * sum(self.interfaces_per_numa.values()) / 2) + extra_lcores > sum([len(lcores['all']) for lcores in self.lcores_per_numa.values()]):
                    lcores_per_dual_if -= 1
                    for numa, lcores_dict in self.lcores_per_numa.items():
                        if not self.interfaces_per_numa[numa]:
                            continue
                        lcores_per_dual_if = min(lcores_per_dual_if, int(2 * len(lcores_dict['all']) / self.interfaces_per_numa[numa]))
                lcores_pool = copy.deepcopy(self.lcores_per_numa)
                # first, allocate lcores for dual_if section
                dual_if_section = ' '*6 + 'dual_if:\n'
                for i in range(0, len(self.interfaces), 2):
                    numa = self.interfaces[i]['NUMA']
                    dual_if_section += ' '*8 + '- socket: %s\n' % numa
                    lcores_for_this_dual_if  = lcores_pool[numa]['all'][:lcores_per_dual_if]
                    lcores_pool[numa]['all'] = lcores_pool[numa]['all'][lcores_per_dual_if:]
                    for lcore in lcores_for_this_dual_if:
                        if lcore in lcores_pool[numa]['main']:
                            lcores_pool[numa]['main'].remove(lcore)
                        elif lcore in lcores_pool[numa]['siblings']:
                            lcores_pool[numa]['siblings'].remove(lcore)
                        else:
                            raise DpdkSetup('lcore not in main nor in siblings list (internal bug, please report with details of setup)')
                    if not lcores_for_this_dual_if:
                        raise DpdkSetup('Not enough cores at NUMA %s. This NUMA has %s processing units and %s interfaces.' % (numa, len(self.lcores_per_numa[numa]), self.interfaces_per_numa[numa]))
                    dual_if_section += ' '*10 + 'threads: [%s]\n\n' % ','.join(list(map(str, sorted(lcores_for_this_dual_if))))

                # take the cores left to master and rx
                mains_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['main']]
                siblings_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['siblings']]
                if mains_left:
                    rx_core = mains_left.pop(0)
                else:
                    rx_core = siblings_left.pop(0)
                if self.has_zero_lcore:
                    master_core = 0
                elif mains_left:
                    master_core = mains_left.pop(0)
                else:
                    master_core = siblings_left.pop(0)
                config_str += ' '*6 + 'master_thread_id: %s\n' % master_core
                config_str += ' '*6 + 'latency_thread_id: %s\n' % rx_core
                # add the dual_if section
                config_str += dual_if_section

        # verify config is correct YAML format
        try:
            yaml.safe_load(config_str)
        except Exception as e:
            raise DpdkSetup('Could not create correct yaml config.\nGenerated YAML:\n%s\nEncountered error:\n%s' % (config_str, e))

        if print_config:
            print(config_str)
        if filename:
            if os.path.exists(filename):
                if not dpdk_nic_bind.confirm('File %s already exist, overwrite? (y/N)' % filename):
                    print('Skipping.')
                    return config_str
            with open(filename, 'w') as f:
                f.write(config_str)
            print('Saved to %s.' % filename)
        return config_str

# only load igb_uio if it's available
def load_igb_uio():
    loaded_mods = dpdk_nic_bind.get_loaded_modules()
    if 'igb_uio' in loaded_mods:
        return True
    if 'uio' not in loaded_mods:
        ret = os.system('modprobe uio')
        if ret:
            return False
    km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
    if os.path.exists(km):
        return os.system('insmod %s' % km) == 0

# try to compile igb_uio if it's missing
def compile_and_load_igb_uio():
    loaded_mods = dpdk_nic_bind.get_loaded_modules()
    if 'igb_uio' in loaded_mods:
        return
    if 'uio' not in loaded_mods:
        ret = os.system('modprobe uio')
        if ret:
            print('Failed inserting uio module, please check if it is installed')
            sys.exit(-1)
    km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
    if not os.path.exists(km):
        print("ERROR: We don't have precompiled igb_uio.ko module for your kernel version")
        print('Will try compiling automatically - make sure you have file-system read/write permission')
        try:
            subprocess.check_output('make', cwd = './ko/src', stderr = subprocess.STDOUT)
            subprocess.check_output(['make', 'install'], cwd = './ko/src', stderr = subprocess.STDOUT)
            print('\nSuccess.')
        except Exception as e:
            print('\nAutomatic compilation failed: (%s)' % e)
            print('You can try compiling yourself, using the following commands:')
            print('  $cd ko/src')
            print('  $make')
            print('  $make install')
            print('  $cd -')
            print('Then, try to run TRex again.')
            print('Note: you might need additional Linux packages for that:')
            print('  * yum based (Fedora, CentOS, RedHat):')
            print('        sudo yum install kernel-devel-`uname -r`')
            print('        sudo yum group install "Development tools"')
            print('  * apt based (Ubuntu):')
            print('        sudo apt install linux-headers-`uname -r`')
            print('        sudo apt install build-essential')<commit_msg>yet, another minor error message change<commit_after>#! /usr/bin/python
# hhaim 
import sys
import os
python_ver = 'python%s' % sys.version_info[0]
yaml_path = os.path.join('external_libs', 'pyyaml-3.11', python_ver)
if yaml_path not in sys.path:
    sys.path.append(yaml_path)
import yaml
import dpdk_nic_bind
import re
import argparse
import copy
import shlex
import traceback
from collections import defaultdict, OrderedDict
from distutils.util import strtobool
import subprocess
import platform

# exit code is Important should be
# -1 : don't continue 
# 0  : no errors - no need to load mlx share object
# 32  : no errors - mlx share object should be loaded 
MLX_EXIT_CODE = 32

class VFIOBindErr(Exception): pass

PATH_ARR = os.getenv('PATH', '').split(':')
for path in ['/usr/local/sbin', '/usr/sbin', '/sbin']:
    if path not in PATH_ARR:
        PATH_ARR.append(path)
os.environ['PATH'] = ':'.join(PATH_ARR)

def if_list_remove_sub_if(if_list):
    return map(lambda x:x.split('/')[0],if_list)

class ConfigCreator(object):
    mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']
    _2hex_re = '[\da-fA-F]{2}'
    mac_re = re.compile('^({0}:){{5}}{0}$'.format(_2hex_re))
    MAC_LCORE_NUM = 63 # current bitmask is 64 bit

    # cpu_topology - dict: physical processor -> physical core -> logical processing unit (thread)
    # interfaces - array of dicts per interface, should include "mandatory_interface_fields" values
    def __init__(self, cpu_topology, interfaces, include_lcores = [], exclude_lcores = [], only_first_thread = False, zmq_rpc_port = None, zmq_pub_port = None, prefix = None, ignore_numa = False):
        self.cpu_topology = copy.deepcopy(cpu_topology)
        self.interfaces   = copy.deepcopy(interfaces)
        del cpu_topology
        del interfaces
        assert isinstance(self.cpu_topology, dict), 'Type of cpu_topology should be dict, got: %s' % type(self.cpu_topology)
        assert len(self.cpu_topology.keys()) > 0, 'cpu_topology should contain at least one processor'
        assert isinstance(self.interfaces, list), 'Type of interfaces should be list, got: %s' % type(list)
        assert len(self.interfaces) % 2 == 0, 'Should be even number of interfaces, got: %s' % len(self.interfaces)
        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
        assert isinstance(include_lcores, list), 'include_lcores should be list, got: %s' % type(include_lcores)
        assert isinstance(exclude_lcores, list), 'exclude_lcores should be list, got: %s' % type(exclude_lcores)
        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
        if only_first_thread:
            for cores in self.cpu_topology.values():
                for core in cores.keys():
                    cores[core] = cores[core][:1]
        include_lcores = [int(x) for x in include_lcores]
        exclude_lcores = [int(x) for x in exclude_lcores]

        self.has_zero_lcore = False
        self.lcores_per_numa = {}
        total_lcores = 0
        for numa, cores in self.cpu_topology.items():
            self.lcores_per_numa[numa] = {'main': [], 'siblings': [], 'all': []}
            for core, lcores in cores.items():
                total_lcores += len(lcores)
                for lcore in list(lcores):
                    if include_lcores and lcore not in include_lcores:
                        cores[core].remove(lcore)
                    if exclude_lcores and lcore in exclude_lcores:
                        cores[core].remove(lcore)
                    if lcore > self.MAC_LCORE_NUM:
                        cores[core].remove(lcore)
                if 0 in lcores:
                    self.has_zero_lcore = True
                    lcores.remove(0)
                    self.lcores_per_numa[numa]['siblings'].extend(lcores)
                else:
                    self.lcores_per_numa[numa]['main'].extend(lcores[:1])
                    self.lcores_per_numa[numa]['siblings'].extend(lcores[1:])
                self.lcores_per_numa[numa]['all'].extend(lcores)

        for interface in self.interfaces:
            for mandatory_interface_field in ConfigCreator.mandatory_interface_fields:
                if mandatory_interface_field not in interface:
                    raise DpdkSetup("Expected '%s' field in interface dictionary, got: %s" % (mandatory_interface_field, interface))

        Device_str = self._verify_devices_same_type(self.interfaces)
        if '40Gb' in Device_str:
            self.speed = 40
        else:
            self.speed = 10

        minimum_required_lcores = len(self.interfaces) // 2 + 2
        if total_lcores < minimum_required_lcores:
            raise DpdkSetup('Your system should have at least %s cores for %s interfaces, and it has: %s.' %
                    (minimum_required_lcores, len(self.interfaces), total_lcores))
        interfaces_per_numa = defaultdict(int)

        for i in range(0, len(self.interfaces), 2):
            numa = self.interfaces[i]['NUMA']
            if numa != self.interfaces[i+1]['NUMA'] and not ignore_numa:
                raise DpdkSetup('NUMA of each pair of interfaces should be the same. Got NUMA %s for client interface %s, NUMA %s for server interface %s' %
                        (numa, self.interfaces[i]['Slot_str'], self.interfaces[i+1]['NUMA'], self.interfaces[i+1]['Slot_str']))
            interfaces_per_numa[numa] += 2

        self.interfaces_per_numa = interfaces_per_numa
        self.prefix              = prefix
        self.zmq_pub_port        = zmq_pub_port
        self.zmq_rpc_port        = zmq_rpc_port
        self.ignore_numa         = ignore_numa

    @staticmethod
    def verify_mac(mac_string):
        if not ConfigCreator.mac_re.match(mac_string):
            raise DpdkSetup('MAC address should be in format of 12:34:56:78:9a:bc, got: %s' % mac_string)
        return mac_string.lower()

    @staticmethod
    def _exit_if_bad_ip(ip):
        if not ConfigCreator._verify_ip(ip):
            raise DpdkSetup("Got bad IP %s" % ip)

    @staticmethod
    def _verify_ip(ip):
        a = ip.split('.')
        if len(a) != 4:
            return False
        for x in a:
            if not x.isdigit():
                return False
            i = int(x)
            if i < 0 or i > 255:
                return False
        return True

    @staticmethod
    def _verify_devices_same_type(interfaces_list):
        Device_str = interfaces_list[0]['Device_str']
        for interface in interfaces_list:
            if Device_str != interface['Device_str']:
                raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (Device_str, interface['Device_str']))
        return Device_str

    def create_config(self, filename = None, print_config = False):
        config_str = '### Config file generated by dpdk_setup_ports.py ###\n\n'
        config_str += '- port_limit: %s\n' % len(self.interfaces)
        config_str += '  version: 2\n'
        config_str += "  interfaces: ['%s']\n" % "', '".join([interface['Slot_str'] for interface in self.interfaces])
        if self.speed > 10:
            config_str += '  port_bandwidth_gb: %s\n' % self.speed
        if self.prefix:
            config_str += '  prefix: %s\n' % self.prefix
        if self.zmq_pub_port:
            config_str += '  zmq_pub_port: %s\n' % self.zmq_pub_port
        if self.zmq_rpc_port:
            config_str += '  zmq_rpc_port: %s\n' % self.zmq_rpc_port
        config_str += '  port_info:\n'
        for index, interface in enumerate(self.interfaces):
            if 'ip' in interface:
                self._exit_if_bad_ip(interface['ip'])
                self._exit_if_bad_ip(interface['def_gw'])
                config_str += ' '*6 + '- ip: %s\n' % interface['ip']
                config_str += ' '*8 + 'default_gw: %s\n' % interface['def_gw']
            else:
                config_str += ' '*6 + '- dest_mac: %s' % self.verify_mac(interface['dest_mac'])
                if interface.get('loopback_dest'):
                    config_str += " # MAC OF LOOPBACK TO IT'S DUAL INTERFACE\n"
                else:
                    config_str += '\n'
                config_str += ' '*8 + 'src_mac:  %s\n' % self.verify_mac(interface['src_mac'])
            if index % 2:
                config_str += '\n' # dual if barrier

        if not self.ignore_numa:
            config_str += '  platform:\n'
            if len(self.interfaces_per_numa.keys()) == 1 and -1 in self.interfaces_per_numa: # VM, use any cores
                lcores_pool = sorted([lcore for lcores in self.lcores_per_numa.values() for lcore in lcores['all']])
                config_str += ' '*6 + 'master_thread_id: %s\n' % (0 if self.has_zero_lcore else lcores_pool.pop(0))
                config_str += ' '*6 + 'latency_thread_id: %s\n' % lcores_pool.pop(0)
                lcores_per_dual_if = int(len(lcores_pool) * 2 / len(self.interfaces))
                config_str += ' '*6 + 'dual_if:\n'
                for i in range(0, len(self.interfaces), 2):
                    lcores_for_this_dual_if = list(map(str, sorted(lcores_pool[:lcores_per_dual_if])))
                    lcores_pool = lcores_pool[lcores_per_dual_if:]
                    if not lcores_for_this_dual_if:
                        raise DpdkSetup('lcores_for_this_dual_if is empty (internal bug, please report with details of setup)')
                    config_str += ' '*8 + '- socket: 0\n'
                    config_str += ' '*10 + 'threads: [%s]\n\n' % ','.join(lcores_for_this_dual_if)
            else:
                # we will take common minimum among all NUMAs, to satisfy all
                lcores_per_dual_if = 99
                extra_lcores = 1 if self.has_zero_lcore else 2
                # worst case 3 iterations, to ensure master and "rx" have cores left
                while (lcores_per_dual_if * sum(self.interfaces_per_numa.values()) / 2) + extra_lcores > sum([len(lcores['all']) for lcores in self.lcores_per_numa.values()]):
                    lcores_per_dual_if -= 1
                    for numa, lcores_dict in self.lcores_per_numa.items():
                        if not self.interfaces_per_numa[numa]:
                            continue
                        lcores_per_dual_if = min(lcores_per_dual_if, int(2 * len(lcores_dict['all']) / self.interfaces_per_numa[numa]))
                lcores_pool = copy.deepcopy(self.lcores_per_numa)
                # first, allocate lcores for dual_if section
                dual_if_section = ' '*6 + 'dual_if:\n'
                for i in range(0, len(self.interfaces), 2):
                    numa = self.interfaces[i]['NUMA']
                    dual_if_section += ' '*8 + '- socket: %s\n' % numa
                    lcores_for_this_dual_if  = lcores_pool[numa]['all'][:lcores_per_dual_if]
                    lcores_pool[numa]['all'] = lcores_pool[numa]['all'][lcores_per_dual_if:]
                    for lcore in lcores_for_this_dual_if:
                        if lcore in lcores_pool[numa]['main']:
                            lcores_pool[numa]['main'].remove(lcore)
                        elif lcore in lcores_pool[numa]['siblings']:
                            lcores_pool[numa]['siblings'].remove(lcore)
                        else:
                            raise DpdkSetup('lcore not in main nor in siblings list (internal bug, please report with details of setup)')
                    if not lcores_for_this_dual_if:
                        raise DpdkSetup('Not enough cores at NUMA %s. This NUMA has %s processing units and %s interfaces.' % (numa, len(self.lcores_per_numa[numa]), self.interfaces_per_numa[numa]))
                    dual_if_section += ' '*10 + 'threads: [%s]\n\n' % ','.join(list(map(str, sorted(lcores_for_this_dual_if))))

                # take the cores left to master and rx
                mains_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['main']]
                siblings_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['siblings']]
                if mains_left:
                    rx_core = mains_left.pop(0)
                else:
                    rx_core = siblings_left.pop(0)
                if self.has_zero_lcore:
                    master_core = 0
                elif mains_left:
                    master_core = mains_left.pop(0)
                else:
                    master_core = siblings_left.pop(0)
                config_str += ' '*6 + 'master_thread_id: %s\n' % master_core
                config_str += ' '*6 + 'latency_thread_id: %s\n' % rx_core
                # add the dual_if section
                config_str += dual_if_section

        # verify config is correct YAML format
        try:
            yaml.safe_load(config_str)
        except Exception as e:
            raise DpdkSetup('Could not create correct yaml config.\nGenerated YAML:\n%s\nEncountered error:\n%s' % (config_str, e))

        if print_config:
            print(config_str)
        if filename:
            if os.path.exists(filename):
                if not dpdk_nic_bind.confirm('File %s already exist, overwrite? (y/N)' % filename):
                    print('Skipping.')
                    return config_str
            with open(filename, 'w') as f:
                f.write(config_str)
            print('Saved to %s.' % filename)
        return config_str

# only load igb_uio if it's available
def load_igb_uio():
    loaded_mods = dpdk_nic_bind.get_loaded_modules()
    if 'igb_uio' in loaded_mods:
        return True
    if 'uio' not in loaded_mods:
        ret = os.system('modprobe uio')
        if ret:
            return False
    km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
    if os.path.exists(km):
        return os.system('insmod %s' % km) == 0

# try to compile igb_uio if it's missing
def compile_and_load_igb_uio():
    loaded_mods = dpdk_nic_bind.get_loaded_modules()
    if 'igb_uio' in loaded_mods:
        return
    if 'uio' not in loaded_mods:
        ret = os.system('modprobe uio')
        if ret:
            print('Failed inserting uio module, please check if it is installed')
            sys.exit(-1)
    km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
    if not os.path.exists(km):
        print("ERROR: We don't have precompiled igb_uio.ko module for your kernel version")
        print('Will try compiling automatically - make sure you have file-system read/write permission')
        try:
            subprocess.check_output('make', cwd = './ko/src', stderr = subprocess.STDOUT)
            subprocess.check_output(['make', 'install'], cwd = './ko/src', stderr = subprocess.STDOUT)
            print('\nSuccess.')
        except Exception as e:
            print('\n ERROR:  Automatic compilation failed: (%s)' % e)
            print('Make sure you have file-system read/write permission')
            print('You can try compiling yourself, using the following commands:')
            print('  $cd ko/src')
            print('  $make')
            print('  $make install')
            print('  $cd -')
            print('Then, try to run TRex again.')
            print('Note: you might need additional Linux packages for that:')
            print('  * yum based (Fedora, CentOS, RedHat):')
            print('        sudo yum install kernel-devel-`uname -r`')
            print('        sudo yum group install "Development tools"')
            print('  * apt based (Ubuntu):')
            print('        sudo apt install linux-headers-`uname -r`')
            print('        sudo apt install build-essential')
{'fraction_non_alphanumeric': 0.11017519136533965, 'fraction_numerical': 0.006853921788425199, 'mean_word_length': 2.1324237560192616, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1524869', 'n_tokens_mistral': 10257, 'n_tokens_neox': 9517, 'n_words': 2383}