<commit_before># Copyright 2015 <NAME>, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Contains helper classes for conversion between different NF-FG representations.
"""
import json
import string
import sys

try:
  # Import for ESCAPEv2
  from escape.nffg_lib.nffg import AbstractNFFG, NFFG
  from escape.util.misc import VERBOSE
except ImportError:
  import os, inspect

  for p in ("../nffg_lib/",
            "../util/"):
    sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), p)))
  # Import for standalone running
  from nffg import AbstractNFFG, NFFG
  from misc import VERBOSE

try:
  # Import for ESCAPEv2
  import virtualizer as virt_lib
  from virtualizer import __version__ as V_VERSION, Virtualizer
except ImportError:
  import os, inspect

  sys.path.append(os.path.abspath(
    os.path.join(os.path.dirname(__file__), "../../../unify_virtualizer/")))
  # Import for standalone running
  import virtualizer as virt_lib
  from virtualizer import __version__ as V_VERSION


class NFFGConverter(object):
  """
  Convert different representation of NFFG in both ways.
  """
  # port types in Virtualizer
  TYPE_VIRTUALIZER_PORT_ABSTRACT = "port-abstract"
  TYPE_VIRTUALIZER_PORT_SAP = "port-sap"
  # General option names in mapped NFFG assembled by the Mapping algorithm
  OP_TAG = 'TAG'
  OP_UNTAG = 'UNTAG'
  OP_INPORT = 'in_port'
  OP_OUTPUT = 'output'
  OP_FLOWCLASS = "flowclass"
  GENERAL_OPERATIONS = (OP_INPORT, OP_OUTPUT, OP_TAG, OP_UNTAG, OP_FLOWCLASS)
  # Specific tags
  TAG_SG_HOP = "sg_hop"
  # SAP id storing prefix
  SAP_NAME_PREFIX = 'SAP:'
  # Operation formats in Virtualizer
  MATCH_TAG = r"dl_tag"
  ACTION_PUSH_TAG = r"push_tag"
  ACTION_POP_TAG = r"pop_tag"
  # Operand delimiters
  LABEL_DELIMITER = '|'
  OP_DELIMITER = ';'
  KV_DELIMITER = '='
  # Other delimiters
  UNIQUE_ID_DELIMITER = '@'
  # Field types
  TYPE_MATCH = "MATCH"
  TYPE_ACTION = "ACTION"
  # Hard-coded constants
  REQUIREMENT_PREFIX = "REQ"
  # Flowrule generation strategy constants
  FR_ID_GEN_HOP = "HOP_ID"
  FR_ID_GEN_GLOBAL = "GLOBAL_CNTR"
  # Flowrule generation strategy
  FR_ID_GEN_STRATEGY = None

  def __init__ (self, domain=None, logger=None, ensure_unique_id=None):
    # Save domain name for define domain attribute in infras
    self.domain = domain
    # If clarify_id is True, add domain name as a prefix to the node ids
    self.ensure_unique_id = ensure_unique_id
    self.log = logger if logger is not None else logging.getLogger(__name__)
    self.FR_ID_GEN_STRATEGY = self.FR_ID_GEN_HOP

  @classmethod
  def field_splitter (cls, type, field):
    """
    Split the match/action field into a dict-based format for flowrule creation.

    :param type: the name of the field ('MATCH' or 'ACTION')
    :type type: str
    :param field: field data
    :type field: str
    :return: splitted data structure
    :rtype: dict
    """
    # match = {}
    # action = {}
    # # if re.search(r';', flowrule.match):
    # #   # multiple elements in match field
    # #   in_port = re.sub(r'.*in_port=(.*);.*', r'\1', flowrule.match)
    # # else:
    # #   # single element in match field
    # #   in_port = re.sub(r'.*in_port=(.*)', r'\1', flowrule.match)
    # match['in_port'] = port.id
    # # Check match fields - currently only vlan_id
    # # TODO: add further match fields
    # if re.search(r'TAG', flowrule.match):
    #   tag = re.sub(r'.*TAG=.*\|(.*);?', r'\1', flowrule.match)
    #   match['vlan_id'] = tag
    #
    # if re.search(r';', flowrule.action):
    #   # multiple elements in action field
    #   out = re.sub(r'.*output=(.*);.*', r'\1', flowrule.action)
    # else:
    #   # single element in action field
    #   out = re.sub(r'.*output=(.*)', r'\1', flowrule.action)
    # action['out'] = out
    #
    # if re.search(r'TAG', flowrule.action):
    #   if re.search(r'UNTAG', flowrule.action):
    #     action['vlan_pop'] = True
    #   else:
    #     push_tag = re.sub(r'.*TAG=.*\|(.*);?', r'\1', flowrule.action)
    #     action['vlan_push'] = push_tag
    ret = {}
    parts = field.split(cls.OP_DELIMITER)
    if len(parts) < 1:
      raise RuntimeError(
        "Wrong format: %s! Separator (%s) not found!" % (
          field, cls.OP_DELIMITER))
    for part in parts:
      kv = part.split(cls.KV_DELIMITER, 1)
      if len(kv) != 2:
        if kv[0] == cls.OP_UNTAG and type.upper() == cls.TYPE_ACTION:
          ret['vlan_pop'] = True
          continue
        else:
          raise RuntimeError("Not a key-value pair: %s" % part)
      if kv[0] == cls.OP_INPORT:
        try:
          ret['in_port'] = int(kv[1])
        except ValueError:
          # log.warning(
          #    "in_port is not a valid port number: %s! Skip "
          #    "converting..." % kv[1])
          ret['in_port'] = kv[1]
      elif kv[0] == cls.OP_TAG:
        if type.upper() == cls.TYPE_MATCH:
          ret['vlan_id'] = kv[1].split(cls.LABEL_DELIMITER)[-1]
        elif type.upper() == cls.TYPE_ACTION:
          ret['vlan_push'] = kv[1].split(cls.LABEL_DELIMITER)[-1]
        else:
          raise RuntimeError('Not supported field type: %s!' % type)
      elif kv[0] == cls.OP_OUTPUT:
        ret['out'] = kv[1]
      elif kv[0] == cls.OP_FLOWCLASS and type.upper() == cls.TYPE_MATCH:
        ret['flowclass'] = kv[1]
      else:
        raise RuntimeError("Unrecognizable key: %s" % kv[0])
    return ret

  def _convert_flowrule_match (self, match):
    """
    Convert Flowrule match field from NFFG format to a unified format used by
    the Virtualizer.

    Based on Open vSwitch syntax:
    http://openvswitch.org/support/dist-docs/ovs-ofctl.8.txt

    :param match: flowrule match field
    :type match: str
    :return: converted data
    :rtype: str
    """
    # E.g.:  "match": "in_port=1;TAG=SAP1|comp|1" -->
    # E.g.:  "match": "in_port=SAP2|fwd|1;TAG=SAP1|comp|1" -->
    # <match>(in_port=1)dl_tag=1</match>
    ret = []
    match_part = match.split(';')
    if len(match_part) < 2:
      if not match_part[0].startswith("in_port"):
        self.log.warning("Invalid match field: %s" % match)
      return
    for kv in match_part:
      op = kv.split('=', 1)
      if op[0] not in self.GENERAL_OPERATIONS:
        self.log.warning("Unsupported match operand: %s" % op[0])
        continue
      if op[0] == self.OP_TAG:
        try:
          vlan_tag = int(op[1].split('|')[-1])
          ret.append("%s=%s" % (self.MATCH_TAG, format(vlan_tag, '#06x')))
        except ValueError:
          self.log.warning(
            "Wrong VLAN format: %s!" % op[1])
          continue
          # elif op[0] == self.OP_SGHOP:
          #   ret.append(kv)
      elif op[0] == self.OP_FLOWCLASS:
        ret.append(op[1])

    return self.OP_DELIMITER.join(ret)

  def _convert_flowrule_action (self, action):
    """
    Convert Flowrule action field from NFFG format to a unified format used by
    the Virtualizer.

    Based on Open vSwitch syntax:
    http://openvswitch.org/support/dist-docs/ovs-ofctl.8.txt

    :param action: flowrule action field
    :type action: str
    :return: converted data
    :rtype: str
    """
    # E.g.:  "action": "output=2;UNTAG"
    ret = []
    action_part = action.split(';')
    if len(action_part) < 2:
      if not action_part[0].startswith("output"):
        self.log.warning("Invalid action field: %s" % action)
      return
    for kv in action_part:
      op = kv.split('=')
      if op[0] not in self.GENERAL_OPERATIONS:
        self.log.warning("Unsupported action operand: %s" % op[0])
        return
      if op[0] == self.OP_TAG:
        # E.g.: <action>push_tag:0x0037</action>
        try:
          vlan = int(op[1].split('|')[-1])
          ret.append("%s:%s" % (self.ACTION_PUSH_TAG, format(vlan, '#06x')))
        except ValueError:
          self.log.warning(
            "Wrong VLAN format: %s! Skip flowrule conversion..." % op[1])
          continue
      elif op[0] == self.OP_UNTAG:
        # E.g.: <action>strip_vlan</action>
        ret.append(self.ACTION_POP_TAG)
    return self.OP_DELIMITER.join(ret)

  def _parse_virtualizer_node_ports (self, nffg, infra, vnode):
    """
    Parse ports from a Virtualizer node into an :any:`NodeInfra` node.

    :param nffg: Container NFFG
    :type nffg: :any:`NFFG`
    :param infra: infrastructure node
    :type infra: :any:`NodeInfra`
    :param vnode: Virtualizer node
    :type vnode: Infra_node
    :return: None
    """
    # Add ports to Infra Node
    for vport in vnode.ports:
      # If it is a port connected to a SAP: <port-type>port-sap</port-type>
      if vport.port_type.get_value() == self.TYPE_VIRTUALIZER_PORT_SAP:
        # If inter-domain SAP -> id = <sap> tag: <sap>SAP14</sap>
        if vport.sap.is_initialized() and vport.sap.get_value():
          # Use unique SAP tag as the id of the SAP
          sap_id = vport.sap.get_value()  # Optional port.sap
          self.log.debug("Detect SAP id from sap field: %s" % sap_id)
        # Regular SAP
        else:
          # port.id is mandatory now
          # Use port name as the SAP.id if it is set else generate one
          # SAP.id <--> virtualizer.node.port.id
          if vport.id.get_as_text().startswith(self.SAP_NAME_PREFIX):
            sap_id = vport.id.get_value()
            log.debug("Detect SAP id from id field: %s" % sap_id)
          # SAP.id <--> virtualizer.node.port.name
          elif vport.name.is_initialized() and \
             vport.name.get_as_text().startswith(self.SAP_NAME_PREFIX):
            sap_id = vport.name.get_as_text()[len(self.SAP_NAME_PREFIX):]
            self.log.debug("Detect SAP id from name field: %s" % sap_id)
          else:
            # Backup SAP id generation
            # sap_id = "SAP%s" % len([s for s in nffg.saps])
            sap_id = "SAP_%s" % vport.id.get_value()
            self.log.warning(
              "No explicit SAP id was detected! Generated: %s" % sap_id)

        # Create SAP and Add port to SAP
        sap = nffg.add_sap(id=sap_id)
        self.log.debug("Created SAP node: %s" % sap)
<commit_msg>Quickfix for SAP id detection<commit_after># Copyright 2015 <NAME>, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Contains helper classes for conversion between different NF-FG representations.
"""
import json
import string
import sys

try:
  # Import for ESCAPEv2
  from escape.nffg_lib.nffg import AbstractNFFG, NFFG
  from escape.util.misc import VERBOSE
except ImportError:
  import os, inspect

  for p in ("../nffg_lib/",
            "../util/"):
    sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), p)))
  # Import for standalone running
  from nffg import AbstractNFFG, NFFG
  from misc import VERBOSE

try:
  # Import for ESCAPEv2
  import virtualizer as virt_lib
  from virtualizer import __version__ as V_VERSION, Virtualizer
except ImportError:
  import os, inspect

  sys.path.append(os.path.abspath(
    os.path.join(os.path.dirname(__file__), "../../../unify_virtualizer/")))
  # Import for standalone running
  import virtualizer as virt_lib
  from virtualizer import __version__ as V_VERSION


class NFFGConverter(object):
  """
  Convert different representation of NFFG in both ways.
  """
  # port types in Virtualizer
  TYPE_VIRTUALIZER_PORT_ABSTRACT = "port-abstract"
  TYPE_VIRTUALIZER_PORT_SAP = "port-sap"
  # General option names in mapped NFFG assembled by the Mapping algorithm
  OP_TAG = 'TAG'
  OP_UNTAG = 'UNTAG'
  OP_INPORT = 'in_port'
  OP_OUTPUT = 'output'
  OP_FLOWCLASS = "flowclass"
  GENERAL_OPERATIONS = (OP_INPORT, OP_OUTPUT, OP_TAG, OP_UNTAG, OP_FLOWCLASS)
  # Specific tags
  TAG_SG_HOP = "sg_hop"
  # SAP id storing prefix
  SAP_NAME_PREFIX = 'SAP'
  # Operation formats in Virtualizer
  MATCH_TAG = r"dl_tag"
  ACTION_PUSH_TAG = r"push_tag"
  ACTION_POP_TAG = r"pop_tag"
  # Operand delimiters
  LABEL_DELIMITER = '|'
  OP_DELIMITER = ';'
  KV_DELIMITER = '='
  # Other delimiters
  UNIQUE_ID_DELIMITER = '@'
  # Field types
  TYPE_MATCH = "MATCH"
  TYPE_ACTION = "ACTION"
  # Hard-coded constants
  REQUIREMENT_PREFIX = "REQ"
  # Flowrule generation strategy constants
  FR_ID_GEN_HOP = "HOP_ID"
  FR_ID_GEN_GLOBAL = "GLOBAL_CNTR"
  # Flowrule generation strategy
  FR_ID_GEN_STRATEGY = None

  def __init__ (self, domain=None, logger=None, ensure_unique_id=None):
    # Save domain name for define domain attribute in infras
    self.domain = domain
    # If clarify_id is True, add domain name as a prefix to the node ids
    self.ensure_unique_id = ensure_unique_id
    self.log = logger if logger is not None else logging.getLogger(__name__)
    self.FR_ID_GEN_STRATEGY = self.FR_ID_GEN_HOP

  @classmethod
  def field_splitter (cls, type, field):
    """
    Split the match/action field into a dict-based format for flowrule creation.

    :param type: the name of the field ('MATCH' or 'ACTION')
    :type type: str
    :param field: field data
    :type field: str
    :return: splitted data structure
    :rtype: dict
    """
    # match = {}
    # action = {}
    # # if re.search(r';', flowrule.match):
    # #   # multiple elements in match field
    # #   in_port = re.sub(r'.*in_port=(.*);.*', r'\1', flowrule.match)
    # # else:
    # #   # single element in match field
    # #   in_port = re.sub(r'.*in_port=(.*)', r'\1', flowrule.match)
    # match['in_port'] = port.id
    # # Check match fields - currently only vlan_id
    # # TODO: add further match fields
    # if re.search(r'TAG', flowrule.match):
    #   tag = re.sub(r'.*TAG=.*\|(.*);?', r'\1', flowrule.match)
    #   match['vlan_id'] = tag
    #
    # if re.search(r';', flowrule.action):
    #   # multiple elements in action field
    #   out = re.sub(r'.*output=(.*);.*', r'\1', flowrule.action)
    # else:
    #   # single element in action field
    #   out = re.sub(r'.*output=(.*)', r'\1', flowrule.action)
    # action['out'] = out
    #
    # if re.search(r'TAG', flowrule.action):
    #   if re.search(r'UNTAG', flowrule.action):
    #     action['vlan_pop'] = True
    #   else:
    #     push_tag = re.sub(r'.*TAG=.*\|(.*);?', r'\1', flowrule.action)
    #     action['vlan_push'] = push_tag
    ret = {}
    parts = field.split(cls.OP_DELIMITER)
    if len(parts) < 1:
      raise RuntimeError(
        "Wrong format: %s! Separator (%s) not found!" % (
          field, cls.OP_DELIMITER))
    for part in parts:
      kv = part.split(cls.KV_DELIMITER, 1)
      if len(kv) != 2:
        if kv[0] == cls.OP_UNTAG and type.upper() == cls.TYPE_ACTION:
          ret['vlan_pop'] = True
          continue
        else:
          raise RuntimeError("Not a key-value pair: %s" % part)
      if kv[0] == cls.OP_INPORT:
        try:
          ret['in_port'] = int(kv[1])
        except ValueError:
          # log.warning(
          #    "in_port is not a valid port number: %s! Skip "
          #    "converting..." % kv[1])
          ret['in_port'] = kv[1]
      elif kv[0] == cls.OP_TAG:
        if type.upper() == cls.TYPE_MATCH:
          ret['vlan_id'] = kv[1].split(cls.LABEL_DELIMITER)[-1]
        elif type.upper() == cls.TYPE_ACTION:
          ret['vlan_push'] = kv[1].split(cls.LABEL_DELIMITER)[-1]
        else:
          raise RuntimeError('Not supported field type: %s!' % type)
      elif kv[0] == cls.OP_OUTPUT:
        ret['out'] = kv[1]
      elif kv[0] == cls.OP_FLOWCLASS and type.upper() == cls.TYPE_MATCH:
        ret['flowclass'] = kv[1]
      else:
        raise RuntimeError("Unrecognizable key: %s" % kv[0])
    return ret

  def _convert_flowrule_match (self, match):
    """
    Convert Flowrule match field from NFFG format to a unified format used by
    the Virtualizer.

    Based on Open vSwitch syntax:
    http://openvswitch.org/support/dist-docs/ovs-ofctl.8.txt

    :param match: flowrule match field
    :type match: str
    :return: converted data
    :rtype: str
    """
    # E.g.:  "match": "in_port=1;TAG=SAP1|comp|1" -->
    # E.g.:  "match": "in_port=SAP2|fwd|1;TAG=SAP1|comp|1" -->
    # <match>(in_port=1)dl_tag=1</match>
    ret = []
    match_part = match.split(';')
    if len(match_part) < 2:
      if not match_part[0].startswith("in_port"):
        self.log.warning("Invalid match field: %s" % match)
      return
    for kv in match_part:
      op = kv.split('=', 1)
      if op[0] not in self.GENERAL_OPERATIONS:
        self.log.warning("Unsupported match operand: %s" % op[0])
        continue
      if op[0] == self.OP_TAG:
        try:
          vlan_tag = int(op[1].split('|')[-1])
          ret.append("%s=%s" % (self.MATCH_TAG, format(vlan_tag, '#06x')))
        except ValueError:
          self.log.warning(
            "Wrong VLAN format: %s!" % op[1])
          continue
          # elif op[0] == self.OP_SGHOP:
          #   ret.append(kv)
      elif op[0] == self.OP_FLOWCLASS:
        ret.append(op[1])

    return self.OP_DELIMITER.join(ret)

  def _convert_flowrule_action (self, action):
    """
    Convert Flowrule action field from NFFG format to a unified format used by
    the Virtualizer.

    Based on Open vSwitch syntax:
    http://openvswitch.org/support/dist-docs/ovs-ofctl.8.txt

    :param action: flowrule action field
    :type action: str
    :return: converted data
    :rtype: str
    """
    # E.g.:  "action": "output=2;UNTAG"
    ret = []
    action_part = action.split(';')
    if len(action_part) < 2:
      if not action_part[0].startswith("output"):
        self.log.warning("Invalid action field: %s" % action)
      return
    for kv in action_part:
      op = kv.split('=')
      if op[0] not in self.GENERAL_OPERATIONS:
        self.log.warning("Unsupported action operand: %s" % op[0])
        return
      if op[0] == self.OP_TAG:
        # E.g.: <action>push_tag:0x0037</action>
        try:
          vlan = int(op[1].split('|')[-1])
          ret.append("%s:%s" % (self.ACTION_PUSH_TAG, format(vlan, '#06x')))
        except ValueError:
          self.log.warning(
            "Wrong VLAN format: %s! Skip flowrule conversion..." % op[1])
          continue
      elif op[0] == self.OP_UNTAG:
        # E.g.: <action>strip_vlan</action>
        ret.append(self.ACTION_POP_TAG)
    return self.OP_DELIMITER.join(ret)

  def _parse_virtualizer_node_ports (self, nffg, infra, vnode):
    """
    Parse ports from a Virtualizer node into an :any:`NodeInfra` node.

    :param nffg: Container NFFG
    :type nffg: :any:`NFFG`
    :param infra: infrastructure node
    :type infra: :any:`NodeInfra`
    :param vnode: Virtualizer node
    :type vnode: Infra_node
    :return: None
    """
    # Add ports to Infra Node
    for vport in vnode.ports:
      # If it is a port connected to a SAP: <port-type>port-sap</port-type>
      if vport.port_type.get_value() == self.TYPE_VIRTUALIZER_PORT_SAP:
        # If inter-domain SAP -> id = <sap> tag: <sap>SAP14</sap>
        if vport.sap.is_initialized() and vport.sap.get_value():
          # Use unique SAP tag as the id of the SAP
          sap_id = vport.sap.get_value()  # Optional port.sap
          self.log.debug("Detect SAP id from sap field: %s" % sap_id)
        # Regular SAP
        else:
          # port.id is mandatory now
          # Use port name as the SAP.id if it is set else generate one
          # SAP.id <--> virtualizer.node.port.id
          if vport.id.get_as_text().startswith(self.SAP_NAME_PREFIX):
            sap_id = vport.id.get_value()
            log.debug("Detect SAP id from id field: %s" % sap_id)
          # SAP.id <--> virtualizer.node.port.name
          elif vport.name.is_initialized() and \
             vport.name.get_as_text().startswith(self.SAP_NAME_PREFIX):
            sap_id = vport.name.get_as_text().upper()[
                     len(self.SAP_NAME_PREFIX) + 1:]
            self.log.debug("Detect SAP id from name field: %s" % sap_id)
          else:
            # Backup SAP id generation
            # sap_id = "SAP%s" % len([s for s in nffg.saps])
            sap_id = "SAP_%s" % vport.id.get_value()
            self.log.warning(
              "No explicit SAP id was detected! Generated: %s" % sap_id)

        # Create SAP and Add port to SAP
        sap = nffg.add_sap(id=sap_id)
        self.log.debug("Created SAP node: %s" % sap)

{'fraction_non_alphanumeric': 0.12985109903096195, 'fraction_numerical': 0.007421413377452139, 'mean_word_length': 2.6300617707618392, 'pattern_counts': {'":': 6, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3929766', 'n_tokens_mistral': 7404, 'n_tokens_neox': 7299, 'n_words': 1894}