<commit_before>'''
Created on Apr 26, 2010

@author: jnaous
'''

import re
from django.db import models
from expedient.clearinghouse.resources import models as resource_models
from expedient.clearinghouse.slice import models as slice_models
from expedient.clearinghouse.aggregate import models as aggregate_models
from expedient.common.xmlrpc_serverproxy.models import PasswordXMLRPCServerProxy
from django.core.urlresolvers import reverse
from django.utils.datetime_safe import datetime
from autoslug.fields import AutoSlugField
from django.db.models import signals
from django.contrib.sites.models import Site
import logging
from expedient.common.utils import create_or_update, modelfields
from expedient.clearinghouse.slice.models import Slice
from django.db.models.aggregates import Count
from django.core.exceptions import ValidationError

logger = logging.getLogger("OpenflowModels")
parse_logger = logging.getLogger("OpenflowModelsParsing")

def as_is_slugify(value):
    return value

cntrlr_url_re = re.compile(r"^((tcp)|(ssl)):(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9]):(?P<port>\d+)$")
def validate_controller_url(value):
    def error():
        raise ValidationError(
            u"Invalid controller URL. The format is "
            "tcp:<hostname>:<port> or ssl:<hostname>:<port>. Port must "
            "be less than %s" % (2**16),
            code="invalid",
        )

    m = cntrlr_url_re.match(value)
    if m:
        port = m.group("port")
        if not port:
            error()
        else:
            port = int(port)
            if port > 2**16-1:
                error()
    else:
        error()
    
class OpenFlowSliceInfo(models.Model):
    slice = models.OneToOneField(slice_models.Slice)
    controller_url = models.CharField(
        "OpenFlow controller URL", max_length=100,
        validators=[validate_controller_url],
        help_text="e.g. tcp:beirut.stanford.edu:6633")
    # TODO: It is not a good idea to store the password in the clear.
    password = models.CharField(
        max_length=64, help_text="This is the password to use when accessing\
 the slice directly at the flowvisors.")

class OpenFlowAggregate(aggregate_models.Aggregate):
    information = \
"""
OpenFlow is an open standard that allows network switches to be controlled by
a remote controller. It enables researchers to run experimental protocols in
production networks, and is currently deployed in several universities.
"""

    client = models.OneToOneField(PasswordXMLRPCServerProxy)
    usage_agreement = models.TextField()
    
    class Meta:
        verbose_name = "OpenFlow Aggregate"
        
    def setup_new_aggregate(self, base_uri):
        self.client.install_trusted_ca()
        # TODO: re-enable this for security. Currently disabled for testing.
#        err = self.client.change_password()
#        if err: return err
        if base_uri.endswith("/"): base_uri = base_uri[:-1]
        try:
            logger.debug("Registering topology callback at %s%s" % (
                base_uri, reverse("openflow_open_xmlrpc")))
            err = self.client.proxy.register_topology_callback(
                "%s%s" % (base_uri, reverse("openflow_open_xmlrpc")),
                "%s" % self.pk,
            )
            if err: return err
        except Exception as ret_exception:
            import traceback
            logger.info("XML RPC call failed to aggregate %s" % self.name)
            traceback.print_exc()
            return str(ret_exception)

        err = self.update_topology()
        if err: return err
        
    def get_raw_topology(self):
        '''
        Get the topology for this aggregate as a set of links.
        '''
        return get_raw_topology(self)

    def parse_switches(self, active_switches_raw):
        '''
        Update the set of available switches.
        '''
        # switches already in the DB.
        current_switches = OpenFlowSwitch.objects.filter(aggregate=self)
        
        switches = {}
        for switch_info in active_switches_raw:
            dpid, info = switch_info
            
            # The ports is a comma-separated set of values (see flowvisor API).
            portstrs = info["portList"].split(",")
            # turn into port numbers
            ports = [int(p) for p in portstrs if p != ""]
            
            switches[dpid] = ports
            
        create_or_update_switches(self, switches)            

    def parse_links(self, active_links_raw):
        '''
        Get the available links and update the network connections.
        '''
        create_or_update_links(self, active_links_raw)
        
    def update_topology(self):
        '''
        Read the topology from the OM and FV, parse it, and store it.
        '''
        try:
            switches = self.client.proxy.get_switches()
            links = self.client.proxy.get_links()
        except:
            import traceback
            traceback.print_exc()
            raise
        self.parse_switches(switches)
        self.parse_links(links)
        
    def _get_slice_id(self, slice):
        """
        Get a slice id to use when creating slices at the OM.
        """
        return "%s_%s" % (Site.objects.get_current().domain, slice.id)
        
    def _get_slivers(self, slice):
        """
        Get the set of slivers in the slice for this aggregate in a format
        that the OM understands.
        
        @param slice: The slice to get slivers from
        @type slice: L{expedient.clearinghouse.slice.models.Slice}
        @return: C{dict} containing a mapping from datapath ids in the aggregate
            to flowspaces on that switch.
        @rtype: C{dict}
        """
        # get all interfaces in this slice
        ifaces = OpenFlowInterface.objects.filter(
            slice_set=slice, aggregate__id=self.id).select_related(
                'flowspacerule_set', 'switch')
        
        # get the list of dpids in the slice
        dpids = set(ifaces.values_list('switch__datapath_id', flat=True))
        
        sw_slivers = []
        # For each dpid, get the flowspace by looking at the interface slivers
        for dpid in dpids:
            d = {}
            d['datapath_id'] = dpid
            d['flowspace'] = []
            for iface in ifaces.filter(switch__datapath_id=dpid):
                sliver = iface.sliver_set.get(slice=slice).as_leaf_class()
                for fs in sliver.flowspacerule_set.all():
                    fsd = {"port_num_start": iface.port_num,
                           "port_num_end": iface.port_num}
                    for f in fs._meta.fields:
                        if f.name != "slivers":
                            v = getattr(fs, f.name)
                            if v is not None and v is not "":
                                fsd[f.name] = v
                            else:
                                fsd[f.name] = "*"
                    d['flowspace'].append(fsd)
            sw_slivers.append(d)
            
        return sw_slivers

    ###################################################################
    # Following are overrides from aggregate_models.Aggregate         #
    ###################################################################
    
    @classmethod
    def get_url_name_prefix(cls):
        return "openflow"
    
    def check_status(self):
        return self.available and self.client.is_available()

    def start_slice(self, slice):
        super(OpenFlowAggregate, self).start_slice(slice)
        sw_slivers = self._get_slivers(slice)
        try:
            return self.client.proxy.create_slice(
                self._get_slice_id(slice), slice.project.name,
                slice.project.description,
                slice.name, slice.description,
                slice.openflowsliceinfo.controller_url,
                slice.owner.email,
                slice.openflowsliceinfo.password, sw_slivers)<commit_msg>account for unicode<commit_after>'''
Created on Apr 26, 2010

@author: jnaous
'''

import re
from django.db import models
from expedient.clearinghouse.resources import models as resource_models
from expedient.clearinghouse.slice import models as slice_models
from expedient.clearinghouse.aggregate import models as aggregate_models
from expedient.common.xmlrpc_serverproxy.models import PasswordXMLRPCServerProxy
from django.core.urlresolvers import reverse
from django.utils.datetime_safe import datetime
from autoslug.fields import AutoSlugField
from django.db.models import signals
from django.contrib.sites.models import Site
import logging
from expedient.common.utils import create_or_update, modelfields
from expedient.clearinghouse.slice.models import Slice
from django.db.models.aggregates import Count
from django.core.exceptions import ValidationError

logger = logging.getLogger("OpenflowModels")
parse_logger = logging.getLogger("OpenflowModelsParsing")

def as_is_slugify(value):
    return value

cntrlr_url_re = re.compile(r"^((tcp)|(ssl)):(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9]):(?P<port>\d+)$")
def validate_controller_url(value):
    def error():
        raise ValidationError(
            u"Invalid controller URL. The format is "
            "tcp:<hostname>:<port> or ssl:<hostname>:<port>. Port must "
            "be less than %s" % (2**16),
            code="invalid",
        )

    m = cntrlr_url_re.match(value)
    if m:
        port = m.group("port")
        if not port:
            error()
        else:
            port = int(port)
            if port > 2**16-1:
                error()
    else:
        error()
    
class OpenFlowSliceInfo(models.Model):
    slice = models.OneToOneField(slice_models.Slice)
    controller_url = models.CharField(
        "OpenFlow controller URL", max_length=100,
        validators=[validate_controller_url],
        help_text="e.g. tcp:beirut.stanford.edu:6633")
    # TODO: It is not a good idea to store the password in the clear.
    password = models.CharField(
        max_length=64, help_text="This is the password to use when accessing\
 the slice directly at the flowvisors.")

class OpenFlowAggregate(aggregate_models.Aggregate):
    information = \
"""
OpenFlow is an open standard that allows network switches to be controlled by
a remote controller. It enables researchers to run experimental protocols in
production networks, and is currently deployed in several universities.
"""

    client = models.OneToOneField(PasswordXMLRPCServerProxy)
    usage_agreement = models.TextField()
    
    class Meta:
        verbose_name = "OpenFlow Aggregate"
        
    def setup_new_aggregate(self, base_uri):
        self.client.install_trusted_ca()
        # TODO: re-enable this for security. Currently disabled for testing.
#        err = self.client.change_password()
#        if err: return err
        if base_uri.endswith("/"): base_uri = base_uri[:-1]
        try:
            logger.debug("Registering topology callback at %s%s" % (
                base_uri, reverse("openflow_open_xmlrpc")))
            err = self.client.proxy.register_topology_callback(
                "%s%s" % (base_uri, reverse("openflow_open_xmlrpc")),
                "%s" % self.pk,
            )
            if err: return err
        except Exception as ret_exception:
            import traceback
            logger.info("XML RPC call failed to aggregate %s" % self.name)
            traceback.print_exc()
            return str(ret_exception)

        err = self.update_topology()
        if err: return err
        
    def get_raw_topology(self):
        '''
        Get the topology for this aggregate as a set of links.
        '''
        return get_raw_topology(self)

    def parse_switches(self, active_switches_raw):
        '''
        Update the set of available switches.
        '''
        # switches already in the DB.
        current_switches = OpenFlowSwitch.objects.filter(aggregate=self)
        
        switches = {}
        for switch_info in active_switches_raw:
            dpid, info = switch_info
            
            # The ports is a comma-separated set of values (see flowvisor API).
            portstrs = info["portList"].split(",")
            # turn into port numbers
            ports = [int(p) for p in portstrs if p != ""]
            
            switches[dpid] = ports
            
        create_or_update_switches(self, switches)            

    def parse_links(self, active_links_raw):
        '''
        Get the available links and update the network connections.
        '''
        create_or_update_links(self, active_links_raw)
        
    def update_topology(self):
        '''
        Read the topology from the OM and FV, parse it, and store it.
        '''
        try:
            switches = self.client.proxy.get_switches()
            links = self.client.proxy.get_links()
        except:
            import traceback
            traceback.print_exc()
            raise
        self.parse_switches(switches)
        self.parse_links(links)
        
    def _get_slice_id(self, slice):
        """
        Get a slice id to use when creating slices at the OM.
        """
        return "%s_%s" % (Site.objects.get_current().domain, slice.id)
        
    def _get_slivers(self, slice):
        """
        Get the set of slivers in the slice for this aggregate in a format
        that the OM understands.
        
        @param slice: The slice to get slivers from
        @type slice: L{expedient.clearinghouse.slice.models.Slice}
        @return: C{dict} containing a mapping from datapath ids in the aggregate
            to flowspaces on that switch.
        @rtype: C{dict}
        """
        # get all interfaces in this slice
        ifaces = OpenFlowInterface.objects.filter(
            slice_set=slice, aggregate__id=self.id).select_related(
                'flowspacerule_set', 'switch')
        
        # get the list of dpids in the slice
        dpids = set(ifaces.values_list('switch__datapath_id', flat=True))
        
        sw_slivers = []
        # For each dpid, get the flowspace by looking at the interface slivers
        for dpid in dpids:
            d = {}
            d['datapath_id'] = dpid
            d['flowspace'] = []
            for iface in ifaces.filter(switch__datapath_id=dpid):
                sliver = iface.sliver_set.get(slice=slice).as_leaf_class()
                for fs in sliver.flowspacerule_set.all():
                    fsd = {"port_num_start": iface.port_num,
                           "port_num_end": iface.port_num}
                    for f in fs._meta.fields:
                        if f.name != "slivers":
                            v = getattr(fs, f.name)
                            if v is not None and v != "":
                                fsd[f.name] = v
                            else:
                                fsd[f.name] = "*"
                    d['flowspace'].append(fsd)
            sw_slivers.append(d)
            
        return sw_slivers

    ###################################################################
    # Following are overrides from aggregate_models.Aggregate         #
    ###################################################################
    
    @classmethod
    def get_url_name_prefix(cls):
        return "openflow"
    
    def check_status(self):
        return self.available and self.client.is_available()

    def start_slice(self, slice):
        super(OpenFlowAggregate, self).start_slice(slice)
        sw_slivers = self._get_slivers(slice)
        try:
            return self.client.proxy.create_slice(
                self._get_slice_id(slice), slice.project.name,
                slice.project.description,
                slice.name, slice.description,
                slice.openflowsliceinfo.controller_url,
                slice.owner.email,
                slice.openflowsliceinfo.password, sw_slivers)
{'fraction_non_alphanumeric': 0.10707600149756645, 'fraction_numerical': 0.004367902158991638, 'mean_word_length': 2.1339460305044975, 'pattern_counts': {'":': 8, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '232458', 'n_tokens_mistral': 4633, 'n_tokens_neox': 4484, 'n_words': 1159}