<commit_before>"""
Generic backend for deployments where OpenNSA is the only NRM (i.e. there is no
other system for interacting with the hardware).

Using this module, such a backend will only have to supply functionality for
setting up and tearing down links and does not have to deal state management.

The use this module a connection manager has to be supplied. The methods
setupLink(source_port, dest_port) and tearDown(source_port, dest_port) must be
implemented in the manager. The methods should return a deferred.

Author: <NAME> <<EMAIL>>
Copyright: NORDUnet (2011-2012)
"""

import datetime

from zope.interface import implements

from twisted.python import log
from twisted.internet import reactor, defer
from twisted.application import service

from opennsa.interface import INSIProvider

from opennsa import error, state, nsa
from opennsa.backends.common import scheduler, calendar

from twistar.dbobject import DBObject



class GenericBackendConnections(DBObject):
    pass



class GenericBackend(service.Service):

    implements(INSIProvider)

    TPC_TIMEOUT = 30 # seconds

    def __init__(self, network, network_topology, connection_manager, parent_requester, log_system):

        assert network == network_topology.name, 'Network name and network topology name does not match %s != %s' % (network, network_topology.name)

        self.network            = network
        self.network_topology   = network_topology
        self.connection_manager = connection_manager
        self.parent_requester   = parent_requester
        self.log_system         = log_system

        self.notification_id = 0

        self.scheduler = scheduler.CallScheduler()
        self.calendar  = calendar.ReservationCalendar()
        # need to build the calendar as well

        # need to build schedule here
        self.restore_defer = defer.Deferred()
        reactor.callWhenRunning(self.buildSchedule)


    def startService(self):
        service.Service.startService(self)


    def stopService(self):
        service.Service.stopService(self)
        return self.restore_defer.addCallback( lambda _ : self.scheduler.cancelAllCalls() )


    def getNotificationId(self):
        nid = self.notification_id
        self.notification_id += 1
        return nid


    @defer.inlineCallbacks
    def buildSchedule(self):

        conns = yield GenericBackendConnections.find(where=['lifecycle_state <> ?', state.TERMINATED])
        for conn in conns:
            # avoid race with newly created connections
            if self.scheduler.hasScheduledCall(conn.connection_id):
                continue

            now = datetime.datetime.utcnow()

            if conn.lifecycle_state in (state.PASSED_ENDTIME, state.TERMINATED):
                pass # This connection has already lived it life to the fullest :-)

            elif conn.end_time < now and conn.lifecycle_state not in (state.PASSED_ENDTIME, state.TERMINATED):
                log.msg('Connection %s: Immediate end during buildSchedule' % conn.connection_id, system=self.log_system)
                yield self._doEndtime(conn)

            elif conn.start_time > now:
                # start time has not yet passed, we most schedule activate or schedule terminate depending on state
                if conn.provision_state == state.PROVISIONED and conn.data_plane_active == False:
                    self.scheduler.scheduleCall(conn.connection_id, conn.start_time, self._doActivate, conn)
                    td = conn.start_time - now
                    log.msg('Connection %s: activate scheduled for %s UTC (%i seconds) (buildSchedule)' % (conn.connection_id, conn.end_time.replace(microsecond=0), td.total_seconds()), system=self.log_system)
                elif conn.provision_state == state.RELEASED:
                    self.scheduler.scheduleCall(conn.connection_id, conn.end_time, self._doEndtime, conn)
                    td = conn.end_time - now
                    log.msg('Connection %s: End scheduled for %s UTC (%i seconds) (buildSchedule' % (conn.connection_id, conn.end_time.replace(microsecond=0), td.total_seconds()), system=self.log_system)
                else:
                    log.msg('Unhandled provision state %s for connection %s in scheduler building' % (conn.provision_state, conn.connection_id))

            elif conn.start_time < now:
                # we have passed start time, we must either: activate, schedule deactive, or schedule terminate
                if conn.provision_state == state.PROVISIONED:
                    if conn.data_plane_active:
                        self.scheduler.scheduleCall(conn.connection_id, conn.end_time, self._doEndtime, conn)
                        td = conn.end_time - now
                        log.msg('Connection %s: already active, scheduling end for %s UTC (%i seconds) (buildSchedule)' % (conn.connection_id, conn.end_time.replace(microsecond=0), td.total_seconds()), system=self.log_system)
                    else:
                        log.msg('Connection %s: Immediate activate during buildSchedule' % conn.connection_id, system=self.log_system)
                        yield self._doActivate(conn)
                elif conn.provision_state == state.RELEASED:
                    self.scheduler.scheduleCall(conn.connection_id, conn.end_time, self._doEndtime, conn)
                    td = conn.end_time - now
                    log.msg('Connection %s: End scheduled for %s UTC (%i seconds) (buildSchedule)' % (conn.connection_id, conn.end_time.replace(microsecond=0), td.total_seconds()), system=self.log_system)
                else:
                    log.msg('Unhandled provision state %s for connection %s in scheduler building' % (conn.provision_state, conn.connection_id))

            else:
                log.msg('Unhandled start/end time configuration for connection %s' % conn.connection_id)

        self.restore_defer.callback(None)



    @defer.inlineCallbacks
    def _getConnection(self, connection_id, requester_nsa):
        # add security check sometime

        conns = yield GenericBackendConnections.findBy(connection_id=connection_id)
        if len(conns) == 0:
            raise error.ConnectionNonExistentError('No connection with id %s' % connection_id)
        defer.returnValue( conns[0] ) # we only get one, unique in db


    def logStateUpdate(self, conn, state_msg):
        src_target = self.connection_manager.getTarget(conn.source_port, conn.source_labels[0].type_, conn.source_labels[0].labelValue())
        dst_target = self.connection_manager.getTarget(conn.dest_port,   conn.dest_labels[0].type_,   conn.dest_labels[0].labelValue())
        log.msg('Connection %s: %s -> %s %s' % (conn.connection_id, src_target, dst_target, state_msg), system=self.log_system)


    @defer.inlineCallbacks
    def reserve(self, header, connection_id, global_reservation_id, description, service_params):

        # return defer.fail( error.InternalNRMError('test reservation failure') )

        # should perhaps verify nsa, but not that important
        log.msg('Reserve request. Connection ID: %s' % connection_id, system=self.log_system)

        if connection_id:
            raise ValueError('Cannot handle cases with existing connection id (yet)')
            #conns = yield GenericBackendConnections.findBy(connection_id=connection_id)

        source_stp = service_params.source_stp
        dest_stp   = service_params.dest_stp

        # check network and ports exist

        if source_stp.network != self.network:
            raise error.TopologyError('Source network does not match network this NSA is managing')
        if dest_stp.network != self.network:
            raise error.TopologyError('Source network does not match network this NSA is managing')

        # ensure that ports actually exists
        topo_source_port = self.network_topology.getPort(source_stp.port)
        topo_dest_port   = self.network_topology.getPort(dest_stp.port)

        # basic label check
        if len(source_stp.labels) == 0:
            raise error.TopologyError('Source STP must specify a label')
        if len(dest_stp.labels) == 0:
            raise error.TopologyError('Destination STP must specify a label')

        if len(source_stp.labels) > 1:
            raise error.TopologyError('Source STP specifies more than one label. Only one label is currently supported')
        if len(dest_stp.labels) > 1:
            raise error.TopologyError('Destination STP specifies more than one label. Only one label is currently supported')

        src_label_candidate = source_stp.labels[0]
        dst_label_candidate = dest_stp.labels[0]
        assert src_label_candidate.type_ == dst_label_candidate.type_, 'Cannot connect ports with different label types'

        # check that we are connecting an stp to itself
        if source_stp == dest_stp and source_stp.labels[0].singleValue():
            raise error.TopologyError('Cannot connect STP %s to itself.' % source_stp)

        # now check that the ports have (some of) the specified labels
        if not topo_source_port.canMatchLabels(source_stp.labels):
            raise error.TopologyError('Source port %s cannot match label set %s' % (topo_source_port.name, src_label_candidate ) )
        if not topo_dest_port.canMatchLabels(dest_stp.labels):
            raise error.TopologyError('Destination port %s cannot match label set %s' % (topo_dest_port.name, dst_label_candidate ) )

        # do the: lets find the labels danace
        if self.connection_manager.canSwapLabel(src_label_candidate.type_):
            for lv in src_label_candidate.enumerateValues():
                src_resource = self.connection_manager.getResource(source_stp.port, src_label_candidate.type_, lv)
                try:
                    self.calendar.checkReservation(src_resource, service_params.start_time, service_params.end_time)
                    self.calendar.addReservation(   src_resource, service_params.start_time, service_params.end_time)
                    src_label = nsa.Label(src_label_candidate.type_, str(lv))
                    break
                except error.STPUnavailableError:
                    pass
            else:
                raise error.STPUnavailableError('STP %s not available in specified time span' % source_stp)


            for lv in dst_label_candidate.enumerateValues():
                dst_resource = self.connection_manager.getResource(dest_stp.port, dst_label_candidate.type_, lv)
                try:
                    self.calendar.checkReservation(dst_resource, service_params.start_time, service_params.end_time)
                    self.calendar.addReservation(   dst_resource, service_params.start_time, service_params.end_time)
                    dst_label = nsa.Label(dst_label_candidate.type_, str(lv))
                    break
                except error.STPUnavailableError:
                    pass
            else:
                raise error.STPUnavailableError('STP %s not available in specified time span' % dest_stp)

        else:
            label_candidate = src_label_candidate.intersect(dst_label_candidate)

            for lv in label_candidate.enumerateValues():
                src_resource = self.connection_manager.getResource(source_stp.port, label_candidate.type_, lv)
                dst_resource = self.connection_manager.getResource(dest_stp.port,   label_candidate.type_, lv)
                try:
                    self.calendar.checkReservation(src_resource, service_params.start_time, service_params.end_time)
                    self.calendar.checkReservation(dst_resource, service_params.start_time, service_params.end_time)
                    self.calendar.addReservation(   src_resource, service_params.start_time, service_params.end_time)
                    self.calendar.addReservation(   dst_resource, service_params.start_time, service_params.end_time)
                    src_label = nsa.Label(label_candidate.type_, str(lv))
                    dst_label = nsa.Label(label_candidate.type_, str(lv))
                    break
                except error.STPUnavailableError:
                    pass
            else:
                raise error.STPUnavailableError('STP combination %s and %s not available in specified time span' % (source_stp, dest_stp))

        now =  datetime.datetime.utcnow()

        source_target = self.connection_manager.getTarget(source_stp.port, src_label.type_, src_label.labelValue())
        dest_target   = self.connection_manager.getTarget(dest_stp.port,   dst_label.type_, dst_label.labelValue())
        connection_id = self.connection_manager.createConnectionId(source_target, dest_target)

        # we should check the schedule here

        # should we save the requester or provider here?
        conn = GenericBackendConnections(connection_id=connection_id, revision=0, global_reservation_id=global_reservation_id, description=description,
                                         requester_nsa=header.requester_nsa, reserve_time=now,
                                         reservation_state=state.RESERVE_START, provision_state=state.RELEASED, lifecycle_state=state.CREATED, data_plane_active=False,
                                         source_network=source_stp.network, source_port=source_stp.port, source_labels=[src_label],
                                         dest_network=dest_stp.network, dest_port=dest_stp.port, dest_labels=[dst_label],
                                         start_time=service_params.start_time, end_time=service_params.end_time,
                                         bandwidth=service_params.bandwidth)
        yield conn.save()
        reactor.callWhenRunning(self._doReserve, conn, header.correlation_id)
        defer.returnValue(connection_id)


    @defer.inlineCallbacks
    def reserveCommit(self, header, connection_id):

        log.msg('ReserveCommit request. Connection ID: %s' % connection_id, system=self.log_system)

        conn = yield self._getConnection(connection_id, header.requester_nsa)
        if conn.lifecycle_state in (state.TERMINATING, state.TERMINATED):<commit_msg>add log message about id assigning in generic backend<commit_after>"""
Generic backend for deployments where OpenNSA is the only NRM (i.e. there is no
other system for interacting with the hardware).

Using this module, such a backend will only have to supply functionality for
setting up and tearing down links and does not have to deal state management.

The use this module a connection manager has to be supplied. The methods
setupLink(source_port, dest_port) and tearDown(source_port, dest_port) must be
implemented in the manager. The methods should return a deferred.

Author: <NAME> <<EMAIL>>
Copyright: NORDUnet (2011-2012)
"""

import datetime

from zope.interface import implements

from twisted.python import log
from twisted.internet import reactor, defer
from twisted.application import service

from opennsa.interface import INSIProvider

from opennsa import error, state, nsa
from opennsa.backends.common import scheduler, calendar

from twistar.dbobject import DBObject



class GenericBackendConnections(DBObject):
    pass



class GenericBackend(service.Service):

    implements(INSIProvider)

    TPC_TIMEOUT = 30 # seconds

    def __init__(self, network, network_topology, connection_manager, parent_requester, log_system):

        assert network == network_topology.name, 'Network name and network topology name does not match %s != %s' % (network, network_topology.name)

        self.network            = network
        self.network_topology   = network_topology
        self.connection_manager = connection_manager
        self.parent_requester   = parent_requester
        self.log_system         = log_system

        self.notification_id = 0

        self.scheduler = scheduler.CallScheduler()
        self.calendar  = calendar.ReservationCalendar()
        # need to build the calendar as well

        # need to build schedule here
        self.restore_defer = defer.Deferred()
        reactor.callWhenRunning(self.buildSchedule)


    def startService(self):
        service.Service.startService(self)


    def stopService(self):
        service.Service.stopService(self)
        return self.restore_defer.addCallback( lambda _ : self.scheduler.cancelAllCalls() )


    def getNotificationId(self):
        nid = self.notification_id
        self.notification_id += 1
        return nid


    @defer.inlineCallbacks
    def buildSchedule(self):

        conns = yield GenericBackendConnections.find(where=['lifecycle_state <> ?', state.TERMINATED])
        for conn in conns:
            # avoid race with newly created connections
            if self.scheduler.hasScheduledCall(conn.connection_id):
                continue

            now = datetime.datetime.utcnow()

            if conn.lifecycle_state in (state.PASSED_ENDTIME, state.TERMINATED):
                pass # This connection has already lived it life to the fullest :-)

            elif conn.end_time < now and conn.lifecycle_state not in (state.PASSED_ENDTIME, state.TERMINATED):
                log.msg('Connection %s: Immediate end during buildSchedule' % conn.connection_id, system=self.log_system)
                yield self._doEndtime(conn)

            elif conn.start_time > now:
                # start time has not yet passed, we most schedule activate or schedule terminate depending on state
                if conn.provision_state == state.PROVISIONED and conn.data_plane_active == False:
                    self.scheduler.scheduleCall(conn.connection_id, conn.start_time, self._doActivate, conn)
                    td = conn.start_time - now
                    log.msg('Connection %s: activate scheduled for %s UTC (%i seconds) (buildSchedule)' % (conn.connection_id, conn.end_time.replace(microsecond=0), td.total_seconds()), system=self.log_system)
                elif conn.provision_state == state.RELEASED:
                    self.scheduler.scheduleCall(conn.connection_id, conn.end_time, self._doEndtime, conn)
                    td = conn.end_time - now
                    log.msg('Connection %s: End scheduled for %s UTC (%i seconds) (buildSchedule' % (conn.connection_id, conn.end_time.replace(microsecond=0), td.total_seconds()), system=self.log_system)
                else:
                    log.msg('Unhandled provision state %s for connection %s in scheduler building' % (conn.provision_state, conn.connection_id))

            elif conn.start_time < now:
                # we have passed start time, we must either: activate, schedule deactive, or schedule terminate
                if conn.provision_state == state.PROVISIONED:
                    if conn.data_plane_active:
                        self.scheduler.scheduleCall(conn.connection_id, conn.end_time, self._doEndtime, conn)
                        td = conn.end_time - now
                        log.msg('Connection %s: already active, scheduling end for %s UTC (%i seconds) (buildSchedule)' % (conn.connection_id, conn.end_time.replace(microsecond=0), td.total_seconds()), system=self.log_system)
                    else:
                        log.msg('Connection %s: Immediate activate during buildSchedule' % conn.connection_id, system=self.log_system)
                        yield self._doActivate(conn)
                elif conn.provision_state == state.RELEASED:
                    self.scheduler.scheduleCall(conn.connection_id, conn.end_time, self._doEndtime, conn)
                    td = conn.end_time - now
                    log.msg('Connection %s: End scheduled for %s UTC (%i seconds) (buildSchedule)' % (conn.connection_id, conn.end_time.replace(microsecond=0), td.total_seconds()), system=self.log_system)
                else:
                    log.msg('Unhandled provision state %s for connection %s in scheduler building' % (conn.provision_state, conn.connection_id))

            else:
                log.msg('Unhandled start/end time configuration for connection %s' % conn.connection_id)

        self.restore_defer.callback(None)



    @defer.inlineCallbacks
    def _getConnection(self, connection_id, requester_nsa):
        # add security check sometime

        conns = yield GenericBackendConnections.findBy(connection_id=connection_id)
        if len(conns) == 0:
            raise error.ConnectionNonExistentError('No connection with id %s' % connection_id)
        defer.returnValue( conns[0] ) # we only get one, unique in db


    def logStateUpdate(self, conn, state_msg):
        src_target = self.connection_manager.getTarget(conn.source_port, conn.source_labels[0].type_, conn.source_labels[0].labelValue())
        dst_target = self.connection_manager.getTarget(conn.dest_port,   conn.dest_labels[0].type_,   conn.dest_labels[0].labelValue())
        log.msg('Connection %s: %s -> %s %s' % (conn.connection_id, src_target, dst_target, state_msg), system=self.log_system)


    @defer.inlineCallbacks
    def reserve(self, header, connection_id, global_reservation_id, description, service_params):

        # return defer.fail( error.InternalNRMError('test reservation failure') )

        # should perhaps verify nsa, but not that important
        log.msg('Reserve request. Connection ID: %s' % connection_id, system=self.log_system)

        if connection_id:
            raise ValueError('Cannot handle cases with existing connection id (yet)')
            #conns = yield GenericBackendConnections.findBy(connection_id=connection_id)

        source_stp = service_params.source_stp
        dest_stp   = service_params.dest_stp

        # check network and ports exist

        if source_stp.network != self.network:
            raise error.TopologyError('Source network does not match network this NSA is managing')
        if dest_stp.network != self.network:
            raise error.TopologyError('Source network does not match network this NSA is managing')

        # ensure that ports actually exists
        topo_source_port = self.network_topology.getPort(source_stp.port)
        topo_dest_port   = self.network_topology.getPort(dest_stp.port)

        # basic label check
        if len(source_stp.labels) == 0:
            raise error.TopologyError('Source STP must specify a label')
        if len(dest_stp.labels) == 0:
            raise error.TopologyError('Destination STP must specify a label')

        if len(source_stp.labels) > 1:
            raise error.TopologyError('Source STP specifies more than one label. Only one label is currently supported')
        if len(dest_stp.labels) > 1:
            raise error.TopologyError('Destination STP specifies more than one label. Only one label is currently supported')

        src_label_candidate = source_stp.labels[0]
        dst_label_candidate = dest_stp.labels[0]
        assert src_label_candidate.type_ == dst_label_candidate.type_, 'Cannot connect ports with different label types'

        # check that we are connecting an stp to itself
        if source_stp == dest_stp and source_stp.labels[0].singleValue():
            raise error.TopologyError('Cannot connect STP %s to itself.' % source_stp)

        # now check that the ports have (some of) the specified labels
        if not topo_source_port.canMatchLabels(source_stp.labels):
            raise error.TopologyError('Source port %s cannot match label set %s' % (topo_source_port.name, src_label_candidate ) )
        if not topo_dest_port.canMatchLabels(dest_stp.labels):
            raise error.TopologyError('Destination port %s cannot match label set %s' % (topo_dest_port.name, dst_label_candidate ) )

        # do the: lets find the labels danace
        if self.connection_manager.canSwapLabel(src_label_candidate.type_):
            for lv in src_label_candidate.enumerateValues():
                src_resource = self.connection_manager.getResource(source_stp.port, src_label_candidate.type_, lv)
                try:
                    self.calendar.checkReservation(src_resource, service_params.start_time, service_params.end_time)
                    self.calendar.addReservation(   src_resource, service_params.start_time, service_params.end_time)
                    src_label = nsa.Label(src_label_candidate.type_, str(lv))
                    break
                except error.STPUnavailableError:
                    pass
            else:
                raise error.STPUnavailableError('STP %s not available in specified time span' % source_stp)


            for lv in dst_label_candidate.enumerateValues():
                dst_resource = self.connection_manager.getResource(dest_stp.port, dst_label_candidate.type_, lv)
                try:
                    self.calendar.checkReservation(dst_resource, service_params.start_time, service_params.end_time)
                    self.calendar.addReservation(   dst_resource, service_params.start_time, service_params.end_time)
                    dst_label = nsa.Label(dst_label_candidate.type_, str(lv))
                    break
                except error.STPUnavailableError:
                    pass
            else:
                raise error.STPUnavailableError('STP %s not available in specified time span' % dest_stp)

        else:
            label_candidate = src_label_candidate.intersect(dst_label_candidate)

            for lv in label_candidate.enumerateValues():
                src_resource = self.connection_manager.getResource(source_stp.port, label_candidate.type_, lv)
                dst_resource = self.connection_manager.getResource(dest_stp.port,   label_candidate.type_, lv)
                try:
                    self.calendar.checkReservation(src_resource, service_params.start_time, service_params.end_time)
                    self.calendar.checkReservation(dst_resource, service_params.start_time, service_params.end_time)
                    self.calendar.addReservation(   src_resource, service_params.start_time, service_params.end_time)
                    self.calendar.addReservation(   dst_resource, service_params.start_time, service_params.end_time)
                    src_label = nsa.Label(label_candidate.type_, str(lv))
                    dst_label = nsa.Label(label_candidate.type_, str(lv))
                    break
                except error.STPUnavailableError:
                    pass
            else:
                raise error.STPUnavailableError('STP combination %s and %s not available in specified time span' % (source_stp, dest_stp))

        now =  datetime.datetime.utcnow()

        source_target = self.connection_manager.getTarget(source_stp.port, src_label.type_, src_label.labelValue())
        dest_target   = self.connection_manager.getTarget(dest_stp.port,   dst_label.type_, dst_label.labelValue())
        connection_id = self.connection_manager.createConnectionId(source_target, dest_target)
        log.msg('Connection %s: Id assigned.' % connection_id, system=self.log_system)

        # we should check the schedule here

        # should we save the requester or provider here?
        conn = GenericBackendConnections(connection_id=connection_id, revision=0, global_reservation_id=global_reservation_id, description=description,
                                         requester_nsa=header.requester_nsa, reserve_time=now,
                                         reservation_state=state.RESERVE_START, provision_state=state.RELEASED, lifecycle_state=state.CREATED, data_plane_active=False,
                                         source_network=source_stp.network, source_port=source_stp.port, source_labels=[src_label],
                                         dest_network=dest_stp.network, dest_port=dest_stp.port, dest_labels=[dst_label],
                                         start_time=service_params.start_time, end_time=service_params.end_time,
                                         bandwidth=service_params.bandwidth)
        yield conn.save()
        reactor.callWhenRunning(self._doReserve, conn, header.correlation_id)
        defer.returnValue(connection_id)


    @defer.inlineCallbacks
    def reserveCommit(self, header, connection_id):

        log.msg('ReserveCommit request. Connection ID: %s' % connection_id, system=self.log_system)

        conn = yield self._getConnection(connection_id, header.requester_nsa)
        if conn.lifecycle_state in (state.TERMINATING, state.TERMINATED):
{'fraction_non_alphanumeric': 0.07836648477380743, 'fraction_numerical': 0.002112304171800739, 'mean_word_length': 2.6934078793394876, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4999657', 'n_tokens_mistral': 7832, 'n_tokens_neox': 7532, 'n_words': 1871}