<commit_before># -*- coding: utf-8 -*-
import json
import time
import traceback
from queue import Queue
from SuperGLU.Core.FIPA.SpeechActs import REQUEST_WHENEVER_ACT
from SuperGLU.Core.Messaging import Message
from SuperGLU.Util.ErrorHandling import logError, logWarning
from SuperGLU.Util.Serialization import (Serializable, serializeObject,
                            nativizeObject)
import datetime
import stomp
import urllib
from edu.usc.ict.superglu.core.blackwhitelist import BlackWhiteListEntry
from edu.usc.ict.superglu.core.config import GatewayBlackWhiteListConfiguration

CATCH_BAD_MESSAGES = False
SESSION_KEY = 'sessionId'
ORIGINATING_SERVICE_ID_KEY = 'originatingServiceId'
USE_BLACK_WHITE_LIST = True

class BaseMessagingNode(Serializable):
    """ Base class for messaging """

    def __init__(self, anId=None, gateway=None, authenticator=None, blackList=[], whiteList=[]):
        super(BaseMessagingNode, self).__init__(anId)
        self._gateway = gateway
        self._authenticator = authenticator
        if gateway is not None:
            self.bindToGateway(gateway)
        self._requests = {}
        
        #don't allow null values for the black and white list
        #not sure if this is necessary but it won't hurt
        if blackList is None:
            blackList = []
        
        if whiteList is None:
            whiteList = []
            
        self._blackList = blackList
        self._whiteList = whiteList

    def acceptIncomingMessage(self, msg):
        result = True
        
        if USE_BLACK_WHITE_LIST:
            for entry in self._whiteList:
                if entry.evaluateMessage(msg):
                    result = True
                    break
                
            for entry in self._blackList:
                if entry.evaluateMessage(msg):
                    result = False
                    break
        return result
    
    def sendMessage(self, msg):
        print("%s sending %s"%(self.__class__.__name__, msg))
        if self._gateway is not None:
            print("Actually sent it.")
            self._gateway.dispatchMessage(msg, self.getId())

    def receiveMessage(self, msg):
        if not self.acceptIncomingMessage(msg):
            return
        self._triggerRequests(msg)

    def bindToGateway(self, gateway):
        self.unbindToGateway()
        self._gateway = gateway
        self._gateway.register(self)
    
    def unbindToGateway(self):
        if self._gateway is not None:
            self._gateway.unregister(self)
    
    def getMessageConditions(self):
        """ Function to check if this node is interested in this message type """
        return None

    def _getRequests(self):
        return list(self._requests.values())

    def _addRequest(self, msg, callback):
        if callback is not None:
            self._requests[msg.getId()] = (msg.clone(), callback)

    def _makeRequest(self, msg, callback):
        self._addRequest(msg, callback)
        self.sendMessage(msg)
    
    def _triggerRequests(self, msg):
        convoId = msg.getContextValue(Message.CONTEXT_CONVERSATION_ID_KEY, None)
        if convoId is not None and convoId in self._requests:
            key = convoId
            oldMsg = self._requests[key][0]
            callback = self._requests[key][1]
            callback(msg, oldMsg)
            # Remove from the requests, unless asked for a permanent feed
            if oldMsg.getSpeechAct() != REQUEST_WHENEVER_ACT:
                del self._requests[key]
                
    def _createRequestReply(self, msg):
        oldId = msg.getId()
        msg = msg.clone()
        msg.updateId()
        msg.setContextValue(Message.CONTEXT_CONVERSATION_ID_KEY, oldId)
        return msg

   # Pack/Unpack Messages
    def messageToString(self, msg):
        return serializeObject(msg)
    
    def stringToMessage(self, msg):
        if (CATCH_BAD_MESSAGES):
            try:
                msg = nativizeObject(msg)
            except:
                logWarning("ERROR: Could not process message data received.  Received:" + str(msg))
                msg = None
        else:
            msg = nativizeObject(msg)
        return msg

    def messagesToStringList(self, msgs):
        return json.dumps([self.messageToString(m) for m in msgs])

    def stringListToMessages(self, strMsgs):
        return [self.stringToMessage(sm) for sm in json.loads(strMsgs)]


class MessagingGateway(BaseMessagingNode):

    def __init__(self, anId=None, nodes=None, gateway=None, authenticator=None, scope=None):
        if scope is None: scope = {}
        super(MessagingGateway, self).__init__(anId, gateway, authenticator)
        if nodes is None: nodes = []
        self._nodes = {}
        self._scope = scope
        self.addNodes(nodes)
        #TODO: read in from service configuration when it's ready
        self._gatewayBlackList = {}
        self._gatewayWhiteList = {}

    # Receive Messages
    def receiveMessage(self, msg):
        """ When gateway receives a message, it distributes it to child nodes """
        super(MessagingGateway, self).receiveMessage(msg)
        self.distributeMessage(msg, None)
    <commit_msg>added service configuration argument to messaging gateway constructor<commit_after># -*- coding: utf-8 -*-
import json
import time
import traceback
from queue import Queue
from SuperGLU.Core.FIPA.SpeechActs import REQUEST_WHENEVER_ACT
from SuperGLU.Core.Messaging import Message
from SuperGLU.Util.ErrorHandling import logError, logWarning
from SuperGLU.Util.Serialization import (Serializable, serializeObject,
                            nativizeObject)
import datetime
import stomp
import urllib
from edu.usc.ict.superglu.core.blackwhitelist import BlackWhiteListEntry
from edu.usc.ict.superglu.core.config import GatewayBlackWhiteListConfiguration, ServiceConfiguration

CATCH_BAD_MESSAGES = False
SESSION_KEY = 'sessionId'
ORIGINATING_SERVICE_ID_KEY = 'originatingServiceId'
USE_BLACK_WHITE_LIST = True

class BaseMessagingNode(Serializable):
    """ Base class for messaging """

    def __init__(self, anId=None, gateway=None, authenticator=None, blackList=[], whiteList=[]):
        super(BaseMessagingNode, self).__init__(anId)
        self._gateway = gateway
        self._authenticator = authenticator
        if gateway is not None:
            self.bindToGateway(gateway)
        self._requests = {}
        
        #don't allow null values for the black and white list
        #not sure if this is necessary but it won't hurt
        if blackList is None:
            blackList = []
        
        if whiteList is None:
            whiteList = []
            
        self._blackList = blackList
        self._whiteList = whiteList

    def acceptIncomingMessage(self, msg):
        result = True
        
        if USE_BLACK_WHITE_LIST:
            for entry in self._whiteList:
                if entry.evaluateMessage(msg):
                    result = True
                    break
                
            for entry in self._blackList:
                if entry.evaluateMessage(msg):
                    result = False
                    break
        return result
    
    def sendMessage(self, msg):
        print("%s sending %s"%(self.__class__.__name__, msg))
        if self._gateway is not None:
            print("Actually sent it.")
            self._gateway.dispatchMessage(msg, self.getId())

    def receiveMessage(self, msg):
        if not self.acceptIncomingMessage(msg):
            return
        self._triggerRequests(msg)

    def bindToGateway(self, gateway):
        self.unbindToGateway()
        self._gateway = gateway
        self._gateway.register(self)
    
    def unbindToGateway(self):
        if self._gateway is not None:
            self._gateway.unregister(self)
    
    def getMessageConditions(self):
        """ Function to check if this node is interested in this message type """
        return None

    def _getRequests(self):
        return list(self._requests.values())

    def _addRequest(self, msg, callback):
        if callback is not None:
            self._requests[msg.getId()] = (msg.clone(), callback)

    def _makeRequest(self, msg, callback):
        self._addRequest(msg, callback)
        self.sendMessage(msg)
    
    def _triggerRequests(self, msg):
        convoId = msg.getContextValue(Message.CONTEXT_CONVERSATION_ID_KEY, None)
        if convoId is not None and convoId in self._requests:
            key = convoId
            oldMsg = self._requests[key][0]
            callback = self._requests[key][1]
            callback(msg, oldMsg)
            # Remove from the requests, unless asked for a permanent feed
            if oldMsg.getSpeechAct() != REQUEST_WHENEVER_ACT:
                del self._requests[key]
                
    def _createRequestReply(self, msg):
        oldId = msg.getId()
        msg = msg.clone()
        msg.updateId()
        msg.setContextValue(Message.CONTEXT_CONVERSATION_ID_KEY, oldId)
        return msg

   # Pack/Unpack Messages
    def messageToString(self, msg):
        return serializeObject(msg)
    
    def stringToMessage(self, msg):
        if (CATCH_BAD_MESSAGES):
            try:
                msg = nativizeObject(msg)
            except:
                logWarning("ERROR: Could not process message data received.  Received:" + str(msg))
                msg = None
        else:
            msg = nativizeObject(msg)
        return msg

    def messagesToStringList(self, msgs):
        return json.dumps([self.messageToString(m) for m in msgs])

    def stringListToMessages(self, strMsgs):
        return [self.stringToMessage(sm) for sm in json.loads(strMsgs)]


class MessagingGateway(BaseMessagingNode):

    def __init__(self, anId=None, nodes=None, gateway=None, authenticator=None, scope=None, serviceConfiguration=None):
        if scope is None: scope = {}
        if serviceConfiguration is not None:
            super(MessagingGateway, self).__init__(anId, gateway, authenticator, serviceConfiguration.getBlackList(), serviceConfiguration.getWhiteList)
        else:
            super(MessagingGateway, self).__init__(anId, geteway, authenticator)
        if nodes is None: nodes = []
        self._nodes = {}
        self._scope = scope
        self.addNodes(nodes)
        #TODO: read in from service configuration when it's ready
        self._gatewayBlackList = {}
        self._gatewayWhiteList = {}

    # Receive Messages
    def receiveMessage(self, msg):
        """ When gateway receives a message, it distributes it to child nodes """
        super(MessagingGateway, self).receiveMessage(msg)
        self.distributeMessage(msg, None)
    
{'fraction_non_alphanumeric': 0.08197027759603702, 'fraction_numerical': 0.0005608000747733433, 'mean_word_length': 2.3406181704651887, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3010304', 'n_tokens_mistral': 2983, 'n_tokens_neox': 2908, 'n_words': 777}