<commit_before># -*- test-case-name: vumi.tests.test_message -*-

import json
from uuid import uuid4
from datetime import datetime

from errors import MissingMessageField, InvalidMessageField

from vumi.utils import to_kwargs


# This is the date format we work with internally
VUMI_DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f"


def date_time_decoder(json_object):
    for key, value in json_object.items():
        try:
            json_object[key] = datetime.strptime(value,
                    VUMI_DATE_FORMAT)
        except ValueError:
            continue
        except TypeError:
            continue
    return json_object


class JSONMessageEncoder(json.JSONEncoder):
    """A JSON encoder that is able to serialize datetime"""
    def default(self, obj):
        if isinstance(obj, datetime):
            return obj.strftime(VUMI_DATE_FORMAT)
        return super(JSONMessageEncoder, self).default(obj)


def from_json(json_string):
    return json.loads(json_string, object_hook=date_time_decoder)


def to_json(obj):
    return json.dumps(obj, cls=JSONMessageEncoder)


class Message(object):
    """
    Start of a somewhat unified message object to be
    used internally in Vumi and while being in transit
    over AMQP

    scary transport format -> Vumi Tansport -> Unified Message -> Vumi Worker

    """

    def __init__(self, _process_fields=True, **kwargs):
        if _process_fields:
            kwargs = self.process_fields(kwargs)
        self.payload = kwargs
        self.validate_fields()

    def process_fields(self, fields):
        return fields

    def validate_fields(self):
        pass

    def assert_field_present(self, *fields):
        for field in fields:
            if field not in self.payload:
                raise MissingMessageField(field)

    def assert_field_value(self, field, *values):
        self.assert_field_present(field)
        if self.payload[field] not in values:
            raise InvalidMessageField(field)

    def to_json(self):
        return to_json(self.payload)

    @classmethod
    def from_json(cls, json_string):
        return cls(_process_fields=False, **to_kwargs(from_json(json_string)))

    def __str__(self):
        return u"<Message payload=\"%s\">" % repr(self.payload)

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        if isinstance(other, Message):
            return self.payload == other.payload
        return False

    def __contains__(self, key):
        return key in self.payload

    def __getitem__(self, key):
        return self.payload[key]

    def __setitem__(self, key, value):
        self.payload[key] = value

    def get(self, key, default=None):
        return self.payload.get(key, default)

    def items(self):
        return self.payload.items()

    def copy(self):
        return self.from_json(self.to_json())


class TransportMessage(Message):
    """Common base class for messages sent to or from a transport."""

    # sub-classes should set the message type
    MESSAGE_TYPE = None
    MESSAGE_VERSION = '20110921'
    DEFAULT_ENDPOINT_NAME = 'default'

    @staticmethod
    def generate_id():
        """
        Generate a unique message id.

        There are places where we want a message id before we can
        build a complete message. This lets us do that in a consistent
        manner.
        """
        return uuid4().get_hex()

    def process_fields(self, fields):
        fields.setdefault('message_version', self.MESSAGE_VERSION)
        fields.setdefault('message_type', self.MESSAGE_TYPE)
        fields.setdefault('timestamp', datetime.utcnow())
        fields.setdefault('routing_metadata', {})
        fields.setdefault('helper_metadata', {})
        return fields

    def validate_fields(self):
        self.assert_field_value('message_version', self.MESSAGE_VERSION)
        # We might get older event messages without the `helper_metadata`
        # field.
        self.payload.setdefault('helper_metadata', {})
        self.assert_field_present(
            'message_type',
            'timestamp',
            'helper_metadata',
            )
        if self['message_type'] is None:
            raise InvalidMessageField('message_type')

    @property
    def routing_metadata(self):
        return self.payload.setdefault('routing_metadata', {})

    @classmethod
    def check_routing_endpoint(cls, endpoint_name):
        if endpoint_name is None:
            return cls.DEFAULT_ENDPOINT_NAME
        return endpoint_name

    def set_routing_endpoint(self, endpoint_name=None):
        endpoint_name = self.check_routing_endpoint(endpoint_name)
        self.routing_metadata['endpoint_name'] = endpoint_name

    def get_routing_endpoint(self):
        endpoint_name = self.routing_metadata.get('endpoint_name')
        return self.check_routing_endpoint(endpoint_name)


class TransportUserMessage(TransportMessage):
    """Message to or from a user.

    transport_type = sms, ussd, etc
    helper_metadata = for use by dispathers and off-to-the-side
                      components like failure workers (not for use
                      by transports or message workers).
    """

    MESSAGE_TYPE = 'user_message'

    # session event constants
    #
    # SESSION_NONE, SESSION_NEW, SESSION_RESUME, and SESSION_CLOSE
    # may be sent from the transport to a worker. SESSION_NONE indicates
    # there is no relevant session for this message.
    #
    # SESSION_NONE and SESSION_CLOSE may be sent from the worker to
    # the transport. SESSION_NONE indicates any existing session
    # should be continued. SESSION_CLOSE indicates that any existing
    # session should be terminated after sending the message.
    SESSION_NONE, SESSION_NEW, SESSION_RESUME, SESSION_CLOSE = (
        None, 'new', 'resume', 'close')

    # list of valid session events
    SESSION_EVENTS = frozenset([SESSION_NONE, SESSION_NEW, SESSION_RESUME,
                                SESSION_CLOSE])

    # canonical transport types
    TT_HTTP_API = 'http_api'
    TT_IRC = 'irc'
    TT_TELNET = 'telnet'
    TT_TWITTER = 'twitter'
    TT_SMS = 'sms'
    TT_USSD = 'ussd'
    TT_XMPP = 'xmpp'
    TT_MXIT = 'mxit'
    TT_WECHAT = 'wechat'
    TRANSPORT_TYPES = set([TT_HTTP_API, TT_IRC, TT_TELNET, TT_TWITTER, TT_SMS,
                           TT_USSD, TT_XMPP, TT_MXIT, TT_WECHAT])

    AT_IRC_NICKNAME = 'irc_nickname'
    AT_TELNET_ADDRESS = 'telnet_address'
    AT_TWITTER_HANDLE = 'twitter_handle'
    AT_MSISDN = 'msisdn'
    AT_GTALK_ID = 'gtalk_id'
    AT_JABBER_ID = 'jabber_id'
    AT_MXIT_ID = 'mxit_id'
    AT_WECHAT_ID = 'wechat_id'
    ADDRESS_TYPES = set([
        AT_IRC_NICKNAME, AT_TELNET_ADDRESS, AT_TWITTER_HANDLE, AT_MSISDN,
        AT_GTALK_ID, AT_JABBER_ID, AT_MXIT_ID, AT_WECHAT_ID])

    def process_fields(self, fields):
        fields = super(TransportUserMessage, self).process_fields(fields)
        fields.setdefault('message_id', self.generate_id())
        fields.setdefault('in_reply_to', None)
        fields.setdefault('provider', None)
        fields.setdefault('session_event', None)<commit_msg>Remove telnet address type.<commit_after># -*- test-case-name: vumi.tests.test_message -*-

import json
from uuid import uuid4
from datetime import datetime

from errors import MissingMessageField, InvalidMessageField

from vumi.utils import to_kwargs


# This is the date format we work with internally
VUMI_DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f"


def date_time_decoder(json_object):
    for key, value in json_object.items():
        try:
            json_object[key] = datetime.strptime(value,
                    VUMI_DATE_FORMAT)
        except ValueError:
            continue
        except TypeError:
            continue
    return json_object


class JSONMessageEncoder(json.JSONEncoder):
    """A JSON encoder that is able to serialize datetime"""
    def default(self, obj):
        if isinstance(obj, datetime):
            return obj.strftime(VUMI_DATE_FORMAT)
        return super(JSONMessageEncoder, self).default(obj)


def from_json(json_string):
    return json.loads(json_string, object_hook=date_time_decoder)


def to_json(obj):
    return json.dumps(obj, cls=JSONMessageEncoder)


class Message(object):
    """
    Start of a somewhat unified message object to be
    used internally in Vumi and while being in transit
    over AMQP

    scary transport format -> Vumi Tansport -> Unified Message -> Vumi Worker

    """

    def __init__(self, _process_fields=True, **kwargs):
        if _process_fields:
            kwargs = self.process_fields(kwargs)
        self.payload = kwargs
        self.validate_fields()

    def process_fields(self, fields):
        return fields

    def validate_fields(self):
        pass

    def assert_field_present(self, *fields):
        for field in fields:
            if field not in self.payload:
                raise MissingMessageField(field)

    def assert_field_value(self, field, *values):
        self.assert_field_present(field)
        if self.payload[field] not in values:
            raise InvalidMessageField(field)

    def to_json(self):
        return to_json(self.payload)

    @classmethod
    def from_json(cls, json_string):
        return cls(_process_fields=False, **to_kwargs(from_json(json_string)))

    def __str__(self):
        return u"<Message payload=\"%s\">" % repr(self.payload)

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        if isinstance(other, Message):
            return self.payload == other.payload
        return False

    def __contains__(self, key):
        return key in self.payload

    def __getitem__(self, key):
        return self.payload[key]

    def __setitem__(self, key, value):
        self.payload[key] = value

    def get(self, key, default=None):
        return self.payload.get(key, default)

    def items(self):
        return self.payload.items()

    def copy(self):
        return self.from_json(self.to_json())


class TransportMessage(Message):
    """Common base class for messages sent to or from a transport."""

    # sub-classes should set the message type
    MESSAGE_TYPE = None
    MESSAGE_VERSION = '20110921'
    DEFAULT_ENDPOINT_NAME = 'default'

    @staticmethod
    def generate_id():
        """
        Generate a unique message id.

        There are places where we want a message id before we can
        build a complete message. This lets us do that in a consistent
        manner.
        """
        return uuid4().get_hex()

    def process_fields(self, fields):
        fields.setdefault('message_version', self.MESSAGE_VERSION)
        fields.setdefault('message_type', self.MESSAGE_TYPE)
        fields.setdefault('timestamp', datetime.utcnow())
        fields.setdefault('routing_metadata', {})
        fields.setdefault('helper_metadata', {})
        return fields

    def validate_fields(self):
        self.assert_field_value('message_version', self.MESSAGE_VERSION)
        # We might get older event messages without the `helper_metadata`
        # field.
        self.payload.setdefault('helper_metadata', {})
        self.assert_field_present(
            'message_type',
            'timestamp',
            'helper_metadata',
            )
        if self['message_type'] is None:
            raise InvalidMessageField('message_type')

    @property
    def routing_metadata(self):
        return self.payload.setdefault('routing_metadata', {})

    @classmethod
    def check_routing_endpoint(cls, endpoint_name):
        if endpoint_name is None:
            return cls.DEFAULT_ENDPOINT_NAME
        return endpoint_name

    def set_routing_endpoint(self, endpoint_name=None):
        endpoint_name = self.check_routing_endpoint(endpoint_name)
        self.routing_metadata['endpoint_name'] = endpoint_name

    def get_routing_endpoint(self):
        endpoint_name = self.routing_metadata.get('endpoint_name')
        return self.check_routing_endpoint(endpoint_name)


class TransportUserMessage(TransportMessage):
    """Message to or from a user.

    transport_type = sms, ussd, etc
    helper_metadata = for use by dispathers and off-to-the-side
                      components like failure workers (not for use
                      by transports or message workers).
    """

    MESSAGE_TYPE = 'user_message'

    # session event constants
    #
    # SESSION_NONE, SESSION_NEW, SESSION_RESUME, and SESSION_CLOSE
    # may be sent from the transport to a worker. SESSION_NONE indicates
    # there is no relevant session for this message.
    #
    # SESSION_NONE and SESSION_CLOSE may be sent from the worker to
    # the transport. SESSION_NONE indicates any existing session
    # should be continued. SESSION_CLOSE indicates that any existing
    # session should be terminated after sending the message.
    SESSION_NONE, SESSION_NEW, SESSION_RESUME, SESSION_CLOSE = (
        None, 'new', 'resume', 'close')

    # list of valid session events
    SESSION_EVENTS = frozenset([SESSION_NONE, SESSION_NEW, SESSION_RESUME,
                                SESSION_CLOSE])

    # canonical transport types
    TT_HTTP_API = 'http_api'
    TT_IRC = 'irc'
    TT_TELNET = 'telnet'
    TT_TWITTER = 'twitter'
    TT_SMS = 'sms'
    TT_USSD = 'ussd'
    TT_XMPP = 'xmpp'
    TT_MXIT = 'mxit'
    TT_WECHAT = 'wechat'
    TRANSPORT_TYPES = set([TT_HTTP_API, TT_IRC, TT_TELNET, TT_TWITTER, TT_SMS,
                           TT_USSD, TT_XMPP, TT_MXIT, TT_WECHAT])

    AT_IRC_NICKNAME = 'irc_nickname'
    AT_TWITTER_HANDLE = 'twitter_handle'
    AT_MSISDN = 'msisdn'
    AT_GTALK_ID = 'gtalk_id'
    AT_JABBER_ID = 'jabber_id'
    AT_MXIT_ID = 'mxit_id'
    AT_WECHAT_ID = 'wechat_id'
    ADDRESS_TYPES = set([
        AT_IRC_NICKNAME, AT_TWITTER_HANDLE, AT_MSISDN, AT_GTALK_ID,
        AT_JABBER_ID, AT_MXIT_ID, AT_WECHAT_ID])

    def process_fields(self, fields):
        fields = super(TransportUserMessage, self).process_fields(fields)
        fields.setdefault('message_id', self.generate_id())
        fields.setdefault('in_reply_to', None)
        fields.setdefault('provider', None)
        fields.setdefault('session_event', None)
{'fraction_non_alphanumeric': 0.08660361377752682, 'fraction_numerical': 0.001411631846414455, 'mean_word_length': 2.8201671609598273, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2830852', 'n_tokens_mistral': 4330, 'n_tokens_neox': 4171, 'n_words': 1071}