<commit_before>from collections import namedtuple
import sys
import traceback

from logconfig import newLogger

log = newLogger(__name__)


# Normal delimiter; separates tokens in messages.
TOKEN_DELIM  = " "

# Used to indicate the start of an unsafe string; must be the first character
# after TOKEN_DELIM.
START_STRING = "|"


# Mapping from command word to Message (sub)classes.
messagesByCommand = {}


def defineMessageType(commandWord, argNamesAndSpecs):
    """
    Define a new message type.

    argNamesAndSpecs should be a list of tuples (name, spec), where name is the
    name of that argument and spec is an ArgumentSpecification object
    describing how it is encoded and decoded when the message is serialized and
    deserialized.
    """

    if commandWord in messagesByCommand:
        raise ValueError("Message command {0!r} is already taken."
                         .format(commandWord))
    assert not any(nameSpec[1].unsafe for nameSpec in argNamesAndSpecs[:-1])

    # TODO: snake_case to BigCamelCase?
    # Ideally we'd choose this name so that it matches the actual message class
    # name. Or just override __str__ in Message.
    NamedTupleType = namedtuple(commandWord + "_message_tuple",
                                [nameSpec[0] for nameSpec in argNamesAndSpecs])

    # Subclass from Message before NamedTupleType, so that we can override some
    # methods of NamedTupleType in Message. (We may want to do this with
    # __str__?)
    class NewMessageType(Message, NamedTupleType):
        command  = commandWord
        argSpecs = [nameSpec[1] for nameSpec in argNamesAndSpecs]

        def __init__(self, *args):
            super(NewMessageType, self).__init__(*args)

    messagesByCommand[commandWord] = NewMessageType
    return NewMessageType

def serializeMessage(message):
    argStrings = []
    assert len(message.argSpecs) == len(message)
    for argSpec, arg in zip(message.argSpecs, message):
        argWords = argSpec.encode(arg)
        if argSpec.count == 1:
            argWords = (argWords,)
        argStrings.extend(argWords)
    lastIsUnsafe = message.argSpecs[-1].unsafe if message.argSpecs else False
    return buildMessage(message.command, argStrings, lastIsUnsafe=lastIsUnsafe)

# Note: errorOnFail might never be passed as False; currently all callers that
# don't want to crash still pass errorOnFail=True and just handle
# InvalidMessageError themselves.
def deserializeMessage(data, errorOnFail=True):
    try:
        cmd, argStrings = tokenize(data)
        if cmd not in messagesByCommand:
            raise InvalidMessageError(data, "Unrecognized message command.")

        messageType = messagesByCommand[cmd]

        args = []
        for argSpec in messageType.argSpecs:
            if argSpec.count > len(argStrings):
                raise InvalidMessageError(data,
                                          "Not enough arguments for command.")
            if argSpec.count == 1:
                args.append(argSpec.decode(argStrings[0]))
            else:
                currWords = tuple(argStrings[:argSpec.count])
                args.append(argSpec.decode(currWords))
            argStrings = argStrings[argSpec.count:]

        if len(argStrings) > 0:
            raise InvalidMessageError(data, "Too many arguments for command.")

        return messageType(*args)
    except StandardError, exc:
        # Log the full traceback, noting where we are, much like what Twisted
        # does for an uncaught exception.
        stackBelow = traceback.format_exception(sys.exc_type, sys.exc_value,
                                                sys.exc_traceback)
        stackAbove = traceback.format_stack()
        # Remove the last stack entry from stackAbove, because that's the call
        # to format_stack() which isn't part of the exception's traceback.
        stackAbove = stackAbove[:-1]
        # Remove the first entry from stackBelow, because that's the "Traceback
        # (most recent call last)" line which we want at the top of the
        # traceback, rather than the middle.
        headerLine = stackBelow[0]
        stackBelow = stackBelow[1:]
        # Add an extra entry in the middle noting that this is where we caught
        # the exception.
        midLine = "--- <exception caught here> ---\n"
        fullStack = [headerLine] + stackAbove + [midLine] + stackBelow
        log.debug("Caught exception in deserializeMessage.\n" +
                  "".join(fullStack))

        if errorOnFail:
            # Reraise the exception, but converted (if necessary) to an
            # InvalidMessageError. This ensures that it'll be handled correctly
            # by the caller and not cause the program to crash unnecessarily.
            if isinstance(exc, InvalidMessageError):
                raise exc
            else:
                raise InvalidMessageError(data, str(exc))
        else:
            return None

class Message(object):
    command  = None
    argSpecs = None

    # Note: this doesn't seem to be necessary, but that might just be because
    # (I think) namedtuple overrides __new__ instead of __init__.
    # def __init__(self, *args):
    #     super(Message, self).__init__(*args)

    def serialize(self):
        return serializeMessage(self)

class ArgumentSpecification:
    """
    Class used to describe how one logical argument to a message is encoded and
    decoded. Note that a single logical argument might be encoded as several
    (space-separated) words in the actual message string -- for example, a
    position is encoded as two words, one for each coordinate.
    """

    def __init__(self, numWords, decodeFunc, encodeFunc=None, unsafe=False):
        """<commit_msg>str(message) now returns message.serialize().<commit_after>from collections import namedtuple
import sys
import traceback

from logconfig import newLogger

log = newLogger(__name__)


# Normal delimiter; separates tokens in messages.
TOKEN_DELIM  = " "

# Used to indicate the start of an unsafe string; must be the first character
# after TOKEN_DELIM.
START_STRING = "|"


# Mapping from command word to Message (sub)classes.
messagesByCommand = {}


def defineMessageType(commandWord, argNamesAndSpecs):
    """
    Define a new message type.

    argNamesAndSpecs should be a list of tuples (name, spec), where name is the
    name of that argument and spec is an ArgumentSpecification object
    describing how it is encoded and decoded when the message is serialized and
    deserialized.
    """

    if commandWord in messagesByCommand:
        raise ValueError("Message command {0!r} is already taken."
                         .format(commandWord))
    assert not any(nameSpec[1].unsafe for nameSpec in argNamesAndSpecs[:-1])

    # TODO: snake_case to BigCamelCase?
    # Ideally we'd choose this name so that it matches the actual message class
    # name. Or just override __str__ in Message.
    NamedTupleType = namedtuple(commandWord + "_message_tuple",
                                [nameSpec[0] for nameSpec in argNamesAndSpecs])

    # Subclass from Message before NamedTupleType, so that we can override some
    # methods of NamedTupleType in Message. (We may want to do this with
    # __str__?)
    class NewMessageType(Message, NamedTupleType):
        command  = commandWord
        argSpecs = [nameSpec[1] for nameSpec in argNamesAndSpecs]

        def __init__(self, *args):
            super(NewMessageType, self).__init__(*args)

    messagesByCommand[commandWord] = NewMessageType
    return NewMessageType

def serializeMessage(message):
    argStrings = []
    assert len(message.argSpecs) == len(message)
    for argSpec, arg in zip(message.argSpecs, message):
        argWords = argSpec.encode(arg)
        if argSpec.count == 1:
            argWords = (argWords,)
        argStrings.extend(argWords)
    lastIsUnsafe = message.argSpecs[-1].unsafe if message.argSpecs else False
    return buildMessage(message.command, argStrings, lastIsUnsafe=lastIsUnsafe)

# Note: errorOnFail might never be passed as False; currently all callers that
# don't want to crash still pass errorOnFail=True and just handle
# InvalidMessageError themselves.
def deserializeMessage(data, errorOnFail=True):
    try:
        cmd, argStrings = tokenize(data)
        if cmd not in messagesByCommand:
            raise InvalidMessageError(data, "Unrecognized message command.")

        messageType = messagesByCommand[cmd]

        args = []
        for argSpec in messageType.argSpecs:
            if argSpec.count > len(argStrings):
                raise InvalidMessageError(data,
                                          "Not enough arguments for command.")
            if argSpec.count == 1:
                args.append(argSpec.decode(argStrings[0]))
            else:
                currWords = tuple(argStrings[:argSpec.count])
                args.append(argSpec.decode(currWords))
            argStrings = argStrings[argSpec.count:]

        if len(argStrings) > 0:
            raise InvalidMessageError(data, "Too many arguments for command.")

        return messageType(*args)
    except StandardError, exc:
        # Log the full traceback, noting where we are, much like what Twisted
        # does for an uncaught exception.
        stackBelow = traceback.format_exception(sys.exc_type, sys.exc_value,
                                                sys.exc_traceback)
        stackAbove = traceback.format_stack()
        # Remove the last stack entry from stackAbove, because that's the call
        # to format_stack() which isn't part of the exception's traceback.
        stackAbove = stackAbove[:-1]
        # Remove the first entry from stackBelow, because that's the "Traceback
        # (most recent call last)" line which we want at the top of the
        # traceback, rather than the middle.
        headerLine = stackBelow[0]
        stackBelow = stackBelow[1:]
        # Add an extra entry in the middle noting that this is where we caught
        # the exception.
        midLine = "--- <exception caught here> ---\n"
        fullStack = [headerLine] + stackAbove + [midLine] + stackBelow
        log.debug("Caught exception in deserializeMessage.\n" +
                  "".join(fullStack))

        if errorOnFail:
            # Reraise the exception, but converted (if necessary) to an
            # InvalidMessageError. This ensures that it'll be handled correctly
            # by the caller and not cause the program to crash unnecessarily.
            if isinstance(exc, InvalidMessageError):
                raise exc
            else:
                raise InvalidMessageError(data, str(exc))
        else:
            return None

class Message(object):
    command  = None
    argSpecs = None

    # Note: this doesn't seem to be necessary, but that might just be because
    # (I think) namedtuple overrides __new__ instead of __init__.
    # def __init__(self, *args):
    #     super(Message, self).__init__(*args)

    # For a message that was just deserialized, maybe we should cache the
    # original string and return it, rather than regenerating it?
    def serialize(self):
        return serializeMessage(self)

    def __str__(self):
        return self.serialize()

class ArgumentSpecification:
    """
    Class used to describe how one logical argument to a message is encoded and
    decoded. Note that a single logical argument might be encoded as several
    (space-separated) words in the actual message string -- for example, a
    position is encoded as two words, one for each coordinate.
    """

    def __init__(self, numWords, decodeFunc, encodeFunc=None, unsafe=False):
        """
{'fraction_non_alphanumeric': 0.07328544552762101, 'fraction_numerical': 0.002223362408072516, 'mean_word_length': 2.626356589147287, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1993064', 'n_tokens_mistral': 3119, 'n_tokens_neox': 2981, 'n_words': 1106}