<commit_before>import re, random, os, importlib

# This module was developed by squinting directly at both the MCP spec
# at http://www.moo.mud.org/mcp2/mcp2.html and tkMOO-light's plugins/mcp21.tcl
# file, to which this code bears more than a little resemblance and owes
# more than a little debt.

OOB_PREFIX   = re.compile('^#\$#')
QUOTE_PREFIX = re.compile('^#\$"')

#    my $simpleChars = q|[-a-z0-9~`!@#$%^&*()=+{}[\]\|';?/><.,]|;
#    while ($raw =~ /([-_*a-z0-9]+)              # keyword
#                        :                       # followed by colon
#                        \s+                     # some space
#                        (                       # and either
#                            (?:"[^"]*")         # a quoted string - TODO - the grammar is more picky than [^"]
#                            |                   # or
#                            (?:$simpleChars)+   # a value
#                        )/igx)
raw_re = re.compile(r'([-_*a-z0-9]+):\s+((?:"[^"]*")|(?:[-a-z0-9~`!@#$%^&*()=+{}[\]\|\';?/><.,])+)')

def _version_cmp(v1, v2):
    v1_maj, v1_min = re.split('\.', v1)
    v2_maj, v2_min = re.split('\.', v2)

    return (v1_maj > v2_maj or (v1_maj == v2_maj and v1_min >= v2_min));

class MCPCore:
    def __init__(self, conn):
        self.multiline_messages = {}
        self.mcp_active = 0
        self.mcp_auth_key = ''
        self.connection = conn

        self.msg_registry = {}
        self.packages = {}

        MCP(self) # initialize the 'mcp' core package

        # walk the packages directory, and instantiate everything we find there.
        # this relies on each package having a class called "MCPPackage" that's a
        # sane and correct subclass of MCPPackageBase.
        for package_file in os.listdir("./mcp21/package"):
            package, ext = package_file.split('.')

            if package == '__init__' or ext != 'py' : continue

            # do the actual importing
            mod = importlib.import_module('mcp21.package.' + package)

            # then go find the thing called "MCPPackage" (the subclass constructor) and call it
            getattr(mod, 'MCPPackage')(self)

    def debug(self, info):
        info = re.sub('\n$', '', info)
        # DebugMCP window monkey-patches this when it shows itself
        # TODO - we might want one debug window per-connection-window
        print(self.connection.world.get('name') + ": " + info)

    def output_filter(self, data):
        # MCP spec, 2.1:
        # A received network line that begins with the characters #$# is translated
        # to an out-of-band (MCP message) line consisting of exactly the same
        # characters.  A received network line that begins with the characters #$"
        # must be translated to an in-band line consisting of the network line with
        # the prefix #$" removed.  Any other received network line translates to an
        # in-band line consisting of exactly the same characters.

        data, matches = QUOTE_PREFIX.subn('', data)  # did we have the quote prefix?
        if matches > 0: return data                  # we did, and removed it.  Return the line

        data, matches = OOB_PREFIX.subn('', data) # did we have the oob prefix?
        if matches == 0: return data              # we did not, so return the line and bail

        # now we have only lines that started with OOB_PREFIX, which has been trimmed
        self.debug("S->C: #$#" + data)

        m = re.match(r'(\S+)\s*(.*)', data)

        message_name, rest = m.group(1,2)

        # if we haven't started yet, and the message isn't a startup negotiation...
        if (not self.mcp_active and message_name != 'mcp'): return

        # multiline message handling.  This is awful.
        if message_name == '*':
            m = re.match(r'^(\S*) ([^:]*): (.*)', rest)
            tag, field, value = m.group(1,2,3)

            message = self.multiline_messages[tag]
            message._data_tag = tag

            if not field in message.data: message.data[field] = []

            message.data[field].append(value)

        elif message_name == ':':
            m = re.match(r'^(\S+)', rest)
            tag = m.group(1)
            message = self.multiline_messages[tag]
            message.multi_in_progress = False
        else:
            message = self.parse(rest)

        # check auth
        if (message_name != '*') and (message_name != 'mcp') and (message.auth_key != self.mcp_auth_key):
            self.debug("mcp - auth failed")
            return

        message.message = message.message or message_name

        if message.multi_in_progress:
            if not self.multiline_messages.has_key(message._data_tag):
                self.multiline_messages[message._data_tag] = message
        else:
            # don't dispatch multilines in progress
            self.dispatch(message)

        # return void/falsy/None so the output widget skips this line
        return

    def parse(self, raw):

        if not raw: return

        first = re.split('\s+', raw)[0]

        message = Message()

        if not re.search(r':$', first):
            message.auth_key = first
            first_re = '^' + re.escape(first) + '\s+'
            raw = re.sub(first_re, '', raw)

        keyvals = re.findall(raw_re, raw)

        for keyval in keyvals:
            keyword, value = keyval
            if re.search('\*$', keyword):
                message.data[keyword] = []
                message.multi_in_progress = True
            elif keyword == '_data-tag':
                message._data_tag = value
            else:
                # if we have a double-quoted string, strip the quotes
                if value.startswith('"') and value.endswith('"'):
                    value = value[1:-1]
                message.data[keyword] = value

        return message

    def dispatch(self, message):
        package = self.msg_registry.get(message.message)
        if not package: return

        if package.activated: package.dispatch(message)<commit_msg>Fix multililne-MCP crash if empty payload<commit_after>import re, random, os, importlib

# This module was developed by squinting directly at both the MCP spec
# at http://www.moo.mud.org/mcp2/mcp2.html and tkMOO-light's plugins/mcp21.tcl
# file, to which this code bears more than a little resemblance and owes
# more than a little debt.

OOB_PREFIX   = re.compile('^#\$#')
QUOTE_PREFIX = re.compile('^#\$"')

#    my $simpleChars = q|[-a-z0-9~`!@#$%^&*()=+{}[\]\|';?/><.,]|;
#    while ($raw =~ /([-_*a-z0-9]+)              # keyword
#                        :                       # followed by colon
#                        \s+                     # some space
#                        (                       # and either
#                            (?:"[^"]*")         # a quoted string - TODO - the grammar is more picky than [^"]
#                            |                   # or
#                            (?:$simpleChars)+   # a value
#                        )/igx)
raw_re = re.compile(r'([-_*a-z0-9]+):\s+((?:"[^"]*")|(?:[-a-z0-9~`!@#$%^&*()=+{}[\]\|\';?/><.,])+)')

def _version_cmp(v1, v2):
    v1_maj, v1_min = re.split('\.', v1)
    v2_maj, v2_min = re.split('\.', v2)

    return (v1_maj > v2_maj or (v1_maj == v2_maj and v1_min >= v2_min));

class MCPCore:
    def __init__(self, conn):
        self.multiline_messages = {}
        self.mcp_active = 0
        self.mcp_auth_key = ''
        self.connection = conn

        self.msg_registry = {}
        self.packages = {}

        MCP(self) # initialize the 'mcp' core package

        # walk the packages directory, and instantiate everything we find there.
        # this relies on each package having a class called "MCPPackage" that's a
        # sane and correct subclass of MCPPackageBase.
        for package_file in os.listdir("./mcp21/package"):
            package, ext = package_file.split('.')

            if package == '__init__' or ext != 'py' : continue

            # do the actual importing
            mod = importlib.import_module('mcp21.package.' + package)

            # then go find the thing called "MCPPackage" (the subclass constructor) and call it
            getattr(mod, 'MCPPackage')(self)

    def debug(self, info):
        info = re.sub('\n$', '', info)
        # DebugMCP window monkey-patches this when it shows itself
        # TODO - we might want one debug window per-connection-window
        print(self.connection.world.get('name') + ": " + info)

    def output_filter(self, data):
        # MCP spec, 2.1:
        # A received network line that begins with the characters #$# is translated
        # to an out-of-band (MCP message) line consisting of exactly the same
        # characters.  A received network line that begins with the characters #$"
        # must be translated to an in-band line consisting of the network line with
        # the prefix #$" removed.  Any other received network line translates to an
        # in-band line consisting of exactly the same characters.

        data, matches = QUOTE_PREFIX.subn('', data)  # did we have the quote prefix?
        if matches > 0: return data                  # we did, and removed it.  Return the line

        data, matches = OOB_PREFIX.subn('', data) # did we have the oob prefix?
        if matches == 0: return data              # we did not, so return the line and bail

        # now we have only lines that started with OOB_PREFIX, which has been trimmed
        self.debug("S->C: #$#" + data)

        m = re.match(r'(\S+)\s*(.*)', data)

        message_name, rest = m.group(1,2)

        # if we haven't started yet, and the message isn't a startup negotiation...
        if (not self.mcp_active and message_name != 'mcp'): return

        # multiline message handling.  This is awful.
        if message_name == '*':
            m = re.match(r'^(\S*) ([^:]*): (.*)', rest)
            tag, field, value = m.group(1,2,3)

            message = self.multiline_messages[tag]
            message._data_tag = tag

            if not field in message.data: message.data[field] = []

            message.data[field].append(value)

        elif message_name == ':':
            m = re.match(r'^(\S+)', rest)
            tag = m.group(1)
            message = self.multiline_messages[tag]
            message.multi_in_progress = False
        else:
            message = self.parse(rest)

        # check auth
        if (message_name != '*') and (message_name != 'mcp') and (message.auth_key != self.mcp_auth_key):
            self.debug("mcp - auth failed")
            return

        message.message = message.message or message_name

        if message.multi_in_progress:
            if not self.multiline_messages.has_key(message._data_tag):
                self.multiline_messages[message._data_tag] = message
        else:
            # don't dispatch multilines in progress
            self.dispatch(message)

        # return void/falsy/None so the output widget skips this line
        return

    def parse(self, raw):

        if not raw: return

        first = re.split('\s+', raw)[0]

        message = Message()

        if not re.search(r':$', first):
            message.auth_key = first
            first_re = '^' + re.escape(first) + '\s+'
            raw = re.sub(first_re, '', raw)

        keyvals = re.findall(raw_re, raw)

        for keyval in keyvals:
            keyword, value = keyval
            m = re.match(r'^(.+)(\*)', keyword)
            if m:
                keyword, splat = m.group(1,2)

                message.data[keyword] = []
                message.multi_in_progress = True
            elif keyword == '_data-tag':
                message._data_tag = value
            else:
                # if we have a double-quoted string, strip the quotes
                if value.startswith('"') and value.endswith('"'):
                    value = value[1:-1]
                message.data[keyword] = value

        return message

    def dispatch(self, message):
        package = self.msg_registry.get(message.message)
        if not package: return

        if package.activated: package.dispatch(message)
{'fraction_non_alphanumeric': 0.12655107239707453, 'fraction_numerical': 0.007395841893335525, 'mean_word_length': 1.8534583821805393, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '174131', 'n_tokens_mistral': 3748, 'n_tokens_neox': 3523, 'n_words': 1047}