<commit_before># -*- coding: utf-8 -*-
# vim: set ts=4 et

from datetime import datetime
import re


message_re = re.compile(
  '^(?:'                            +
    ':(?P<prefix>'                  +
      '(?P<source>[^ !@]+)'         +
      '(?:'                         +
        '(?:!(?P<user>[^ @]+))?'    +
        '@(?P<host>[^ ]+)'          +
      ')?'                          +
    ') '                            +
  ')?'                              +
  '(?P<cmd>[^ :]+)'                 +
  '(?: (?P<params>.+))?$'
)


def parse_params(params):
    l = []
    while params:
        if params[0] == ':':
            l.append(params[1:])
            break
        if len(l) == 14:
            l.append(params)
            break
        param, _, params = params.partition(' ')
        l.append(param)
    return l

def parse_message(message):
    match = message_re.match(message)
    if match:
        d = match.groupdict()
        d['cmd'] = d['cmd'].upper()
        d['param'] = parse_params(d['params'])
        del d['params']
    else:
        d = {'prefix': None, 'source': None, 'user': None, 'host': None,
             'command': '', 'param': []}
    return d

class Message(object):
    def __init__(self, line, bot=None):
        self.bot = bot
        self.raw = line
        self.reply_to = None
        self.time = datetime.utcnow()
        self.channel = None
        self.trigger = None
        self.level = 0
        self.__dict__.update(parse_message(line))

        if self.cmd in ('PRIVMSG', 'NOTICE'):
            if self.param[0].startswith(('&', '#', '+', '!')):
                self.channel = self.param[0]
                self.reply_to = self.param[0]<commit_msg>make msg.channel always lowercase<commit_after># -*- coding: utf-8 -*-
# vim: set ts=4 et

from datetime import datetime
import re


message_re = re.compile(
  '^(?:'                            +
    ':(?P<prefix>'                  +
      '(?P<source>[^ !@]+)'         +
      '(?:'                         +
        '(?:!(?P<user>[^ @]+))?'    +
        '@(?P<host>[^ ]+)'          +
      ')?'                          +
    ') '                            +
  ')?'                              +
  '(?P<cmd>[^ :]+)'                 +
  '(?: (?P<params>.+))?$'
)


def parse_params(params):
    l = []
    while params:
        if params[0] == ':':
            l.append(params[1:])
            break
        if len(l) == 14:
            l.append(params)
            break
        param, _, params = params.partition(' ')
        l.append(param)
    return l

def parse_message(message):
    match = message_re.match(message)
    if match:
        d = match.groupdict()
        d['cmd'] = d['cmd'].upper()
        d['param'] = parse_params(d['params'])
        del d['params']
    else:
        d = {'prefix': None, 'source': None, 'user': None, 'host': None,
             'command': '', 'param': []}
    return d

class Message(object):
    def __init__(self, line, bot=None):
        self.bot = bot
        self.raw = line
        self.reply_to = None
        self.time = datetime.utcnow()
        self.channel = None
        self.trigger = None
        self.level = 0
        self.__dict__.update(parse_message(line))

        if self.cmd in ('PRIVMSG', 'NOTICE'):
            if self.param[0].startswith(('&', '#', '+', '!')):
                self.channel = self.param[0].lower()
                self.reply_to = self.param[0]
{'fraction_non_alphanumeric': 0.1834595224228305, 'fraction_numerical': 0.005824111822947, 'mean_word_length': 1.4156118143459915, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2435532', 'n_tokens_mistral': 1102, 'n_tokens_neox': 1063, 'n_words': 204}