<commit_before>from .util.decorators import command, initializer, require_admin
import itertools
import re
import codecs
import traceback
from itertools import zip_longest
import string

string.ascii = [chr(i) for i in range(128)]

SUB_REGEX = re.compile(r"^(?:(\S+)[:,]\s)?(?:s|(.+?)/s)/((?:\\/|[^/])+)\/((?:\\/|[^/])*?)/([gixs]{0,4})?")
TRANS_REGEX = re.compile(r"^(?:(\S+)[:,]\s)?(?:y|(.+?)/y)/((?:\\/|[^/])+)\/((?:\\/|[^/])*?)/([cds]+)?")

def sed(bot, nick, chan, msg):
    if not SUB_REGEX.match(msg) and not TRANS_REGEX.match(msg):
        return # this isn't a sed message

    if SUB_REGEX.match(msg):
        do_sub(bot, nick, chan, msg)
    elif TRANS_REGEX.match(msg):
        tr(bot, nick, chan, msg)


def squeeze(lst, src):
    for ch in lst:
        src = re.sub(ch + r"{2,}", ch, src)
    return src

def sugar(src):
    return src.replace("\\/", "/") \
            .replace("[:upper:]", string.ascii_uppercase) \
            .replace("[:lower:]", string.ascii_lowercase) \
            .replace("[:alpha:]", string.ascii_letters) \
            .replace("[:digit:]", string.digits) \
            .replace("[:xdigit:]", string.hexdigits) \
            .replace("[:alnum:]", string.digits + string.ascii_letters) \
            .replace("[:blank:]", string.whitespace) \
            .replace("[:punct:]", string.punctuation) \
            .replace("[:cntrl:]", "".join([i for i in string.ascii if not i in string.printable])) \
            .replace("[:print:]", string.printable)

def expand(src):
    out = []
    escaped = False
    hyphen = False

    for char in src:
        if char == "\\":
            escaped = True
            continue
        elif char == '-' and not escaped:
            hyphen = True
            escaped = False
            continue
        elif hyphen:
            out.extend(range(out[-1] + 1, ord(char)))
            hyphen = False
        out.append(ord(char))
    return "".join([chr(i) for i in out])


def tr(bot, nick, chan, msg):
    target, qual, lhs, rhs, flags = TRANS_REGEX.match(msg).groups()


    # if there is no qualifier, match the last message from this chan using .+
    target = target or nick

    lhs = expand(sugar(lhs))
    rhs = expand(sugar(rhs))

    qual = qual or '|'.join(list(lhs))

    trans_table = {ord(i[0]):i[1] for i in itertools.zip_longest(lhs, rhs, fillvalue=rhs[-1])}

    msg = get_tr_message(bot, qual, target.lower(), chan)

    msg = msg.translate(trans_table)
    if '\x01' in msg:
        msg = msg.split(" ", 1)[1][:-1]
        return bot.msg(chan, "* %s %s" % (target, msg))
    return bot.msg(chan, "<%s> %s" % (target, msg))



class Substitution(object):
    """ A class representing a substitution in sed. """
    def __init__(self, pattern):
        self.pattern = pattern
        self.target, self.qual, self.re, self.sub, self.flags, self.count = self.parse(pattern)

    def __repr__(self):
        return "Substitution(%r)" % (self.pattern)

    def parse(self, pattern):
        m = SUB_REGEX.match(pattern)
        if not m:
            raise TypeError("Pattern %r isn't a proper substitution pattern!" % (pattern))
        count = 1

        groups = m.groups()
        if 'g' in groups[4]:
            count = 0

        flags = 0
        for c in groups[4]: # for character in flags
            if c != 'g':
                flags += getattr(re, c.upper())

        sub = groups[3].replace("\\/", "/")
        sub = re.sub(r"(?<!\\)(\\)(?=\d+|g<\w+>)", r"\\\\", sub)
        sub = codecs.escape_decode(bytes(sub, "utf-8"))[0].decode('utf-8')

        return list(groups[:-2]) + [sub, flags, count]

    def do(self, on):
        return re.sub(self.re, self.sub, on, count=self.count, flags=self.flags)


def do_sub(bot, nick, chan, msg):
    sub = Substitution(msg)

    if sub.target:
        nick = sub.target

    message = get_sub_message(bot, sub, nick.lower(), chan)

    if not message:
        print(nick.lower(), str(sub))
        return

    new_msg = sub.do(message.msg)
    if message.is_action:
        new_msg = new_msg.split(" ", 1)[1][:-1]
        return bot.msg(chan, "* %s %s" % (nick, new_msg))

    return bot.msg(chan, "<%s> %s" % (nick, new_msg))

def get_tr_message(bot, qual, nick, chan):
    if not nick in bot.state.messages:<commit_msg>Add sed-enabled check<commit_after>from .util.decorators import command, initializer, require_admin
import itertools
import re
import codecs
import traceback
from itertools import zip_longest
import string

string.ascii = [chr(i) for i in range(128)]

SUB_REGEX = re.compile(r"^(?:(\S+)[:,]\s)?(?:s|(.+?)/s)/((?:\\/|[^/])+)\/((?:\\/|[^/])*?)/([gixs]{0,4})?")
TRANS_REGEX = re.compile(r"^(?:(\S+)[:,]\s)?(?:y|(.+?)/y)/((?:\\/|[^/])+)\/((?:\\/|[^/])*?)/([cds]+)?")

def sed(bot, nick, chan, msg):
    if not SUB_REGEX.match(msg) and not TRANS_REGEX.match(msg):
        return # this isn't a sed message

    if SUB_REGEX.match(msg):
        do_sub(bot, nick, chan, msg)
    elif TRANS_REGEX.match(msg):
        tr(bot, nick, chan, msg)


def squeeze(lst, src):
    for ch in lst:
        src = re.sub(ch + r"{2,}", ch, src)
    return src

def sugar(src):
    return src.replace("\\/", "/") \
            .replace("[:upper:]", string.ascii_uppercase) \
            .replace("[:lower:]", string.ascii_lowercase) \
            .replace("[:alpha:]", string.ascii_letters) \
            .replace("[:digit:]", string.digits) \
            .replace("[:xdigit:]", string.hexdigits) \
            .replace("[:alnum:]", string.digits + string.ascii_letters) \
            .replace("[:blank:]", string.whitespace) \
            .replace("[:punct:]", string.punctuation) \
            .replace("[:cntrl:]", "".join([i for i in string.ascii if not i in string.printable])) \
            .replace("[:print:]", string.printable)

def expand(src):
    out = []
    escaped = False
    hyphen = False

    for char in src:
        if char == "\\":
            escaped = True
            continue
        elif char == '-' and not escaped:
            hyphen = True
            escaped = False
            continue
        elif hyphen:
            out.extend(range(out[-1] + 1, ord(char)))
            hyphen = False
        out.append(ord(char))
    return "".join([chr(i) for i in out])


def tr(bot, nick, chan, msg):
    target, qual, lhs, rhs, flags = TRANS_REGEX.match(msg).groups()


    # if there is no qualifier, match the last message from this chan using .+
    target = target or nick

    lhs = expand(sugar(lhs))
    rhs = expand(sugar(rhs))

    qual = qual or '|'.join(list(lhs))

    trans_table = {ord(i[0]):i[1] for i in itertools.zip_longest(lhs, rhs, fillvalue=rhs[-1])}

    msg = get_tr_message(bot, qual, target.lower(), chan)

    msg = msg.translate(trans_table)
    if '\x01' in msg:
        msg = msg.split(" ", 1)[1][:-1]
        return bot.msg(chan, "* %s %s" % (target, msg))
    return bot.msg(chan, "<%s> %s" % (target, msg))



class Substitution(object):
    """ A class representing a substitution in sed. """
    def __init__(self, pattern):
        self.pattern = pattern
        self.target, self.qual, self.re, self.sub, self.flags, self.count = self.parse(pattern)

    def __repr__(self):
        return "Substitution(%r)" % (self.pattern)

    def parse(self, pattern):
        m = SUB_REGEX.match(pattern)
        if not m:
            raise TypeError("Pattern %r isn't a proper substitution pattern!" % (pattern))
        count = 1

        groups = m.groups()
        if 'g' in groups[4]:
            count = 0

        flags = 0
        for c in groups[4]: # for character in flags
            if c != 'g':
                flags += getattr(re, c.upper())

        sub = groups[3].replace("\\/", "/")
        sub = re.sub(r"(?<!\\)(\\)(?=\d+|g<\w+>)", r"\\\\", sub)
        sub = codecs.escape_decode(bytes(sub, "utf-8"))[0].decode('utf-8')

        return list(groups[:-2]) + [sub, flags, count]

    def do(self, on):
        return re.sub(self.re, self.sub, on, count=self.count, flags=self.flags)


def do_sub(bot, nick, chan, msg):
    if !(bot.getconfig()["sed-enabled"]):
        return
    sub = Substitution(msg)

    if sub.target:
        nick = sub.target

    message = get_sub_message(bot, sub, nick.lower(), chan)

    if not message:
        print(nick.lower(), str(sub))
        return

    new_msg = sub.do(message.msg)
    if message.is_action:
        new_msg = new_msg.split(" ", 1)[1][:-1]
        return bot.msg(chan, "* %s %s" % (nick, new_msg))

    return bot.msg(chan, "<%s> %s" % (nick, new_msg))

def get_tr_message(bot, qual, nick, chan):
    if not nick in bot.state.messages:
{'fraction_non_alphanumeric': 0.18819016622108567, 'fraction_numerical': 0.006741834243868418, 'mean_word_length': 2.3940828402366865, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '567515', 'n_tokens_mistral': 3079, 'n_tokens_neox': 2937, 'n_words': 724}