<commit_before>import json
import re
import logging
import time

log = logging.getLogger('tyggbot')

try:
    from userdispatch import UserDispatch
    Dispatch = UserDispatch
    log.info('Using UserDispatch')
except:
    log.exception('No user dispatch available')
    from dispatch import Dispatch


def parse_action(raw_data=None, data=None):
    from tbactions import FuncAction, SayAction, MeAction, MultiAction, WhisperAction
    if not data:
        data = json.loads(raw_data)

    if data['type'] == 'say':
        action = SayAction(data['message'])
    elif data['type'] == 'me':
        action = MeAction(data['message'])
    elif data['type'] == 'whisper':
        action = WhisperAction(data['message'])
    elif data['type'] == 'func':
        try:
            action = FuncAction(getattr(Dispatch, data['cb']))
        except AttributeError as e:
            log.error('AttributeError caught when parsing action: {0}'.format(e))
            return None
    elif data['type'] == 'multi':
        action = MultiAction(data['args'], data['default'])
    else:
        raise Exception('Unknown action type: {0}'.format(data['type']))

    return action


class Command:
    def __init__(self, do_sync=True):
        self.id = -1
        self.extra_args = {'command': self}
        self.synced = True
        self.last_run = 0
        self.last_run_by_user = {}
        self.delay_all = 0
        self.delay_user = 0
        self.num_uses = 0
        self.level = 100
        self.enabled = 1
        self.do_sync = do_sync
        self.type = '?'
        self.cost = 0
        self.can_execute_with_whisper = False

    @classmethod
    def from_json(cls, json):
        cmd = cls()
        if 'level' in json:
            cmd.level = json['level']
        cmd.action = parse_action(data=json['action'])
        return cmd

    @classmethod
    def admin_command(cls, action, type='raw_func', level=1000):
        from tbactions import RawFuncAction, FuncAction
        cmd = cls(False)
        cmd.level = level
        cmd.can_execute_with_whisper = True
        if type == 'raw_func':
            cmd.action = RawFuncAction(action)
        elif type == 'func':
            cmd.action = FuncAction(action)
        else:
            log.error('Unknown admin command type: {0}'.format(type))
            cmd.action = False
        return cmd

    def load_from_db(self, data):
        self.id = data['id']
        self.level = data['level']
        self.action = parse_action(data['action'])
        self.delay_all = data['delay_all']
        self.delay_user = data['delay_user']
        try:
            self.enabled = data['enabled']
        except:
            self.enabled = True

        try:
            self.num_uses = data['num_uses']
        except:
            self.num_uses = 0

        try:
            self.cost = data['cost']
        except:
            self.cost = 0

        try:
            self.can_execute_with_whisper = data['can_execute_with_whisper']
        except:
            self.can_execute_with_whisper = False

        if data['extra_args']:
            try:
                self.extra_args.update(json.loads(data['extra_args']))
            except Exception as e:
                log.error('Exception caught while loading Filter extra arguments ({0}): {1}'.format(data['extra_args'], e))

    def load_args(self, level, action):
        self.level = level
        self.action = action

    def is_enabled(self):
        return self.enabled == 1 and self.action is not None

    def sync(self, cursor):
        if self.do_sync:
            cursor.execute('UPDATE `tb_commands` SET `num_uses`=%s WHERE `id`=%s', (self.num_uses, self.id))
            self.synced = True

    # (cur_time - self.last_run) = time since last run
    def run(self, tyggbot, source, message, event={}, args={}):
        cur_time = time.time()
        if cur_time - self.last_run > self.delay_all or source.level >= 2000:
            if source.username not in self.last_run_by_user or cur_time - self.last_run_by_user[source.username] > self.delay_user or source.level >= 2000:
                if self.cost > 0 and source.points < self.cost:
                    # User does not have enough points to use the command
                    return False

                args.update(self.extra_args)
                ret = self.action.run(tyggbot, source, message, event, args)
                self.num_uses += 1
                self.synced = False
                if ret is not False:
                    if self.cost > 0:
                        # Only spend points if the action did not fail
                        if not source.spend(self.cost):
                            # The user does not have enough points to spend!
                            return False
                    self.last_run = cur_time
                    self.last_run_by_user[source.username] = cur_time
            else:
                log.debug('{1} ran command {0:.2f} seconds ago, waiting...'.format(cur_time - self.last_run_by_user[source.username], source.username))
        else:
            log.debug('Command was run {0:.2f} seconds ago, waiting...'.format(cur_time - self.last_run))


class Filter:
    def __init__(self, data):
        self.id = data['id']
        self.name = data['name']
        self.action = parse_action(data['action'])
        self.filter = data['filter']
        self.type = data['type']
        try:
            self.regex = re.compile(data['filter'].lower())
        except Exception as e:
            log.exception('Uncaught exception in filter')
        self.source = data['source']
        self.num_uses = data['num_uses']
        self.enabled = data['enabled']
        self.regexstr = data['filter']
        self.extra_args = {'filter': self}
        if data['extra_args']:
            try:
                self.extra_args.update(json.loads(data['extra_args']))
            except Exception as e:
                log.error('Exception caught while loading Filter extra arguments ({0}): {1}'.format(data['extra_args'], e))

        self.synced = True

    def is_enabled(self):
        return self.enabled == 1 and self.action is not None

    def sync(self, cursor):
        cursor.execute('UPDATE `tb_filters` SET `num_uses`=%s WHERE `id`=%s', (self.num_uses, self.id))
        self.synced = True

    def match(self, source, message):
        if not self.source or self.source == source:
            return self.regex.match(message)

    def search(self, source, message):
        if not self.source or self.source == source.username:
            return self.regex.search(message)

        return None

    def run(self, tyggbot, source, message, event={}, args={}):
        args.update(self.extra_args)
        self.action.run(tyggbot, source, message, event, args)
        self.num_uses += 1
        self.synced = False
<commit_msg>Invalid regex filters no longer get added.<commit_after>import json
import re
import logging
import time

log = logging.getLogger('tyggbot')

try:
    from userdispatch import UserDispatch
    Dispatch = UserDispatch
    log.info('Using UserDispatch')
except:
    log.exception('No user dispatch available')
    from dispatch import Dispatch


def parse_action(raw_data=None, data=None):
    from tbactions import FuncAction, SayAction, MeAction, MultiAction, WhisperAction
    if not data:
        data = json.loads(raw_data)

    if data['type'] == 'say':
        action = SayAction(data['message'])
    elif data['type'] == 'me':
        action = MeAction(data['message'])
    elif data['type'] == 'whisper':
        action = WhisperAction(data['message'])
    elif data['type'] == 'func':
        try:
            action = FuncAction(getattr(Dispatch, data['cb']))
        except AttributeError as e:
            log.error('AttributeError caught when parsing action: {0}'.format(e))
            return None
    elif data['type'] == 'multi':
        action = MultiAction(data['args'], data['default'])
    else:
        raise Exception('Unknown action type: {0}'.format(data['type']))

    return action


class Command:
    def __init__(self, do_sync=True):
        self.id = -1
        self.extra_args = {'command': self}
        self.synced = True
        self.last_run = 0
        self.last_run_by_user = {}
        self.delay_all = 0
        self.delay_user = 0
        self.num_uses = 0
        self.level = 100
        self.enabled = 1
        self.do_sync = do_sync
        self.type = '?'
        self.cost = 0
        self.can_execute_with_whisper = False

    @classmethod
    def from_json(cls, json):
        cmd = cls()
        if 'level' in json:
            cmd.level = json['level']
        cmd.action = parse_action(data=json['action'])
        return cmd

    @classmethod
    def admin_command(cls, action, type='raw_func', level=1000):
        from tbactions import RawFuncAction, FuncAction
        cmd = cls(False)
        cmd.level = level
        cmd.can_execute_with_whisper = True
        if type == 'raw_func':
            cmd.action = RawFuncAction(action)
        elif type == 'func':
            cmd.action = FuncAction(action)
        else:
            log.error('Unknown admin command type: {0}'.format(type))
            cmd.action = False
        return cmd

    def load_from_db(self, data):
        self.id = data['id']
        self.level = data['level']
        self.action = parse_action(data['action'])
        self.delay_all = data['delay_all']
        self.delay_user = data['delay_user']
        try:
            self.enabled = data['enabled']
        except:
            self.enabled = True

        try:
            self.num_uses = data['num_uses']
        except:
            self.num_uses = 0

        try:
            self.cost = data['cost']
        except:
            self.cost = 0

        try:
            self.can_execute_with_whisper = data['can_execute_with_whisper']
        except:
            self.can_execute_with_whisper = False

        if data['extra_args']:
            try:
                self.extra_args.update(json.loads(data['extra_args']))
            except Exception as e:
                log.error('Exception caught while loading Filter extra arguments ({0}): {1}'.format(data['extra_args'], e))

    def load_args(self, level, action):
        self.level = level
        self.action = action

    def is_enabled(self):
        return self.enabled == 1 and self.action is not None

    def sync(self, cursor):
        if self.do_sync:
            cursor.execute('UPDATE `tb_commands` SET `num_uses`=%s WHERE `id`=%s', (self.num_uses, self.id))
            self.synced = True

    # (cur_time - self.last_run) = time since last run
    def run(self, tyggbot, source, message, event={}, args={}):
        cur_time = time.time()
        if cur_time - self.last_run > self.delay_all or source.level >= 2000:
            if source.username not in self.last_run_by_user or cur_time - self.last_run_by_user[source.username] > self.delay_user or source.level >= 2000:
                if self.cost > 0 and source.points < self.cost:
                    # User does not have enough points to use the command
                    return False

                args.update(self.extra_args)
                ret = self.action.run(tyggbot, source, message, event, args)
                self.num_uses += 1
                self.synced = False
                if ret is not False:
                    if self.cost > 0:
                        # Only spend points if the action did not fail
                        if not source.spend(self.cost):
                            # The user does not have enough points to spend!
                            return False
                    self.last_run = cur_time
                    self.last_run_by_user[source.username] = cur_time
            else:
                log.debug('{1} ran command {0:.2f} seconds ago, waiting...'.format(cur_time - self.last_run_by_user[source.username], source.username))
        else:
            log.debug('Command was run {0:.2f} seconds ago, waiting...'.format(cur_time - self.last_run))


class Filter:
    def __init__(self, data):
        self.id = data['id']
        self.name = data['name']
        self.action = parse_action(data['action'])
        self.filter = data['filter']
        self.type = data['type']
        self.source = data['source']
        self.num_uses = data['num_uses']
        self.enabled = data['enabled']
        self.regexstr = data['filter']
        self.extra_args = {'filter': self}
        if data['extra_args']:
            try:
                self.extra_args.update(json.loads(data['extra_args']))
            except Exception as e:
                log.error('Exception caught while loading Filter extra arguments ({0}): {1}'.format(data['extra_args'], e))

        self.synced = True

        try:
            self.regex = re.compile(data['filter'].lower())
        except Exception as e:
            log.exception('Uncaught exception in filter {0}'.format(self.name))
            self.enabled = False

    def is_enabled(self):
        return self.enabled == 1 and self.action is not None

    def sync(self, cursor):
        cursor.execute('UPDATE `tb_filters` SET `num_uses`=%s WHERE `id`=%s', (self.num_uses, self.id))
        self.synced = True

    def match(self, source, message):
        if not self.source or self.source == source:
            return self.regex.match(message)

    def search(self, source, message):
        if not self.source or self.source == source.username:
            return self.regex.search(message)

        return None

    def run(self, tyggbot, source, message, event={}, args={}):
        args.update(self.extra_args)
        self.action.run(tyggbot, source, message, event, args)
        self.num_uses += 1
        self.synced = False

{'fraction_non_alphanumeric': 0.11484188436211014, 'fraction_numerical': 0.006150951588392793, 'mean_word_length': 2.0360281195079084, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1908666', 'n_tokens_mistral': 4062, 'n_tokens_neox': 3918, 'n_words': 1019}