<commit_before>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;
using Discord;
using Discord.Net;
using Discord.WebSocket;
using Mitternacht.Common.Collections;
using Mitternacht.Common.ModuleBehaviors;
using Mitternacht.Extensions;
using Mitternacht.Services;
using Mitternacht.Services.Database.Models;
using NLog;

namespace Mitternacht.Modules.Permissions.Services
{
    public class FilterService : IEarlyBlocker, INService
    {
        private readonly Logger _log;

        public ConcurrentHashSet<ulong> InviteFilteringChannels { get; }
        public ConcurrentHashSet<ulong> InviteFilteringServers { get; }

        //serverid, filteredwords
        public ConcurrentDictionary<ulong, ConcurrentHashSet<string>> ServerFilteredWords { get; }

        public ConcurrentHashSet<ulong> WordFilteringChannels { get; }
        public ConcurrentHashSet<ulong> WordFilteringServers { get; }

        public ConcurrentHashSet<ulong> ZalgoFilteringServers { get; }
        public ConcurrentHashSet<ulong> ZalgoFilteringChannels { get; }

        public ConcurrentHashSet<string> FilteredWordsForChannel(ulong channelId, ulong guildId)
        {
            var words = new ConcurrentHashSet<string>();
            if (WordFilteringChannels.Contains(channelId))
                ServerFilteredWords.TryGetValue(guildId, out words);
            return words;
        }

        public ConcurrentHashSet<string> FilteredWordsForServer(ulong guildId)
        {
            var words = new ConcurrentHashSet<string>();
            if (WordFilteringServers.Contains(guildId))
                ServerFilteredWords.TryGetValue(guildId, out words);
            return words;
        }

        public FilterService(DiscordSocketClient client, IEnumerable<GuildConfig> igcs)
        {
            _log = LogManager.GetCurrentClassLogger();
            var gcs = igcs.ToList();
            
            InviteFilteringServers = new ConcurrentHashSet<ulong>(gcs.Where(gc => gc.FilterInvites).Select(gc => gc.GuildId));
            InviteFilteringChannels = new ConcurrentHashSet<ulong>(gcs.SelectMany(gc => gc.FilterInvitesChannelIds.Select(fci => fci.ChannelId)));

            var dict = gcs.ToDictionary(gc => gc.GuildId, gc => new ConcurrentHashSet<string>(gc.FilteredWords.Select(fw => fw.Word)));

            ServerFilteredWords = new ConcurrentDictionary<ulong, ConcurrentHashSet<string>>(dict);

            var serverFiltering = gcs.Where(gc => gc.FilterWords);
            WordFilteringServers = new ConcurrentHashSet<ulong>(serverFiltering.Select(gc => gc.GuildId));

            WordFilteringChannels = new ConcurrentHashSet<ulong>(gcs.SelectMany(gc => gc.FilterWordsChannelIds.Select(fwci => fwci.ChannelId)));

            ZalgoFilteringServers = new ConcurrentHashSet<ulong>(gcs.Where(gc => gc.FilterZalgo).Select(gc => gc.GuildId));
            ZalgoFilteringChannels = new ConcurrentHashSet<ulong>(gcs.SelectMany(gc => gc.FilterZalgoChannelIds.Select(zfc => zfc.ChannelId)));

            client.MessageUpdated += (oldData, newMsg, channel) =>
            {
                var _ = Task.Run(() =>
                {
                    var guild = (channel as ITextChannel)?.Guild;

                    if (guild == null || !(newMsg is IUserMessage usrMsg))
                        return Task.CompletedTask;

                    return TryBlockEarly(guild, usrMsg);
                });
                return Task.CompletedTask;
            };
        }

        public async Task<bool> TryBlockEarly(IGuild guild, IUserMessage msg, bool realExecution = true)
            => msg.Author is IGuildUser gu && !gu.GuildPermissions.Administrator && (await FilterInvites(guild, msg, realExecution) || await FilterWords(guild, msg, realExecution) || await FilterZalgo(guild, msg, realExecution));

        public async Task<bool> FilterWords(IGuild guild, IUserMessage usrMsg, bool realExecution = true)
        {
            if (guild is null || usrMsg is null) return false;

            var filteredChannelWords = FilteredWordsForChannel(usrMsg.Channel.Id, guild.Id) ?? new ConcurrentHashSet<string>();
            var filteredServerWords = FilteredWordsForServer(guild.Id) ?? new ConcurrentHashSet<string>();
            var wordsInMessage = usrMsg.Content.ToLowerInvariant().Split(' ');
            if (filteredChannelWords.Count == 0 && filteredServerWords.Count == 0) return false;
            if (!wordsInMessage.Any(word => filteredChannelWords.Contains(word) || filteredServerWords.Contains(word))) return false;<commit_msg>Do not apply message filter to users with permission ManageMessages.<commit_after>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;
using Discord;
using Discord.Net;
using Discord.WebSocket;
using Mitternacht.Common.Collections;
using Mitternacht.Common.ModuleBehaviors;
using Mitternacht.Extensions;
using Mitternacht.Services;
using Mitternacht.Services.Database.Models;
using NLog;

namespace Mitternacht.Modules.Permissions.Services
{
    public class FilterService : IEarlyBlocker, INService
    {
        private readonly Logger _log;

        public ConcurrentHashSet<ulong> InviteFilteringChannels { get; }
        public ConcurrentHashSet<ulong> InviteFilteringServers { get; }

        //serverid, filteredwords
        public ConcurrentDictionary<ulong, ConcurrentHashSet<string>> ServerFilteredWords { get; }

        public ConcurrentHashSet<ulong> WordFilteringChannels { get; }
        public ConcurrentHashSet<ulong> WordFilteringServers { get; }

        public ConcurrentHashSet<ulong> ZalgoFilteringServers { get; }
        public ConcurrentHashSet<ulong> ZalgoFilteringChannels { get; }

        public ConcurrentHashSet<string> FilteredWordsForChannel(ulong channelId, ulong guildId)
        {
            var words = new ConcurrentHashSet<string>();
            if (WordFilteringChannels.Contains(channelId))
                ServerFilteredWords.TryGetValue(guildId, out words);
            return words;
        }

        public ConcurrentHashSet<string> FilteredWordsForServer(ulong guildId)
        {
            var words = new ConcurrentHashSet<string>();
            if (WordFilteringServers.Contains(guildId))
                ServerFilteredWords.TryGetValue(guildId, out words);
            return words;
        }

        public FilterService(DiscordSocketClient client, IEnumerable<GuildConfig> igcs)
        {
            _log = LogManager.GetCurrentClassLogger();
            var gcs = igcs.ToList();
            
            InviteFilteringServers = new ConcurrentHashSet<ulong>(gcs.Where(gc => gc.FilterInvites).Select(gc => gc.GuildId));
            InviteFilteringChannels = new ConcurrentHashSet<ulong>(gcs.SelectMany(gc => gc.FilterInvitesChannelIds.Select(fci => fci.ChannelId)));

            var dict = gcs.ToDictionary(gc => gc.GuildId, gc => new ConcurrentHashSet<string>(gc.FilteredWords.Select(fw => fw.Word)));

            ServerFilteredWords = new ConcurrentDictionary<ulong, ConcurrentHashSet<string>>(dict);

            var serverFiltering = gcs.Where(gc => gc.FilterWords);
            WordFilteringServers = new ConcurrentHashSet<ulong>(serverFiltering.Select(gc => gc.GuildId));

            WordFilteringChannels = new ConcurrentHashSet<ulong>(gcs.SelectMany(gc => gc.FilterWordsChannelIds.Select(fwci => fwci.ChannelId)));

            ZalgoFilteringServers = new ConcurrentHashSet<ulong>(gcs.Where(gc => gc.FilterZalgo).Select(gc => gc.GuildId));
            ZalgoFilteringChannels = new ConcurrentHashSet<ulong>(gcs.SelectMany(gc => gc.FilterZalgoChannelIds.Select(zfc => zfc.ChannelId)));

            client.MessageUpdated += (oldData, newMsg, channel) =>
            {
                var _ = Task.Run(() =>
                {
                    var guild = (channel as ITextChannel)?.Guild;

                    if (guild == null || !(newMsg is IUserMessage usrMsg))
                        return Task.CompletedTask;

                    return TryBlockEarly(guild, usrMsg);
                });
                return Task.CompletedTask;
            };
        }

        public async Task<bool> TryBlockEarly(IGuild guild, IUserMessage msg, bool realExecution = true)
            => msg.Author is IGuildUser gu && !gu.GuildPermissions.ManageMessages && (await FilterInvites(guild, msg, realExecution) || await FilterWords(guild, msg, realExecution) || await FilterZalgo(guild, msg, realExecution));

        public async Task<bool> FilterWords(IGuild guild, IUserMessage usrMsg, bool realExecution = true)
        {
            if (guild is null || usrMsg is null) return false;

            var filteredChannelWords = FilteredWordsForChannel(usrMsg.Channel.Id, guild.Id) ?? new ConcurrentHashSet<string>();
            var filteredServerWords = FilteredWordsForServer(guild.Id) ?? new ConcurrentHashSet<string>();
            var wordsInMessage = usrMsg.Content.ToLowerInvariant().Split(' ');
            if (filteredChannelWords.Count == 0 && filteredServerWords.Count == 0) return false;
            if (!wordsInMessage.Any(word => filteredChannelWords.Contains(word) || filteredServerWords.Contains(word))) return false;
{'fraction_non_alphanumeric': 0.09657754010695187, 'fraction_numerical': 0.000427807486631016, 'mean_word_length': 3.117569352708058, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5626896', 'n_tokens_mistral': 2678, 'n_tokens_neox': 2571, 'n_words': 522}