<commit_before>﻿using Discord;
using Discord.Commands;
using Microsoft.EntityFrameworkCore;
using NadekoBot.Attributes;
using NadekoBot.Extensions;
using NadekoBot.Services;
using NadekoBot.Services.Database.Models;
using NLog;
using System;
using System.Collections.Concurrent;
using System.Linq;
using System.Threading.Tasks;

namespace NadekoBot.Modules.Administration
{
    public partial class Administration
    {
        public enum ProtectionType
        {
            Raiding,
            Spamming,
        }

        public class AntiRaidStats
        {
            public AntiRaidSetting AntiRaidSettings { get; set; }
            public int UsersCount { get; set; }
            public ConcurrentHashSet<IGuildUser> RaidUsers { get; set; } = new ConcurrentHashSet<IGuildUser>();
        }

        public class AntiSpamStats
        {
            public AntiSpamSetting AntiSpamSettings { get; set; }
            public ConcurrentDictionary<ulong, UserSpamStats> UserStats { get; set; }
                = new ConcurrentDictionary<ulong, UserSpamStats>();
        }

        public class UserSpamStats
        {
            public int Count { get; set; }
            public string LastMessage { get; set; }

            public UserSpamStats(string msg)
            {
                Count = 1;
                LastMessage = msg.ToUpperInvariant();
            }

            public void ApplyNextMessage(string message)
            {
                var upperMsg = message.ToUpperInvariant();
                if (upperMsg == LastMessage)
                    Count++;
                else
                {
                    LastMessage = upperMsg;
                    Count = 0;
                }
            }
        }

        [Group]
        public class ProtectionCommands : NadekoSubmodule
        {
            private static readonly ConcurrentDictionary<ulong, AntiRaidStats> _antiRaidGuilds =
                    new ConcurrentDictionary<ulong, AntiRaidStats>();
            // guildId | (userId|messages)
            private static readonly ConcurrentDictionary<ulong, AntiSpamStats> _antiSpamGuilds =
                    new ConcurrentDictionary<ulong, AntiSpamStats>();

            private new static readonly Logger _log;

            static ProtectionCommands()
            {
                _log = LogManager.GetCurrentClassLogger();

                foreach (var gc in NadekoBot.AllGuildConfigs)
                {
                    var raid = gc.AntiRaidSetting;
                    var spam = gc.AntiSpamSetting;

                    if (raid != null)
                    {
                        var raidStats = new AntiRaidStats() { AntiRaidSettings = raid };
                        _antiRaidGuilds.TryAdd(gc.GuildId, raidStats);
                    }

                    if (spam != null)
                        _antiSpamGuilds.TryAdd(gc.GuildId, new AntiSpamStats() { AntiSpamSettings = spam });
                }

                NadekoBot.Client.MessageReceived += (imsg) =>
                {
                    var msg = imsg as IUserMessage;
                    if (msg == null || msg.Author.IsBot)
                        return Task.CompletedTask;

                    var channel = msg.Channel as ITextChannel;
                    if (channel == null)
                        return Task.CompletedTask;
                    var _ = Task.Run(async () =>
                    {
                        try
                        {
                            AntiSpamStats spamSettings;
                            if (!_antiSpamGuilds.TryGetValue(channel.Guild.Id, out spamSettings) ||
                                spamSettings.AntiSpamSettings.IgnoredChannels.Contains(new AntiSpamIgnore()
                                {
                                    ChannelId = channel.Id
                                }))
                                return;

                            var stats = spamSettings.UserStats.AddOrUpdate(msg.Author.Id, new UserSpamStats(msg.Content),
                                (id, old) =>
                                {
                                    old.ApplyNextMessage(msg.Content); return old;
                                });

                            if (stats.Count >= spamSettings.AntiSpamSettings.MessageThreshold)
                            {
                                if (spamSettings.UserStats.TryRemove(msg.Author.Id, out stats))
                                {
                                    await PunishUsers(spamSettings.AntiSpamSettings.Action, ProtectionType.Spamming, (IGuildUser)msg.Author)
                                        .ConfigureAwait(false);
                                }
                            }
                        }
                        catch
                        {
                            // ignored
                        }
                    });
                    return Task.CompletedTask;
                };

                NadekoBot.Client.UserJoined += (usr) =>
                {
                    if (usr.IsBot)
                        return Task.CompletedTask;
                    AntiRaidStats settings;
                    if (!_antiRaidGuilds.TryGetValue(usr.Guild.Id, out settings))
                        return Task.CompletedTask;
                    if (!settings.RaidUsers.Add(usr))
                        return Task.CompletedTask;

                    var _ = Task.Run(async () =>
                    {<commit_msg>.antispam will ignore image uploads with no message content<commit_after>﻿using Discord;
using Discord.Commands;
using Microsoft.EntityFrameworkCore;
using NadekoBot.Attributes;
using NadekoBot.Extensions;
using NadekoBot.Services;
using NadekoBot.Services.Database.Models;
using NLog;
using System;
using System.Collections.Concurrent;
using System.Linq;
using System.Threading.Tasks;

namespace NadekoBot.Modules.Administration
{
    public partial class Administration
    {
        public enum ProtectionType
        {
            Raiding,
            Spamming,
        }

        public class AntiRaidStats
        {
            public AntiRaidSetting AntiRaidSettings { get; set; }
            public int UsersCount { get; set; }
            public ConcurrentHashSet<IGuildUser> RaidUsers { get; set; } = new ConcurrentHashSet<IGuildUser>();
        }

        public class AntiSpamStats
        {
            public AntiSpamSetting AntiSpamSettings { get; set; }
            public ConcurrentDictionary<ulong, UserSpamStats> UserStats { get; set; }
                = new ConcurrentDictionary<ulong, UserSpamStats>();
        }

        public class UserSpamStats
        {
            public int Count { get; set; }
            public string LastMessage { get; set; }

            public UserSpamStats(string msg)
            {
                Count = 1;
                LastMessage = msg.ToUpperInvariant();
            }

            public void ApplyNextMessage(IUserMessage message)
            {
                var upperMsg = message.Content.ToUpperInvariant();
                if (upperMsg != LastMessage || (string.IsNullOrWhiteSpace(upperMsg) && message.Attachments.Any()))
                {
                    LastMessage = upperMsg;
                    Count = 0;
                }
                else
                {
                    Count++;
                }
            }
        }

        [Group]
        public class ProtectionCommands : NadekoSubmodule
        {
            private static readonly ConcurrentDictionary<ulong, AntiRaidStats> _antiRaidGuilds =
                    new ConcurrentDictionary<ulong, AntiRaidStats>();
            // guildId | (userId|messages)
            private static readonly ConcurrentDictionary<ulong, AntiSpamStats> _antiSpamGuilds =
                    new ConcurrentDictionary<ulong, AntiSpamStats>();

            private new static readonly Logger _log;

            static ProtectionCommands()
            {
                _log = LogManager.GetCurrentClassLogger();

                foreach (var gc in NadekoBot.AllGuildConfigs)
                {
                    var raid = gc.AntiRaidSetting;
                    var spam = gc.AntiSpamSetting;

                    if (raid != null)
                    {
                        var raidStats = new AntiRaidStats() { AntiRaidSettings = raid };
                        _antiRaidGuilds.TryAdd(gc.GuildId, raidStats);
                    }

                    if (spam != null)
                        _antiSpamGuilds.TryAdd(gc.GuildId, new AntiSpamStats() { AntiSpamSettings = spam });
                }

                NadekoBot.Client.MessageReceived += (imsg) =>
                {
                    var msg = imsg as IUserMessage;
                    if (msg == null || msg.Author.IsBot)
                        return Task.CompletedTask;

                    var channel = msg.Channel as ITextChannel;
                    if (channel == null)
                        return Task.CompletedTask;
                    var _ = Task.Run(async () =>
                    {
                        try
                        {
                            AntiSpamStats spamSettings;
                            if (!_antiSpamGuilds.TryGetValue(channel.Guild.Id, out spamSettings) ||
                                spamSettings.AntiSpamSettings.IgnoredChannels.Contains(new AntiSpamIgnore()
                                {
                                    ChannelId = channel.Id
                                }))
                                return;

                            var stats = spamSettings.UserStats.AddOrUpdate(msg.Author.Id, new UserSpamStats(msg.Content),
                                (id, old) =>
                                {
                                    old.ApplyNextMessage(msg); return old;
                                });

                            if (stats.Count >= spamSettings.AntiSpamSettings.MessageThreshold)
                            {
                                if (spamSettings.UserStats.TryRemove(msg.Author.Id, out stats))
                                {
                                    await PunishUsers(spamSettings.AntiSpamSettings.Action, ProtectionType.Spamming, (IGuildUser)msg.Author)
                                        .ConfigureAwait(false);
                                }
                            }
                        }
                        catch
                        {
                            // ignored
                        }
                    });
                    return Task.CompletedTask;
                };

                NadekoBot.Client.UserJoined += (usr) =>
                {
                    if (usr.IsBot)
                        return Task.CompletedTask;
                    AntiRaidStats settings;
                    if (!_antiRaidGuilds.TryGetValue(usr.Guild.Id, out settings))
                        return Task.CompletedTask;
                    if (!settings.RaidUsers.Add(usr))
                        return Task.CompletedTask;

                    var _ = Task.Run(async () =>
                    {
{'fraction_non_alphanumeric': 0.07002500893176135, 'fraction_numerical': 0.0003572704537334762, 'mean_word_length': 1.1299220087502377, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1116252', 'n_tokens_mistral': 2704, 'n_tokens_neox': 2510, 'n_words': 522}