<commit_before>﻿using NLog;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Discord;
using Discord.Commands;
using Discord.WebSocket;
using Microsoft.Extensions.DependencyInjection;
using MongoDB.Driver;
using NoAdsHere.Common;
using NoAdsHere.Database;
using NoAdsHere.Database.Models.Global;
using NoAdsHere.Database.Models.GuildSettings;

namespace NoAdsHere.Services.AntiAds
{
    public static class AntiAds
    {
        private static readonly Logger Logger = LogManager.GetLogger("AntiAds");
        private static DiscordShardedClient _client;
        private static MongoClient _mongo;
        private static readonly Dictionary<BlockType, List<ulong>> ActiveGuilds = new Dictionary<BlockType, List<ulong>>(0);
        
        private static readonly Regex InstantInvite =
            new Regex(
                @"(?:(?i)discord(?:(?:\.|.?dot.?)(?i)gg|app(?:\.|.?dot.?)com\/invite)\/(?<id>([\w]{10,16}|[a-zA-Z1-9]{4,8})))",
                RegexOptions.Compiled);
        private static readonly Regex TwitchStream = new Regex(@"(?i)twitch\.(?i)tv\/(#)?([a-zA-Z0-9][\w]{2,24})",
            RegexOptions.Compiled);
        private static readonly Regex TwitchVideo = new Regex(@"(?i)twitch\.(?i)tv\/(?i)videos\/(#)?([0-9]{2,24})",
            RegexOptions.Compiled);
        private static readonly Regex TwitchClip = new Regex(@"(?i)clips\.(?i)twitch\.(?i)tv\/(#)?([a-zA-Z0-9][\w]{4,50})",
            RegexOptions.Compiled);
        private static readonly Regex YoutubeLink =
            new Regex(@"(?i)youtu(?:\.(?i)be|be\.com)\/(?:.*v(?:\/|=)|(?:.*\/)?)([a-zA-Z0-9-_]+)",
                RegexOptions.Compiled);

        public static Task Install(IServiceProvider provider)
        {
            _client = provider.GetService<DiscordShardedClient>();
            _mongo = provider.GetService<MongoClient>();
            return Task.CompletedTask;
        }

        public static async Task StartAsync()
        {
            Logger.Info("Loading Blocktypes into ActiveGuilds Dictionary");
            await PopulateDictionary().ConfigureAwait(false);
            Logger.Info("Hooking GuildAvailable event to Populate Active Guilds Dictionary");
            _client.GuildAvailable += GuildLoader;
            
            _client.MessageReceived += AdsHandler;
            _client.MessageUpdated += MessageUpdateAntiAds;
            Logger.Info("AntiAds service started.");
        }

        private static async Task MessageUpdateAntiAds(Cacheable<IMessage, ulong> cacheable, SocketMessage socketMessage, ISocketMessageChannel channel)
        {
            await AdsHandler(socketMessage).ConfigureAwait(false);
        }

        private static Task PopulateDictionary()
        {
            foreach (BlockType type in Enum.GetValues(typeof(BlockType)))
                ActiveGuilds.Add(type, new List<ulong>(0));
            return Task.CompletedTask;
        }

        private static async Task GuildLoader(SocketGuild socketGuild)
        {
            if (ActiveGuilds.Keys.Any(blocktype => !ActiveGuilds[blocktype].Contains(socketGuild.Id)))
            {
                var blocks = await _mongo.GetCollection<Block>(_client.GetShardFor(socketGuild)).GetGuildBlocksAsync(socketGuild.Id);
                foreach (var block in blocks.OrderBy(block => block.GuildId))
                {
                    if (!block.IsEnabled) continue;
                    ActiveGuilds[block.BlockType].Add(block.GuildId);<commit_msg>Fixed continues ReAdding of Guilds<commit_after>﻿using NLog;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Discord;
using Discord.Commands;
using Discord.WebSocket;
using Microsoft.Extensions.DependencyInjection;
using MongoDB.Driver;
using NoAdsHere.Common;
using NoAdsHere.Database;
using NoAdsHere.Database.Models.Global;
using NoAdsHere.Database.Models.GuildSettings;

namespace NoAdsHere.Services.AntiAds
{
    public static class AntiAds
    {
        private static readonly Logger Logger = LogManager.GetLogger("AntiAds");
        private static DiscordShardedClient _client;
        private static MongoClient _mongo;
        private static readonly Dictionary<BlockType, List<ulong>> ActiveGuilds = new Dictionary<BlockType, List<ulong>>(0);
        
        private static readonly Regex InstantInvite =
            new Regex(
                @"(?:(?i)discord(?:(?:\.|.?dot.?)(?i)gg|app(?:\.|.?dot.?)com\/invite)\/(?<id>([\w]{10,16}|[a-zA-Z1-9]{4,8})))",
                RegexOptions.Compiled);
        private static readonly Regex TwitchStream = new Regex(@"(?i)twitch\.(?i)tv\/(#)?([a-zA-Z0-9][\w]{2,24})",
            RegexOptions.Compiled);
        private static readonly Regex TwitchVideo = new Regex(@"(?i)twitch\.(?i)tv\/(?i)videos\/(#)?([0-9]{2,24})",
            RegexOptions.Compiled);
        private static readonly Regex TwitchClip = new Regex(@"(?i)clips\.(?i)twitch\.(?i)tv\/(#)?([a-zA-Z0-9][\w]{4,50})",
            RegexOptions.Compiled);
        private static readonly Regex YoutubeLink =
            new Regex(@"(?i)youtu(?:\.(?i)be|be\.com)\/(?:.*v(?:\/|=)|(?:.*\/)?)([a-zA-Z0-9-_]+)",
                RegexOptions.Compiled);

        public static Task Install(IServiceProvider provider)
        {
            _client = provider.GetService<DiscordShardedClient>();
            _mongo = provider.GetService<MongoClient>();
            return Task.CompletedTask;
        }

        public static async Task StartAsync()
        {
            Logger.Info("Loading Blocktypes into ActiveGuilds Dictionary");
            await PopulateDictionary().ConfigureAwait(false);
            Logger.Info("Hooking GuildAvailable event to Populate Active Guilds Dictionary");
            _client.GuildAvailable += GuildLoader;
            
            _client.MessageReceived += AdsHandler;
            _client.MessageUpdated += MessageUpdateAntiAds;
            Logger.Info("AntiAds service started.");
        }

        private static async Task MessageUpdateAntiAds(Cacheable<IMessage, ulong> _, SocketMessage socketMessage, ISocketMessageChannel channel)
        {
            await AdsHandler(socketMessage).ConfigureAwait(false);
        }

        private static Task PopulateDictionary()
        {
            foreach (BlockType type in Enum.GetValues(typeof(BlockType)))
                ActiveGuilds.Add(type, new List<ulong>(0));
            return Task.CompletedTask;
        }

        private static async Task GuildLoader(SocketGuild socketGuild)
        {
            if (ActiveGuilds.Keys.All(blocktype => !ActiveGuilds[blocktype].Contains(socketGuild.Id)))
            {
                var blocks = await _mongo.GetCollection<Block>(_client.GetShardFor(socketGuild)).GetGuildBlocksAsync(socketGuild.Id);
                foreach (var block in blocks.OrderBy(block => block.GuildId))
                {
                    if (!block.IsEnabled) continue;
                    ActiveGuilds[block.BlockType].Add(block.GuildId);
{'fraction_non_alphanumeric': 0.13051444679351656, 'fraction_numerical': 0.007610993657505285, 'mean_word_length': 2.75251189846642, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4424029', 'n_tokens_mistral': 2135, 'n_tokens_neox': 2030, 'n_words': 382}