<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using CompatApiClient.Utils;
using CompatBot.Commands.Attributes;
using CompatBot.Database;
using CompatBot.Database.Providers;
using CompatBot.Utils;
using CompatBot.Utils.Extensions;
using DSharpPlus.CommandsNext;
using DSharpPlus.CommandsNext.Attributes;
using DSharpPlus.Entities;
using DSharpPlus.EventArgs;
using DSharpPlus.Interactivity.Extensions;
using Microsoft.EntityFrameworkCore;
using Exception = System.Exception;

namespace CompatBot.Commands
{
    [Group("filters"), Aliases("piracy", "filter"), RequiresBotSudoerRole, RequiresDm]
    [Description("Used to manage content filters. **Works only in DM**")]
    internal sealed class ContentFilters: BaseCommandModuleCustom
    {
        private static readonly TimeSpan InteractTimeout = TimeSpan.FromMinutes(5);
        private static readonly char[] Separators = {' ', ',', ';', '|'};

        [Command("list")]
        [Description("Lists all filters")]
        public async Task List(CommandContext ctx)
        {
            var table = new AsciiTable(
                new AsciiColumn("ID", alignToRight: true),
                new AsciiColumn("Trigger"),
                new AsciiColumn("Validation", maxWidth: 2048),
                new AsciiColumn("Context", maxWidth: 4096),
                new AsciiColumn("Actions"),
                new AsciiColumn("Custom message", maxWidth: 2048)
            );
            await using var db = new BotDb();
            var duplicates = new Dictionary<string, FilterContext>(StringComparer.InvariantCultureIgnoreCase);
            var filters = db.Piracystring.Where(ps => !ps.Disabled).AsNoTracking().AsEnumerable().OrderBy(ps => ps.String.ToUpperInvariant()).ToList();
            var nonUniqueTriggers = (
                from f in filters
                group f by f.String.ToUpperInvariant()
                into g
                where g.Count() > 1
                select g.Key
            ).ToList();
            foreach (var t in nonUniqueTriggers)
            {
                var duplicateFilters = filters.Where(ps => ps.String.Equals(t, StringComparison.InvariantCultureIgnoreCase)).ToList();
                foreach (FilterContext fctx in Enum.GetValues(typeof(FilterContext)))
                {
                    if (duplicateFilters.Count(f => (f.Context & fctx) == fctx) > 1)
                    {
                        if (duplicates.TryGetValue(t, out var fctxDup))
                            duplicates[t] = fctxDup | fctx;
                        else
                            duplicates[t] = fctx;
                    }
                }
            }
            foreach (var item in filters)
            {
                var ctxl = item.Context.ToString();
                if (duplicates.Count > 0
                    && duplicates.TryGetValue(item.String, out var fctx)
                    && (item.Context & fctx) != 0)
                    ctxl = "❗ " + ctxl;
                table.Add(
                    item.Id.ToString(),
                    item.String.Sanitize(),
                    item.ValidatingRegex ?? "",
                    ctxl,
                    item.Actions.ToFlagsString(),
                    item.CustomMessage ?? ""
                );
            }
            var result = new StringBuilder(table.ToString(false)).AppendLine()
                .AppendLine(FilterActionExtensions.GetLegend(""));
            await using var output = Config.MemoryStreamManager.GetStream();
            //await using (var gzip = new GZipStream(output, CompressionLevel.Optimal, true))
            await using (var writer = new StreamWriter(output, leaveOpen: true))
                await writer.WriteAsync(result.ToString()).ConfigureAwait(false);
            output.Seek(0, SeekOrigin.Begin);
            await ctx.RespondAsync(new DiscordMessageBuilder().WithFile("filters.txt", output)).ConfigureAwait(false);
        }

        [Command("add"), Aliases("create")]
        [Description("Adds a new content filter")]
        public async Task Add(CommandContext ctx, [RemainingText, Description("A plain string to match")] string trigger)
        {
            await using var db = new BotDb();
            Piracystring filter;
            if (string.IsNullOrEmpty(trigger))
                filter = new Piracystring();
            else
            {
                filter = await db.Piracystring.FirstOrDefaultAsync(ps => ps.String == trigger && ps.Disabled).ConfigureAwait(false);
                if (filter == null)
                    filter = new Piracystring {String = trigger};
                else
                    filter.Disabled = false;
            }
            var isNewFilter = filter.Id == default;
            if (isNewFilter)
            {
                filter.Context = FilterContext.Chat | FilterContext.Log;
                filter.Actions = FilterAction.RemoveContent | FilterAction.IssueWarning | FilterAction.SendMessage;
            }

            var (success, msg) = await EditFilterPropertiesAsync(ctx, db, filter).ConfigureAwait(false);
            if (success)
            {
                if (isNewFilter)
                    await db.Piracystring.AddAsync(filter).ConfigureAwait(false);
                await db.SaveChangesAsync().ConfigureAwait(false);
                await msg.UpdateOrCreateMessageAsync(ctx.Channel, embed: FormatFilter(filter).WithTitle("Created a new content filter #" + filter.Id)).ConfigureAwait(false);
                var member = ctx.Member ?? ctx.Client.GetMember(ctx.User);
                var reportMsg = $"{member?.GetMentionWithNickname()} added a new content filter: `{filter.String.Sanitize()}`";
                if (!string.IsNullOrEmpty(filter.ValidatingRegex))
                    reportMsg += $"\nValidation: `{filter.ValidatingRegex}`";
                await ctx.Client.ReportAsync("🆕 Content filter created", reportMsg, null, ReportSeverity.Low).ConfigureAwait(false);
                ContentFilter.RebuildMatcher();
            }
            else
                await msg.UpdateOrCreateMessageAsync(ctx.Channel, "Content filter creation aborted").ConfigureAwait(false);
        }

        [Command("edit"), Aliases("fix", "update", "change")]
        [Description("Modifies the specified content filter")]
        public async Task Edit(CommandContext ctx, [Description("Filter ID")] int id)
        {
            await using var db = new BotDb();
            var filter = await db.Piracystring.FirstOrDefaultAsync(ps => ps.Id == id && !ps.Disabled).ConfigureAwait(false);
            if (filter is null)
            {
                await ctx.RespondAsync("Specified filter does not exist").ConfigureAwait(false);
                return;
            }

            await EditFilterCmd(ctx, db, filter).ConfigureAwait(false);
        }

        [Command("edit")]
        public async Task Edit(CommandContext ctx, [Description("Trigger to edit"), RemainingText] string trigger)
        {
            await using var db = new BotDb();
            var filter = await db.Piracystring.FirstOrDefaultAsync(ps => ps.String.Equals(trigger, StringComparison.InvariantCultureIgnoreCase) && !ps.Disabled).ConfigureAwait(false);
            if (filter is null)
            {
                await ctx.RespondAsync("Specified filter does not exist").ConfigureAwait(false);
                return;
            }

            await EditFilterCmd(ctx, db, filter).ConfigureAwait(false);
        }
        
        [Command("view"), Aliases("show")]
        [Description("Shows the details of the specified content filter")]
        public async Task View(CommandContext ctx, [Description("Filter ID")] int id)
        {
            await using var db = new BotDb();
            var filter = await db.Piracystring.FirstOrDefaultAsync(ps => ps.Id == id && !ps.Disabled).ConfigureAwait(false);
            if (filter is null)
            {
                await ctx.RespondAsync("Specified filter does not exist").ConfigureAwait(false);
                return;
            }

            await ctx.RespondAsync(new DiscordMessageBuilder().WithEmbed(FormatFilter(filter))).ConfigureAwait(false);
        }
        
        [Command("view")]
        [Description("Shows the details of the specified content filter")]
        public async Task View(CommandContext ctx, [Description("Trigger to view"), RemainingText] string trigger)
        {
            await using var db = new BotDb();
            var filter = await db.Piracystring.FirstOrDefaultAsync(ps => ps.String.Equals(trigger, StringComparison.InvariantCultureIgnoreCase) && !ps.Disabled).ConfigureAwait(false);
            if (filter is null)
            {
                await ctx.RespondAsync("Specified filter does not exist").ConfigureAwait(false);
                return;
            }

            await ctx.RespondAsync(new DiscordMessageBuilder().WithEmbed(FormatFilter(filter))).ConfigureAwait(false);
        }

        [Command("remove"), Aliases("delete", "del")]
        [Description("Removes a content filter trigger")]
        public async Task Remove(CommandContext ctx, [Description("Filter IDs to remove, separated with spaces")] params int[] ids)
        {
            int removedFilters;
            var removedTriggers = new StringBuilder();
            await using (var db = new BotDb())
            {
                foreach (var f in db.Piracystring.Where(ps => ids.Contains(ps.Id) && !ps.Disabled))
                {
                    f.Disabled = true;
                    removedTriggers.Append($"\n`{f.String.Sanitize()}`");
                }
                removedFilters = await db.SaveChangesAsync(Config.Cts.Token).ConfigureAwait(false);
            }
<commit_msg>fix queries for new ef version<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using CompatApiClient.Utils;
using CompatBot.Commands.Attributes;
using CompatBot.Database;
using CompatBot.Database.Providers;
using CompatBot.Utils;
using CompatBot.Utils.Extensions;
using DSharpPlus.CommandsNext;
using DSharpPlus.CommandsNext.Attributes;
using DSharpPlus.Entities;
using DSharpPlus.EventArgs;
using DSharpPlus.Interactivity.Extensions;
using Microsoft.EntityFrameworkCore;
using Exception = System.Exception;

namespace CompatBot.Commands
{
    [Group("filters"), Aliases("piracy", "filter"), RequiresBotSudoerRole, RequiresDm]
    [Description("Used to manage content filters. **Works only in DM**")]
    internal sealed class ContentFilters: BaseCommandModuleCustom
    {
        private static readonly TimeSpan InteractTimeout = TimeSpan.FromMinutes(5);
        private static readonly char[] Separators = {' ', ',', ';', '|'};

        [Command("list")]
        [Description("Lists all filters")]
        public async Task List(CommandContext ctx)
        {
            var table = new AsciiTable(
                new AsciiColumn("ID", alignToRight: true),
                new AsciiColumn("Trigger"),
                new AsciiColumn("Validation", maxWidth: 2048),
                new AsciiColumn("Context", maxWidth: 4096),
                new AsciiColumn("Actions"),
                new AsciiColumn("Custom message", maxWidth: 2048)
            );
            await using var db = new BotDb();
            var duplicates = new Dictionary<string, FilterContext>(StringComparer.InvariantCultureIgnoreCase);
            var filters = db.Piracystring.Where(ps => !ps.Disabled).AsNoTracking().AsEnumerable().OrderBy(ps => ps.String.ToUpperInvariant()).ToList();
            var nonUniqueTriggers = (
                from f in filters
                group f by f.String.ToUpperInvariant()
                into g
                where g.Count() > 1
                select g.Key
            ).ToList();
            foreach (var t in nonUniqueTriggers)
            {
                var duplicateFilters = filters.Where(ps => ps.String.Equals(t, StringComparison.InvariantCultureIgnoreCase)).ToList();
                foreach (FilterContext fctx in Enum.GetValues(typeof(FilterContext)))
                {
                    if (duplicateFilters.Count(f => (f.Context & fctx) == fctx) > 1)
                    {
                        if (duplicates.TryGetValue(t, out var fctxDup))
                            duplicates[t] = fctxDup | fctx;
                        else
                            duplicates[t] = fctx;
                    }
                }
            }
            foreach (var item in filters)
            {
                var ctxl = item.Context.ToString();
                if (duplicates.Count > 0
                    && duplicates.TryGetValue(item.String, out var fctx)
                    && (item.Context & fctx) != 0)
                    ctxl = "❗ " + ctxl;
                table.Add(
                    item.Id.ToString(),
                    item.String.Sanitize(),
                    item.ValidatingRegex ?? "",
                    ctxl,
                    item.Actions.ToFlagsString(),
                    item.CustomMessage ?? ""
                );
            }
            var result = new StringBuilder(table.ToString(false)).AppendLine()
                .AppendLine(FilterActionExtensions.GetLegend(""));
            await using var output = Config.MemoryStreamManager.GetStream();
            //await using (var gzip = new GZipStream(output, CompressionLevel.Optimal, true))
            await using (var writer = new StreamWriter(output, leaveOpen: true))
                await writer.WriteAsync(result.ToString()).ConfigureAwait(false);
            output.Seek(0, SeekOrigin.Begin);
            await ctx.RespondAsync(new DiscordMessageBuilder().WithFile("filters.txt", output)).ConfigureAwait(false);
        }

        [Command("add"), Aliases("create")]
        [Description("Adds a new content filter")]
        public async Task Add(CommandContext ctx, [RemainingText, Description("A plain string to match")] string trigger)
        {
            await using var db = new BotDb();
            Piracystring filter;
            if (string.IsNullOrEmpty(trigger))
                filter = new Piracystring();
            else
            {
                filter = await db.Piracystring.FirstOrDefaultAsync(ps => ps.String == trigger && ps.Disabled).ConfigureAwait(false);
                if (filter == null)
                    filter = new Piracystring {String = trigger};
                else
                    filter.Disabled = false;
            }
            var isNewFilter = filter.Id == default;
            if (isNewFilter)
            {
                filter.Context = FilterContext.Chat | FilterContext.Log;
                filter.Actions = FilterAction.RemoveContent | FilterAction.IssueWarning | FilterAction.SendMessage;
            }

            var (success, msg) = await EditFilterPropertiesAsync(ctx, db, filter).ConfigureAwait(false);
            if (success)
            {
                if (isNewFilter)
                    await db.Piracystring.AddAsync(filter).ConfigureAwait(false);
                await db.SaveChangesAsync().ConfigureAwait(false);
                await msg.UpdateOrCreateMessageAsync(ctx.Channel, embed: FormatFilter(filter).WithTitle("Created a new content filter #" + filter.Id)).ConfigureAwait(false);
                var member = ctx.Member ?? ctx.Client.GetMember(ctx.User);
                var reportMsg = $"{member?.GetMentionWithNickname()} added a new content filter: `{filter.String.Sanitize()}`";
                if (!string.IsNullOrEmpty(filter.ValidatingRegex))
                    reportMsg += $"\nValidation: `{filter.ValidatingRegex}`";
                await ctx.Client.ReportAsync("🆕 Content filter created", reportMsg, null, ReportSeverity.Low).ConfigureAwait(false);
                ContentFilter.RebuildMatcher();
            }
            else
                await msg.UpdateOrCreateMessageAsync(ctx.Channel, "Content filter creation aborted").ConfigureAwait(false);
        }

        [Command("edit"), Aliases("fix", "update", "change")]
        [Description("Modifies the specified content filter")]
        public async Task Edit(CommandContext ctx, [Description("Filter ID")] int id)
        {
            await using var db = new BotDb();
            var filter = await db.Piracystring.FirstOrDefaultAsync(ps => ps.Id == id && !ps.Disabled).ConfigureAwait(false);
            if (filter is null)
            {
                await ctx.RespondAsync("Specified filter does not exist").ConfigureAwait(false);
                return;
            }

            await EditFilterCmd(ctx, db, filter).ConfigureAwait(false);
        }

        [Command("edit")]
        public async Task Edit(CommandContext ctx, [Description("Trigger to edit"), RemainingText] string trigger)
        {
            await using var db = new BotDb();
            var filter = await db.Piracystring.FirstOrDefaultAsync(ps => ps.String == trigger && !ps.Disabled).ConfigureAwait(false);
            if (filter is null)
            {
                await ctx.RespondAsync("Specified filter does not exist").ConfigureAwait(false);
                return;
            }

            await EditFilterCmd(ctx, db, filter).ConfigureAwait(false);
        }
        
        [Command("view"), Aliases("show")]
        [Description("Shows the details of the specified content filter")]
        public async Task View(CommandContext ctx, [Description("Filter ID")] int id)
        {
            await using var db = new BotDb();
            var filter = await db.Piracystring.FirstOrDefaultAsync(ps => ps.Id == id && !ps.Disabled).ConfigureAwait(false);
            if (filter is null)
            {
                await ctx.RespondAsync("Specified filter does not exist").ConfigureAwait(false);
                return;
            }

            await ctx.RespondAsync(new DiscordMessageBuilder().WithEmbed(FormatFilter(filter))).ConfigureAwait(false);
        }
        
        [Command("view")]
        [Description("Shows the details of the specified content filter")]
        public async Task View(CommandContext ctx, [Description("Trigger to view"), RemainingText] string trigger)
        {
            await using var db = new BotDb();
            var filter = await db.Piracystring.FirstOrDefaultAsync(ps => ps.String == trigger && !ps.Disabled).ConfigureAwait(false);
            if (filter is null)
            {
                await ctx.RespondAsync("Specified filter does not exist").ConfigureAwait(false);
                return;
            }

            await ctx.RespondAsync(new DiscordMessageBuilder().WithEmbed(FormatFilter(filter))).ConfigureAwait(false);
        }

        [Command("remove"), Aliases("delete", "del")]
        [Description("Removes a content filter trigger")]
        public async Task Remove(CommandContext ctx, [Description("Filter IDs to remove, separated with spaces")] params int[] ids)
        {
            int removedFilters;
            var removedTriggers = new StringBuilder();
            await using (var db = new BotDb())
            {
                foreach (var f in db.Piracystring.Where(ps => ids.Contains(ps.Id) && !ps.Disabled))
                {
                    f.Disabled = true;
                    removedTriggers.Append($"\n`{f.String.Sanitize()}`");
                }
                removedFilters = await db.SaveChangesAsync(Config.Cts.Token).ConfigureAwait(false);
            }

{'fraction_non_alphanumeric': 0.10556768004872355, 'fraction_numerical': 0.0018271329239202153, 'mean_word_length': 2.0567793980763263, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4651646', 'n_tokens_mistral': 5093, 'n_tokens_neox': 4933, 'n_words': 1177}