<commit_before>﻿using System;
using System.Diagnostics;
using System.IO;
using System.IO.Pipelines;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CompatApiClient.Utils;
using CompatBot.Commands;
using CompatBot.Commands.Attributes;
using CompatBot.Database;
using CompatBot.Database.Providers;
using CompatBot.EventHandlers.LogParsing;
using CompatBot.EventHandlers.LogParsing.POCOs;
using CompatBot.EventHandlers.LogParsing.ArchiveHandlers;
using CompatBot.Utils;
using CompatBot.Utils.ResultFormatters;
using DSharpPlus;
using DSharpPlus.Entities;
using DSharpPlus.EventArgs;
using CompatBot.EventHandlers.LogParsing.SourceHandlers;
using CompatBot.Utils.Extensions;

namespace CompatBot.EventHandlers
{
    public static class LogParsingHandler
    {
        private static readonly char[] linkSeparator = { ' ', '>', '\r', '\n' };
        private static readonly ISourceHandler[] sourceHandlers =
        {
            new DiscordAttachmentHandler(),
            new GoogleDriveHandler(),
            new DropboxHandler(),
            new MegaHandler(),
            new PastebinHandler(),
        };
        private static readonly IArchiveHandler[] archiveHandlers =
        {
            new GzipHandler(),
            new ZipHandler(),
            new RarHandler(),
            new SevenZipHandler(),
            new PlainTextHandler(),
        };

        private static readonly SemaphoreSlim QueueLimiter = new SemaphoreSlim(Math.Max(1, Environment.ProcessorCount / 2), Math.Max(1, Environment.ProcessorCount / 2));
        private delegate void OnLog(DiscordClient client, DiscordChannel channel, DiscordMessage message, DiscordMember requester = null, bool checkExternalLinks = false);
        private static event OnLog OnNewLog;

        static LogParsingHandler()
        {
            OnNewLog += EnqueueLogProcessing;
        }

        public static Task OnMessageCreated(MessageCreateEventArgs args)
        {
            var message = args.Message;
            if (message.Author.IsBotSafeCheck())
                return Task.CompletedTask;

            if (!string.IsNullOrEmpty(message.Content)
                && (message.Content.StartsWith(Config.CommandPrefix)
                    || message.Content.StartsWith(Config.AutoRemoveCommandPrefix)))
                return Task.CompletedTask;

            var checkExternalLinks = "help".Equals(args.Channel.Name, StringComparison.InvariantCultureIgnoreCase)
                                     || LimitedToSpamChannel.IsSpamChannel(args.Channel);
            OnNewLog(args.Client, args.Channel, args.Message, checkExternalLinks: checkExternalLinks);
            return Task.CompletedTask;
        }

        public static async void EnqueueLogProcessing(DiscordClient client, DiscordChannel channel, DiscordMessage message, DiscordMember requester = null, bool checkExternalLinks = false)
        {
            try
            {
                if (!QueueLimiter.Wait(0))
                {
                    await channel.SendMessageAsync("Log processing is rate limited, try again a bit later").ConfigureAwait(false);
                    return;
                }

                bool parsedLog = false;
                var startTime = Stopwatch.StartNew();
                DiscordMessage botMsg = null;
                try
                {
                    var possibleHandlers = sourceHandlers.Select(h => h.FindHandlerAsync(message, archiveHandlers).ConfigureAwait(false).GetAwaiter().GetResult()).ToList();
                    var source = possibleHandlers.FirstOrDefault(h => h.source != null).source;
                    var fail = possibleHandlers.FirstOrDefault(h => !string.IsNullOrEmpty(h.failReason)).failReason;
                    if (source != null)
                    {
                        Config.Log.Debug($">>>>>>> {message.Id % 100} Parsing log '{source.FileName}' from {message.Author.Username}#{message.Author.Discriminator} ({message.Author.Id}) using {source.GetType().Name} ({source.SourceFileSize} bytes)...");
                        var analyzingProgressEmbed = GetAnalyzingMsgEmbed();
                        botMsg = await channel.SendMessageAsync(embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false);
                        parsedLog = true;

                        var timeout = new CancellationTokenSource(Config.LogParsingTimeout);
                        var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, Config.Cts.Token);
                        var tries = 0;
                        LogParseState result = null;
                        do
                        {
                                result = await ParseLogAsync(
                                    source,
                                    async () => botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false),
                                    combinedTokenSource.Token
                                ).ConfigureAwait(false);
                                tries++;
                        } while (result == null && !combinedTokenSource.IsCancellationRequested && tries < 3);
                        if (result == null)
                        {
                            botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: new DiscordEmbedBuilder
                                {
                                    Description = "Log analysis failed, most likely cause is a truncated/invalid log.\n" +
                                                  "Please run the game again and re-upload a new copy.",
                                    Color = Config.Colors.LogResultFailed,
                                }
                                .AddAuthor(client, message, source)
                                .Build()
                            ).ConfigureAwait(false);
                        }
                        else
                        {
                            result.ParsingTime = startTime.Elapsed;
                            try
                            {
                                if (result.Error == LogParseState.ErrorCode.PiracyDetected)
                                {
                                    var yarr = client.GetEmoji(":piratethink:", "☠");
                                    result.ReadBytes = 0;
                                    if (message.Author.IsWhitelisted(client, channel.Guild))
                                    {
                                        var piracyWarning = await result.AsEmbedAsync(client, message, source).ConfigureAwait(false);
                                        piracyWarning = piracyWarning.WithDescription("Please remove the log and issue warning to the original author of the log");
                                        botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: piracyWarning).ConfigureAwait(false);
                                        await client.ReportAsync(yarr + " Pirated Release (whitelisted by role)", message, result.SelectedFilter?.String, result.SelectedFilterContext, ReportSeverity.Low).ConfigureAwait(false);
                                    }
                                    else
                                    {
                                        var severity = ReportSeverity.Low;
                                        try
                                        {
                                            await message.DeleteAsync("Piracy detected in log").ConfigureAwait(false);
                                        }
                                        catch (Exception e)
                                        {
                                            severity = ReportSeverity.High;
                                            Config.Log.Warn(e, $"Unable to delete message in {channel.Name}");
                                        }
                                        try
                                        {
                                            botMsg = await botMsg.UpdateOrCreateMessageAsync(channel,
                                                $"{message.Author.Mention}, please read carefully:",
                                                embed: await result.AsEmbedAsync(client, message, source).ConfigureAwait(false)
                                            ).ConfigureAwait(false);
                                        }
                                        catch (Exception e)
                                        {
                                            Config.Log.Error(e, "Failed to send piracy warning");
                                        }
                                        try
                                        {
                                            await client.ReportAsync(yarr + " Pirated Release", message, result.SelectedFilter?.String, result.SelectedFilterContext, severity).ConfigureAwait(false);
                                        }
                                        catch (Exception e)
                                        {
                                            Config.Log.Error(e, "Failed to send piracy report");
                                        }
                                        if (!(message.Channel.IsPrivate || (message.Channel.Name?.Contains("spam") ?? true)))
                                            await Warnings.AddAsync(client, message, message.Author.Id, message.Author.Username, client.CurrentUser, "Pirated Release", $"{result.SelectedFilter?.String} - {result.SelectedFilterContext?.Sanitize()}");
                                    }
                                }
                                else
                                {
                                    if (result.SelectedFilter != null)
                                        await ContentFilter.PerformFilterActions(client, message, result.SelectedFilter, result.SelectedFilterContext).ConfigureAwait(false);
                                    if (result.SelectedFilter == null || !result.SelectedFilter.Actions.HasFlag(FilterAction.RemoveContent))
                                        botMsg = await botMsg.UpdateOrCreateMessageAsync(channel,
                                            requester == null ? null : $"Analyzed log from {client.GetMember(channel.Guild, message.Author)?.GetUsernameWithNickname()} by request from {requester.Mention}:",
                                            embed: await result.AsEmbedAsync(client, message, source).ConfigureAwait(false)
                                        ).ConfigureAwait(false);
                                }
                            }
                            catch (Exception e)
                            {
                                Config.Log.Error(e, "Sending log results failed");
                            }
                        }
                        return;
                    }
                    else if (!string.IsNullOrEmpty(fail)
                             && ("help".Equals(channel.Name, StringComparison.InvariantCultureIgnoreCase) || LimitedToSpamChannel.IsSpamChannel(channel)))<commit_msg>show log parsing results even if we hit any filter, except for blatant piracy<commit_after>﻿using System;
using System.Diagnostics;
using System.IO;
using System.IO.Pipelines;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CompatApiClient.Utils;
using CompatBot.Commands;
using CompatBot.Commands.Attributes;
using CompatBot.Database;
using CompatBot.Database.Providers;
using CompatBot.EventHandlers.LogParsing;
using CompatBot.EventHandlers.LogParsing.POCOs;
using CompatBot.EventHandlers.LogParsing.ArchiveHandlers;
using CompatBot.Utils;
using CompatBot.Utils.ResultFormatters;
using DSharpPlus;
using DSharpPlus.Entities;
using DSharpPlus.EventArgs;
using CompatBot.EventHandlers.LogParsing.SourceHandlers;
using CompatBot.Utils.Extensions;

namespace CompatBot.EventHandlers
{
    public static class LogParsingHandler
    {
        private static readonly char[] linkSeparator = { ' ', '>', '\r', '\n' };
        private static readonly ISourceHandler[] sourceHandlers =
        {
            new DiscordAttachmentHandler(),
            new GoogleDriveHandler(),
            new DropboxHandler(),
            new MegaHandler(),
            new PastebinHandler(),
        };
        private static readonly IArchiveHandler[] archiveHandlers =
        {
            new GzipHandler(),
            new ZipHandler(),
            new RarHandler(),
            new SevenZipHandler(),
            new PlainTextHandler(),
        };

        private static readonly SemaphoreSlim QueueLimiter = new SemaphoreSlim(Math.Max(1, Environment.ProcessorCount / 2), Math.Max(1, Environment.ProcessorCount / 2));
        private delegate void OnLog(DiscordClient client, DiscordChannel channel, DiscordMessage message, DiscordMember requester = null, bool checkExternalLinks = false);
        private static event OnLog OnNewLog;

        static LogParsingHandler()
        {
            OnNewLog += EnqueueLogProcessing;
        }

        public static Task OnMessageCreated(MessageCreateEventArgs args)
        {
            var message = args.Message;
            if (message.Author.IsBotSafeCheck())
                return Task.CompletedTask;

            if (!string.IsNullOrEmpty(message.Content)
                && (message.Content.StartsWith(Config.CommandPrefix)
                    || message.Content.StartsWith(Config.AutoRemoveCommandPrefix)))
                return Task.CompletedTask;

            var checkExternalLinks = "help".Equals(args.Channel.Name, StringComparison.InvariantCultureIgnoreCase)
                                     || LimitedToSpamChannel.IsSpamChannel(args.Channel);
            OnNewLog(args.Client, args.Channel, args.Message, checkExternalLinks: checkExternalLinks);
            return Task.CompletedTask;
        }

        public static async void EnqueueLogProcessing(DiscordClient client, DiscordChannel channel, DiscordMessage message, DiscordMember requester = null, bool checkExternalLinks = false)
        {
            try
            {
                if (!QueueLimiter.Wait(0))
                {
                    await channel.SendMessageAsync("Log processing is rate limited, try again a bit later").ConfigureAwait(false);
                    return;
                }

                bool parsedLog = false;
                var startTime = Stopwatch.StartNew();
                DiscordMessage botMsg = null;
                try
                {
                    var possibleHandlers = sourceHandlers.Select(h => h.FindHandlerAsync(message, archiveHandlers).ConfigureAwait(false).GetAwaiter().GetResult()).ToList();
                    var source = possibleHandlers.FirstOrDefault(h => h.source != null).source;
                    var fail = possibleHandlers.FirstOrDefault(h => !string.IsNullOrEmpty(h.failReason)).failReason;
                    if (source != null)
                    {
                        Config.Log.Debug($">>>>>>> {message.Id % 100} Parsing log '{source.FileName}' from {message.Author.Username}#{message.Author.Discriminator} ({message.Author.Id}) using {source.GetType().Name} ({source.SourceFileSize} bytes)...");
                        var analyzingProgressEmbed = GetAnalyzingMsgEmbed();
                        botMsg = await channel.SendMessageAsync(embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false);
                        parsedLog = true;

                        var timeout = new CancellationTokenSource(Config.LogParsingTimeout);
                        var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, Config.Cts.Token);
                        var tries = 0;
                        LogParseState result = null;
                        do
                        {
                                result = await ParseLogAsync(
                                    source,
                                    async () => botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false),
                                    combinedTokenSource.Token
                                ).ConfigureAwait(false);
                                tries++;
                        } while (result == null && !combinedTokenSource.IsCancellationRequested && tries < 3);
                        if (result == null)
                        {
                            botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: new DiscordEmbedBuilder
                                {
                                    Description = "Log analysis failed, most likely cause is a truncated/invalid log.\n" +
                                                  "Please run the game again and re-upload a new copy.",
                                    Color = Config.Colors.LogResultFailed,
                                }
                                .AddAuthor(client, message, source)
                                .Build()
                            ).ConfigureAwait(false);
                        }
                        else
                        {
                            result.ParsingTime = startTime.Elapsed;
                            try
                            {
                                if (result.Error == LogParseState.ErrorCode.PiracyDetected)
                                {
                                    var yarr = client.GetEmoji(":piratethink:", "☠");
                                    result.ReadBytes = 0;
                                    if (message.Author.IsWhitelisted(client, channel.Guild))
                                    {
                                        var piracyWarning = await result.AsEmbedAsync(client, message, source).ConfigureAwait(false);
                                        piracyWarning = piracyWarning.WithDescription("Please remove the log and issue warning to the original author of the log");
                                        botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: piracyWarning).ConfigureAwait(false);
                                        await client.ReportAsync(yarr + " Pirated Release (whitelisted by role)", message, result.SelectedFilter?.String, result.SelectedFilterContext, ReportSeverity.Low).ConfigureAwait(false);
                                    }
                                    else
                                    {
                                        var severity = ReportSeverity.Low;
                                        try
                                        {
                                            await message.DeleteAsync("Piracy detected in log").ConfigureAwait(false);
                                        }
                                        catch (Exception e)
                                        {
                                            severity = ReportSeverity.High;
                                            Config.Log.Warn(e, $"Unable to delete message in {channel.Name}");
                                        }
                                        try
                                        {
                                            botMsg = await botMsg.UpdateOrCreateMessageAsync(channel,
                                                $"{message.Author.Mention}, please read carefully:",
                                                embed: await result.AsEmbedAsync(client, message, source).ConfigureAwait(false)
                                            ).ConfigureAwait(false);
                                        }
                                        catch (Exception e)
                                        {
                                            Config.Log.Error(e, "Failed to send piracy warning");
                                        }
                                        try
                                        {
                                            await client.ReportAsync(yarr + " Pirated Release", message, result.SelectedFilter?.String, result.SelectedFilterContext, severity).ConfigureAwait(false);
                                        }
                                        catch (Exception e)
                                        {
                                            Config.Log.Error(e, "Failed to send piracy report");
                                        }
                                        if (!(message.Channel.IsPrivate || (message.Channel.Name?.Contains("spam") ?? true)))
                                            await Warnings.AddAsync(client, message, message.Author.Id, message.Author.Username, client.CurrentUser, "Pirated Release", $"{result.SelectedFilter?.String} - {result.SelectedFilterContext?.Sanitize()}");
                                    }
                                }
                                else
                                {
                                    if (result.SelectedFilter != null)
                                        await ContentFilter.PerformFilterActions(client, message, result.SelectedFilter, result.SelectedFilterContext).ConfigureAwait(false);
                                    botMsg = await botMsg.UpdateOrCreateMessageAsync(channel,
                                        requester == null ? null : $"Analyzed log from {client.GetMember(channel.Guild, message.Author)?.GetUsernameWithNickname()} by request from {requester.Mention}:",
                                        embed: await result.AsEmbedAsync(client, message, source).ConfigureAwait(false)
                                    ).ConfigureAwait(false);
                                }
                            }
                            catch (Exception e)
                            {
                                Config.Log.Error(e, "Sending log results failed");
                            }
                        }
                        return;
                    }
                    else if (!string.IsNullOrEmpty(fail)
                             && ("help".Equals(channel.Name, StringComparison.InvariantCultureIgnoreCase) || LimitedToSpamChannel.IsSpamChannel(channel)))
{'fraction_non_alphanumeric': 0.0758871217826607, 'fraction_numerical': 0.0009746156913126302, 'mean_word_length': 1.2526693942720288, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 149, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '664276', 'n_tokens_mistral': 4667, 'n_tokens_neox': 4499, 'n_words': 982}