<commit_before>// <copyright file="CharsetDetectorDataUnitTests.cs" company="<NAME>">
// Copyright (c) <NAME> with all rights reserved. Part of the IX Framework.
// </copyright>

using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Text;
using IX.StandardExtensions.Contracts;
using IX.StandardExtensions.Globalization;
using JetBrains.Annotations;
using Xunit;
using Xunit.Abstractions;

namespace IX.UnitTests.StandardExtensions.Globalization;

public class CharsetDetectorDataUnitTests
{
    private readonly ITestOutputHelper outputHelper;

    public CharsetDetectorDataUnitTests(ITestOutputHelper outputHelper)
    {
        Requires.NotNull(out this.outputHelper, outputHelper);
    }

    public static IEnumerable<object[]> AllTestFiles()
    {
        var stream = Requires.NotNull(Assembly.GetExecutingAssembly()
            .GetManifestResourceStream("IX.UnitTests.StandardExtensions.Globalization.data.zip"));

        ZipArchive za = new ZipArchive(
            stream,
            ZipArchiveMode.Read,
            true);

        return za.Entries.Select(
            p =>
            {
                var folder = Path.GetDirectoryName(p.FullName)?.Split('(').First().Trim() ?? "ascii";
                var name = p.Name;
                return new TestCase(
                    p.FullName,
                    name,
                    folder);
            }).Select(p => new object[] { p }).ToList();
    }

    [Theory]
    [MemberData(nameof(AllTestFiles))]
    public void TestFile(TestCase testCase)
    {
        Encoding expectedEncoding = CharsetDetectionEngine.GetCompatibleEncodingByShortName(testCase.ExpectedEncoding);

        var stream = Requires.NotNull(Assembly.GetExecutingAssembly()
            .GetManifestResourceStream("IX.UnitTests.StandardExtensions.Globalization.data.zip"));

        ZipArchive za = new ZipArchive(
            stream,
            ZipArchiveMode.Read,
            true);

        var result = new CharsetDetectionEngine().Read(za.Entries.First(p => p.FullName == testCase.InputFile).Open());

        Assert.NotNull(result.Encoding);
        this.outputHelper.WriteLine($"- {testCase.FileName} ({testCase.ExpectedEncoding}) -> {result.Encoding.WebName}");
        Assert.Equal(expectedEncoding, result.Encoding);
    }

    public class TestCase : IXunitSerializable
    {
        [UsedImplicitly]
        public TestCase()<commit_msg>Extra unit tests for sync and async.<commit_after>// <copyright file="CharsetDetectorDataUnitTests.cs" company="<NAME>">
// Copyright (c) <NAME> with all rights reserved. Part of the IX Framework.
// </copyright>

using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using IX.StandardExtensions.Contracts;
using IX.StandardExtensions.Extensions;
using IX.StandardExtensions.Globalization;
using JetBrains.Annotations;
using Xunit;
using Xunit.Abstractions;

namespace IX.UnitTests.StandardExtensions.Globalization;

public class CharsetDetectorDataUnitTests
{
    private readonly ITestOutputHelper outputHelper;

    public CharsetDetectorDataUnitTests(ITestOutputHelper outputHelper)
    {
        Requires.NotNull(out this.outputHelper, outputHelper);
    }

    public static IEnumerable<object[]> AllTestFiles()
    {
        var stream = Requires.NotNull(Assembly.GetExecutingAssembly()
            .GetManifestResourceStream("IX.UnitTests.StandardExtensions.Globalization.data.zip"));

        ZipArchive za = new ZipArchive(
            stream,
            ZipArchiveMode.Read,
            true);

        return za.Entries.Select(
            p =>
            {
                var folder = Path.GetDirectoryName(p.FullName)?.Split('(').First().Trim() ?? "ascii";
                var name = p.Name;
                return new TestCase(
                    p.FullName,
                    name,
                    folder);
            }).Select(p => new object[] { p }).ToList();
    }

    [Theory(DisplayName = "Synchronous encoding stream tests")]
    [MemberData(nameof(AllTestFiles))]
    public void TestStreamSync(TestCase testCase)
    {
        Encoding expectedEncoding = CharsetDetectionEngine.GetCompatibleEncodingByShortName(testCase.ExpectedEncoding);

        var stream = Requires.NotNull(Assembly.GetExecutingAssembly()
            .GetManifestResourceStream("IX.UnitTests.StandardExtensions.Globalization.data.zip"));

        ZipArchive za = new ZipArchive(
            stream,
            ZipArchiveMode.Read,
            true);

        var result = new CharsetDetectionEngine().Read(za.Entries.First(p => p.FullName == testCase.InputFile).Open());

        Assert.NotNull(result.Encoding);
        this.outputHelper.WriteLine($"- {testCase.FileName} ({testCase.ExpectedEncoding}) -> {result.Encoding.WebName}");
        Assert.Equal(expectedEncoding, result.Encoding);
    }

    [Theory(DisplayName = "Asynchronous encoding stream tests")]
    [MemberData(nameof(AllTestFiles))]
    public async Task TestStreamAsync(TestCase testCase, CancellationToken cancellationToken = default)
    {
        Encoding expectedEncoding = CharsetDetectionEngine.GetCompatibleEncodingByShortName(testCase.ExpectedEncoding);

        var stream = Requires.NotNull(Assembly.GetExecutingAssembly()
            .GetManifestResourceStream("IX.UnitTests.StandardExtensions.Globalization.data.zip"));

        ZipArchive za = new ZipArchive(
            stream,
            ZipArchiveMode.Read,
            true);

        var result = await new CharsetDetectionEngine().ReadAsync(za.Entries.First(p => p.FullName == testCase.InputFile).Open(), cancellationToken);

        Assert.NotNull(result.Encoding);
        this.outputHelper.WriteLine($"- {testCase.FileName} ({testCase.ExpectedEncoding}) -> {result.Encoding.WebName}");
        Assert.Equal(expectedEncoding, result.Encoding);
    }

    [Theory(DisplayName = "Synchronous encoding buffer tests")]
    [MemberData(nameof(AllTestFiles))]
    public void TestBufferSync(TestCase testCase)
    {
        Encoding expectedEncoding = CharsetDetectionEngine.GetCompatibleEncodingByShortName(testCase.ExpectedEncoding);

        var stream = Requires.NotNull(Assembly.GetExecutingAssembly()
            .GetManifestResourceStream("IX.UnitTests.StandardExtensions.Globalization.data.zip"));

        ZipArchive za = new ZipArchive(
            stream,
            ZipArchiveMode.Read,
            true);

        var result = new CharsetDetectionEngine().Read(za.Entries.First(p => p.FullName == testCase.InputFile).Open().ReadAllBytes());

        Assert.NotNull(result.Encoding);
        this.outputHelper.WriteLine($"- {testCase.FileName} ({testCase.ExpectedEncoding}) -> {result.Encoding.WebName}");
        Assert.Equal(expectedEncoding, result.Encoding);
    }

    [Theory(DisplayName = "Asynchronous encoding buffer tests")]
    [MemberData(nameof(AllTestFiles))]
    public async Task TestBufferAsync(TestCase testCase, CancellationToken cancellationToken = default)
    {
        Encoding expectedEncoding = CharsetDetectionEngine.GetCompatibleEncodingByShortName(testCase.ExpectedEncoding);

        var stream = Requires.NotNull(Assembly.GetExecutingAssembly()
            .GetManifestResourceStream("IX.UnitTests.StandardExtensions.Globalization.data.zip"));

        ZipArchive za = new ZipArchive(
            stream,
            ZipArchiveMode.Read,
            true);

        var result = await new CharsetDetectionEngine().ReadAsync(await za.Entries.First(p => p.FullName == testCase.InputFile).Open().ReadAllBytesAsync(cancellationToken), cancellationToken);

        Assert.NotNull(result.Encoding);
        this.outputHelper.WriteLine($"- {testCase.FileName} ({testCase.ExpectedEncoding}) -> {result.Encoding.WebName}");
        Assert.Equal(expectedEncoding, result.Encoding);
    }

    public class TestCase : IXunitSerializable
    {
        [UsedImplicitly]
        public TestCase()
{'fraction_non_alphanumeric': 0.10500308832612724, 'fraction_numerical': 0.0, 'mean_word_length': 3.1517948717948716, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7324427', 'n_tokens_mistral': 2168, 'n_tokens_neox': 2094, 'n_words': 399}