<commit_before>// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using Xunit;

namespace System.Text.Utf8.Tests
{
    public class Utf8EncoderTests
    {
        private const ushort Utf16HighSurrogateFirstCodePoint = 0xD800;
        private const ushort Utf16HighSurrogateLastCodePoint = 0xDBFF;
        private const ushort Utf16LowSurrogateFirstCodePoint = 0xDC00;
        private const ushort Utf16LowSurrogateLastCodePoint = 0xDFFF;

        private const byte Utf8OneByteLastCodePoint = 0x7F;
        private const ushort Utf8TwoBytesLastCodePoint = 0x7FF;
        private const ushort Utf8ThreeBytesLastCodePoint = 0xFFFF;

        private const int CharLength = 999;

        [Fact]
        public void TestEncodingInputBufferEmpty()
        {
            string unicodeString = "";
            ReadOnlySpan<char> characters = unicodeString.Slice();
            int expectedBytesWritten = GetByteCount(characters);
            char[] charArray = characters.ToArray();
            byte[] utf8BufferExpected = new byte[Encoding.UTF8.GetByteCount(charArray)];

            Encoding.UTF8.GetBytes(charArray, 0, characters.Length, utf8BufferExpected, 0);
            byte[] utf8Buffer = new byte[expectedBytesWritten + 25];
            Span<byte> span = new Span<byte>(utf8Buffer);
            int charactersConsumed;
            int bytesWritten;
            Assert.True(TextEncoder.Utf8.TryEncode(characters, span, out charactersConsumed, out bytesWritten));

            Assert.Equal(0, charactersConsumed);
            Assert.Equal(expectedBytesWritten, bytesWritten);
        }

        [Fact]
        public void TestEncodingOutputBufferEmpty()
        {
            string unicodeString = GenerateValidString(CharLength, 0, Utf8ThreeBytesLastCodePoint);
            ReadOnlySpan<char> characters = unicodeString.Slice();
            char[] charArray = characters.ToArray();
            byte[] utf8BufferExpected = new byte[Encoding.UTF8.GetByteCount(charArray)];

            Encoding.UTF8.GetBytes(charArray, 0, characters.Length, utf8BufferExpected, 0);
            byte[] utf8Buffer = new byte[0];
            Span<byte> span = new Span<byte>(utf8Buffer);
            int charactersConsumed;
            int bytesWritten;
            Assert.False(TextEncoder.Utf8.TryEncode(characters, span, out charactersConsumed, out bytesWritten));

            Assert.Equal(0, charactersConsumed);
            Assert.Equal(0, bytesWritten);
        }

        [Fact]
        public void TestEncodingInputBufferMaxSize()
        {
            int maxCharArraySize = int.MaxValue / 4; // TODO: want this to be int.MaxValue (need gcAllowVeryLargeObjects = true)
            char[] charArray = new char[maxCharArraySize];

            ReadOnlySpan<char> characters = new ReadOnlySpan<char>(charArray);
            int expectedBytesWritten = GetByteCount(characters);
            byte[] utf8Buffer = new byte[expectedBytesWritten];
            Span<byte> span = new Span<byte>(utf8Buffer);
            int charactersConsumed;
            int bytesWritten;
            Assert.True(TextEncoder.Utf8.TryEncode(characters, span, out charactersConsumed, out bytesWritten));

            Assert.Equal(maxCharArraySize, charactersConsumed);
            Assert.Equal(expectedBytesWritten, bytesWritten);
        }

        [Fact]
        public void TestEncodingOutputBufferMaxSize()
        {
            int maxByteArraySize = int.MaxValue / 2; // TODO: want this to be int.MaxValue (need gcAllowVeryLargeObjects = true)
            string unicodeString = GenerateValidString(CharLength, 0, Utf8ThreeBytesLastCodePoint);
            ReadOnlySpan<char> characters = unicodeString.Slice();
            int expectedBytesWritten = GetByteCount(characters);
            char[] charArray = characters.ToArray();
            byte[] utf8BufferExpected = new byte[Encoding.UTF8.GetByteCount(charArray)];

            Encoding.UTF8.GetBytes(charArray, 0, characters.Length, utf8BufferExpected, 0);
            byte[] utf8Buffer = new byte[maxByteArraySize];
            Span<byte> span = new Span<byte>(utf8Buffer);
            int charactersConsumed;
            int bytesWritten;
            Assert.True(TextEncoder.Utf8.TryEncode(characters, span, out charactersConsumed, out bytesWritten));

            byte[] utf8BufferActual = span.ToArray();

            Assert.Equal(unicodeString.Length, charactersConsumed);
            Assert.Equal(utf8BufferExpected.Length, bytesWritten);
            Assert.Equal(utf8BufferExpected.Length, expectedBytesWritten);

            for (int i = 0; i < utf8BufferExpected.Length; i++)
            {
                Assert.Equal(utf8BufferExpected[i], utf8BufferActual[i]);
            }<commit_msg>Removing Memory Intensive Tests (#1212)<commit_after>// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using Xunit;

namespace System.Text.Utf8.Tests
{
    public class Utf8EncoderTests
    {
        private const ushort Utf16HighSurrogateFirstCodePoint = 0xD800;
        private const ushort Utf16HighSurrogateLastCodePoint = 0xDBFF;
        private const ushort Utf16LowSurrogateFirstCodePoint = 0xDC00;
        private const ushort Utf16LowSurrogateLastCodePoint = 0xDFFF;

        private const byte Utf8OneByteLastCodePoint = 0x7F;
        private const ushort Utf8TwoBytesLastCodePoint = 0x7FF;
        private const ushort Utf8ThreeBytesLastCodePoint = 0xFFFF;

        private const int CharLength = 999;

        [Fact]
        public void TestEncodingInputBufferEmpty()
        {
            string unicodeString = "";
            ReadOnlySpan<char> characters = unicodeString.Slice();
            int expectedBytesWritten = GetByteCount(characters);
            char[] charArray = characters.ToArray();
            byte[] utf8BufferExpected = new byte[Encoding.UTF8.GetByteCount(charArray)];

            Encoding.UTF8.GetBytes(charArray, 0, characters.Length, utf8BufferExpected, 0);
            byte[] utf8Buffer = new byte[expectedBytesWritten + 25];
            Span<byte> span = new Span<byte>(utf8Buffer);
            int charactersConsumed;
            int bytesWritten;
            Assert.True(TextEncoder.Utf8.TryEncode(characters, span, out charactersConsumed, out bytesWritten));

            Assert.Equal(0, charactersConsumed);
            Assert.Equal(expectedBytesWritten, bytesWritten);
        }

        [Fact]
        public void TestEncodingOutputBufferEmpty()
        {
            string unicodeString = GenerateValidString(CharLength, 0, Utf8ThreeBytesLastCodePoint);
            ReadOnlySpan<char> characters = unicodeString.Slice();
            char[] charArray = characters.ToArray();
            byte[] utf8BufferExpected = new byte[Encoding.UTF8.GetByteCount(charArray)];

            Encoding.UTF8.GetBytes(charArray, 0, characters.Length, utf8BufferExpected, 0);
            byte[] utf8Buffer = new byte[0];
            Span<byte> span = new Span<byte>(utf8Buffer);
            int charactersConsumed;
            int bytesWritten;
            Assert.False(TextEncoder.Utf8.TryEncode(characters, span, out charactersConsumed, out bytesWritten));

            Assert.Equal(0, charactersConsumed);
            Assert.Equal(0, bytesWritten);
{'fraction_non_alphanumeric': 0.08206767918998135, 'fraction_numerical': 0.018518518518518517, 'mean_word_length': 2.7535, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6815615', 'n_tokens_mistral': 2073, 'n_tokens_neox': 1934, 'n_words': 478}