<commit_before>// This file makes use of the LeMP 'unroll' macro to avoid copy-pasting code.
// See http://ecsharp.net/lemp/avoid-tedium-with-LeMP.html for an explanation.

#importMacros(LeMP);

using System.IO;

namespace System
{
    public static class Console
    {
        private static readonly byte[] utf8Buffer = new byte[4];
        private static readonly char[] utf16Buffer = new char[2];
        private static char cachedHighSurrogate = '\0';

        private static void WriteUtf8Bytes(byte[] str, int count)
        {
            for (int i = 0; i < count; i++)
            {
                IOPrimitives.WriteStdout(str[i]);
            }
        }

        private static void WriteUtf16CodePoint(char* begin, char* end)
        {
            int byteCount = UnicodeConverter.WriteUtf8CodePoint(
                UnicodeConverter.ReadUtf16CodePoint(ref begin, end),
                &utf8Buffer[0]);

            WriteUtf8Bytes(utf8Buffer, byteCount);
        }

        /// <summary>
        /// Writes a character to standard output.
        /// </summary>
        /// <param name="c">The character to write.</param>
        public static void Write(char c)
        {
            char highSurrogate = cachedHighSurrogate;
            if (highSurrogate == '\0')
            {
                if (char.IsHighSurrogate(c))
                {
                    cachedHighSurrogate = c;
                }
                else
                {
                    char* argBegin = &c;
                    WriteUtf16CodePoint(argBegin, argBegin + 1);
                }
            }
            else
            {
                cachedHighSurrogate = '\0';
                utf16Buffer[0] = highSurrogate;
                utf16Buffer[1] = c;
                char* argBegin = &utf16Buffer[0];
                WriteUtf16CodePoint(argBegin, argBegin + 2);
            }
        }

        public static void Flush()
        {
            char highSurrogate = cachedHighSurrogate;
            if (highSurrogate != '\0')
            {
                cachedHighSurrogate = '\0';
                char* argBegin = &highSurrogate;
                WriteUtf16CodePoint(argBegin, argBegin + 1);
            }
        }

        /// <summary>
        /// Writes a string to standard output.
        /// </summary>
        /// <param name="str">The string to write.</param>
        public static void Write(string str)
        {
            for (int i = 0; i < str.Length; i++)
            {
                Write(str[i]);
            }
        }

        /// <summary>
        /// Writes an end-of-line sequence to standard output.
        /// </summary>
        public static void WriteLine()
        {
            Write('\n');
        }

        /// <summary>
        /// Writes a character to standard output, followed by an end-of-line sequence.
        /// </summary>
        /// <param name="c">The character to write.</param>
        public static void WriteLine(char c)
        {
            Write(c);
            WriteLine();
        }

        /// <summary>
        /// Writes a string to standard output, followed by an end-of-line sequence.
        /// </summary>
        /// <param name="str">The string to write.</param>
        public static void WriteLine(string str)
        {
            Write(str);
            WriteLine();
        }

        unroll ((TYPE) in (bool, int, long, double))
        {
            /// <summary>
            /// Writes the given value to standard output.
            /// </summary>
            /// <param name="value">The value to write.</param>
            public static void Write(TYPE value)
            {
                Write(value.ToString());
            }

            /// <summary>
            /// Writes the given value to standard output, followed by an end-of-line sequence.
            /// </summary>
            /// <param name="value">The value to write.</param>
            public static void WriteLine(TYPE value)
            {
                Write(value);
                WriteLine();
            }
        }
    }
}<commit_msg>Ensure surrogate pairs match in 'Console.Write(char)'<commit_after>// This file makes use of the LeMP 'unroll' macro to avoid copy-pasting code.
// See http://ecsharp.net/lemp/avoid-tedium-with-LeMP.html for an explanation.

#importMacros(LeMP);

using System.IO;

namespace System
{
    public static class Console
    {
        private static readonly byte[] utf8Buffer = new byte[4];
        private static readonly char[] utf16Buffer = new char[2];
        private static char cachedHighSurrogate = '\0';

        private static void WriteUtf8Bytes(byte[] str, int count)
        {
            for (int i = 0; i < count; i++)
            {
                IOPrimitives.WriteStdout(str[i]);
            }
        }

        private static void WriteUtf16CodePoint(char* begin, char* end)
        {
            int byteCount = UnicodeConverter.WriteUtf8CodePoint(
                UnicodeConverter.ReadUtf16CodePoint(ref begin, end),
                &utf8Buffer[0]);

            WriteUtf8Bytes(utf8Buffer, byteCount);
        }

        /// <summary>
        /// Writes a character to standard output.
        /// </summary>
        /// <param name="c">The character to write.</param>
        public static void Write(char c)
        {
            char highSurrogate = cachedHighSurrogate;
            if (highSurrogate == '\0')
            {
                if (char.IsHighSurrogate(c))
                {
                    // We'll remember high surrogates instead of printing them
                    // right away. (They need to be matched to a low surrogate,
                    // which is yet to come.)
                    cachedHighSurrogate = c;
                }
                else
                {
                    // This character represents a single code point write it
                    // to standard output immediately.
                    char* argBegin = &c;
                    WriteUtf16CodePoint(argBegin, argBegin + 1);
                }
            }
            else
            {
                cachedHighSurrogate = '\0';
                if (char.IsLowSurrogate(c))
                {
                    // Copy both characters in the surrogate pair into the UTF-16
                    // buffer, transcribe the UTF-16 buffer's contents into UTF-8
                    // and write the result to standard output.
                    utf16Buffer[0] = highSurrogate;
                    utf16Buffer[1] = c;
                    char* argBegin = &utf16Buffer[0];
                    WriteUtf16CodePoint(argBegin, argBegin + 2);
                }
                else
                {
                    // Bad surrogate pair. Write both characters individually.
                    char* argBegin = &highSurrogate;
                    WriteUtf16CodePoint(argBegin, argBegin + 1);
                    argBegin = &c;
                    WriteUtf16CodePoint(argBegin, argBegin + 1);
                }
            }
        }

        /// <summary>
        /// Flushes the standard output buffer.
        /// </summary>
        public static void Flush()
        {
            char highSurrogate = cachedHighSurrogate;
            if (highSurrogate != '\0')
            {
                cachedHighSurrogate = '\0';
                char* argBegin = &highSurrogate;
                WriteUtf16CodePoint(argBegin, argBegin + 1);
            }
        }

        /// <summary>
        /// Writes a string to standard output.
        /// </summary>
        /// <param name="str">The string to write.</param>
        public static void Write(string str)
        {
            for (int i = 0; i < str.Length; i++)
            {
                Write(str[i]);
            }
        }

        /// <summary>
        /// Writes an end-of-line sequence to standard output.
        /// </summary>
        public static void WriteLine()
        {
            Write('\n');
        }

        /// <summary>
        /// Writes a character to standard output, followed by an end-of-line sequence.
        /// </summary>
        /// <param name="c">The character to write.</param>
        public static void WriteLine(char c)
        {
            Write(c);
            WriteLine();
        }

        /// <summary>
        /// Writes a string to standard output, followed by an end-of-line sequence.
        /// </summary>
        /// <param name="str">The string to write.</param>
        public static void WriteLine(string str)
        {
            Write(str);
            WriteLine();
        }

        unroll ((TYPE) in (bool, int, long, double))
        {
            /// <summary>
            /// Writes the given value to standard output.
            /// </summary>
            /// <param name="value">The value to write.</param>
            public static void Write(TYPE value)
            {
                Write(value.ToString());
            }

            /// <summary>
            /// Writes the given value to standard output, followed by an end-of-line sequence.
            /// </summary>
            /// <param name="value">The value to write.</param>
            public static void WriteLine(TYPE value)
            {
                Write(value);
                WriteLine();
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.10589998929221545, 'fraction_numerical': 0.009744083949030945, 'mean_word_length': 1.259313014029995, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1986604', 'n_tokens_mistral': 2487, 'n_tokens_neox': 2310, 'n_words': 669}