<commit_before>﻿//
// Copyright (c) Antmicro
//
// Full license details are defined in the 'LICENSE' file.
//
using System;
using System.Collections.Generic;
using System.Text;

namespace Terminal.Vt100
{
    public sealed class ByteUtf8Decoder
    {
        public ByteUtf8Decoder(Action<string> charDecodedCallback)
        {
            this.charDecodedCallback = charDecodedCallback;
            utfBytes = new byte[4];
            utf8Decoder = Encoding.UTF8.GetDecoder();
            result = new char[2];
        }

        public void Feed(byte b)
        {
            if(currentCount == 0)
            {
                currentCount = GetUtf8ByteCount(b);
            }
            utfBytes[currentIndex++] = b;
            if(currentIndex == currentCount)
            {
                var utf16CharCount = utf8Decoder.GetChars(utfBytes, 0, currentCount, result, 0, true);
                var resultAsString = new string(result, 0, utf16CharCount);
                charDecodedCallback(resultAsString);
                currentCount = 0;
                currentIndex = 0;
            }
        }

        private static int GetUtf8ByteCount(byte leadingByte)
        {
            if(leadingByte < 128)
            {
                return 1;
            }
            if(((leadingByte ^ 0xC0) >> 5) == 0)
            {
                return 2;
            }
            if(((leadingByte ^ 0xE0) >> 4) == 0)
            {
                return 3;
            }
            return 4;
        }

        private int currentCount;
        private int currentIndex;
        private readonly byte[] utfBytes;
        private readonly Action<string> charDecodedCallback;
        private readonly char[] result;
        private readonly System.Text.Decoder utf8Decoder;
    }
}

<commit_msg>Be prepared for multichar input.<commit_after>﻿//
// Copyright (c) Antmicro
//
// Full license details are defined in the 'LICENSE' file.
//
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text;

namespace Terminal.Vt100
{
    public sealed class ByteUtf8Decoder
    {
        public ByteUtf8Decoder(Action<string> charDecodedCallback)
        {
            this.charDecodedCallback = charDecodedCallback;
            utfBytes = new byte[4];
            utf8Decoder = Encoding.UTF8.GetDecoder();
            result = new char[4];
        }

        public void Feed(byte b)
        {
            if(currentCount == 0)
            {
                currentCount = GetUtf8ByteCount(b);
            }
            utfBytes[currentIndex++] = b;
            if(currentIndex == currentCount)
            {
                var utf16CharCount = utf8Decoder.GetChars(utfBytes, 0, currentCount, result, 0, true);
                var resultAsString = new string(result, 0, utf16CharCount);
                var textElements = StringInfo.GetTextElementEnumerator(resultAsString);
                while(textElements.MoveNext())
                {
                    charDecodedCallback((string)textElements.Current);
                }
                currentCount = 0;
                currentIndex = 0;
            }
        }

        private static int GetUtf8ByteCount(byte leadingByte)
        {
            if(leadingByte < 128)
            {
                return 1;
            }
            if(((leadingByte ^ 0xC0) >> 5) == 0)
            {
                return 2;
            }
            if(((leadingByte ^ 0xE0) >> 4) == 0)
            {
                return 3;
            }
            return 4;
        }

        private int currentCount;
        private int currentIndex;
        private readonly byte[] utfBytes;
        private readonly Action<string> charDecodedCallback;
        private readonly char[] result;
        private readonly System.Text.Decoder utf8Decoder;
    }
}


{'fraction_non_alphanumeric': 0.08572922111866178, 'fraction_numerical': 0.01986408782017773, 'mean_word_length': 1.5361166335321406, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6504894', 'n_tokens_mistral': 1054, 'n_tokens_neox': 971, 'n_words': 238}