<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;

namespace Bjd.Memory
{
    public static class MemoryExtensions
    {

        public static CharsData ToAsciiCharsData(this BufferData data)
        {
            var ascii = System.Text.Encoding.ASCII;
            var size = ascii.GetCharCount(data.Data, 0, data.DataSize);
            var chars = CharsPool.GetMaximum(size);
            chars.DataSize = ascii.GetChars(data.Data, 0, data.DataSize, chars.Data, 0);
            return chars;
        }

        public static CharsData ToCharsData(this StringBuilder sb)
        {
            var len = sb.Length;
            var chars = CharsPool.GetMaximum(len);
            sb.CopyTo(0, chars.Data, 0, sb.Length);
            chars.DataSize = len;
            return chars;
        }

        public static CharsData ToCharsData(this string text)
        {
            var len = text.Length;
            var chars = CharsPool.GetMaximum(len);
            text.CopyTo(0, chars.Data, chars.DataSize, len);
            chars.DataSize = len;
            return chars;
        }

        public static CharsData ToCharsData(this string[] array)
        {
            var len = array.Sum(_ => _?.Length ?? 0);
            var chars = CharsPool.GetMaximum(len);
            foreach (var msg in array)
            {
                if (msg == null) continue;
                var l = msg.Length;
                msg.CopyTo(0, chars.Data, chars.DataSize, l);
                chars.DataSize += l;
            }
            return chars;
        }

        public static void Append(this CharsData chars, string appendText)
        {
            var len = appendText.Length;
            appendText.CopyTo(0, chars.Data, chars.DataSize, len);
            chars.DataSize += len;
        }

        public static void Append(this CharsData chars, char[] appendText)
        {
            var len = appendText.Length;
            //appendText.CopyTo(0, chars.Data, chars.DataSize, len);
            Buffer.BlockCopy(appendText, 0, chars.Data, chars.DataSize, len);
            chars.DataSize += len;<commit_msg>fix BufferCopy<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;

namespace Bjd.Memory
{
    public static class MemoryExtensions
    {

        public static CharsData ToAsciiCharsData(this BufferData data)
        {
            var ascii = System.Text.Encoding.ASCII;
            var size = ascii.GetCharCount(data.Data, 0, data.DataSize);
            var chars = CharsPool.GetMaximum(size);
            chars.DataSize = ascii.GetChars(data.Data, 0, data.DataSize, chars.Data, 0);
            return chars;
        }

        public static CharsData ToCharsData(this StringBuilder sb)
        {
            var len = sb.Length;
            var chars = CharsPool.GetMaximum(len);
            sb.CopyTo(0, chars.Data, 0, sb.Length);
            chars.DataSize = len;
            return chars;
        }

        public static CharsData ToCharsData(this string text)
        {
            var len = text.Length;
            var chars = CharsPool.GetMaximum(len);
            text.CopyTo(0, chars.Data, chars.DataSize, len);
            chars.DataSize = len;
            return chars;
        }

        public static CharsData ToCharsData(this string[] array)
        {
            var len = array.Sum(_ => _?.Length ?? 0);
            var chars = CharsPool.GetMaximum(len);
            foreach (var msg in array)
            {
                if (msg == null) continue;
                var l = msg.Length;
                msg.CopyTo(0, chars.Data, chars.DataSize, l);
                chars.DataSize += l;
            }
            return chars;
        }

        public static void Append(this CharsData chars, string appendText)
        {
            var len = appendText.Length;
            appendText.CopyTo(0, chars.Data, chars.DataSize, len);
            chars.DataSize += len;
        }

        public static void Append(this CharsData chars, char[] appendText)
        {
            var len = appendText.Length;
            //appendText.CopyTo(0, chars.Data, chars.DataSize, len);
            Buffer.BlockCopy(appendText, 0, chars.Data, chars.DataSize * 2, len * 2);
            chars.DataSize += len;
{'fraction_non_alphanumeric': 0.09528151356279918, 'fraction_numerical': 0.0054707089126966035, 'mean_word_length': 1.752823086574655, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6943586', 'n_tokens_mistral': 1313, 'n_tokens_neox': 1201, 'n_words': 310}