<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;

namespace iMobileDevice
{
    public static class Utf8Marshal
    {
        public static string PtrToStringUtf8(IntPtr value)
        {
            if (value == IntPtr.Zero)
            {
                return null;
            }

            ulong size = 0;
            LibiMobileDevice.Instance.Pinvoke.pinvoke_get_string_length(value, out size);
            byte[] data = new byte[size];
            Marshal.Copy(value, data, 0, (int)size);
            string result = Encoding.UTF8.GetString(data);
            return result;
        }

        public static IntPtr StringToHGlobalUtf8(string value)
        {
            if (value == null)
            {
                return IntPtr.Zero;
            }

            byte[] data = Encoding.UTF8.GetBytes(value + '\0');

            // Make sure the string is 0-terminated
            IntPtr handle = Marshal.AllocHGlobal(data.Length);
            Marshal.Copy(data, 0, handle, data.Length);

            return handle;
        }
    }
}
<commit_msg>Utf8Marshal: Allow users to provide an implementation of the ILibiMobileDeviceAPI.<commit_after>﻿using System;
using System.Runtime.InteropServices;
using System.Text;

namespace iMobileDevice
{
    /// <summary>
    /// Provides marshalling capabilities for UTF-8 strings.
    /// </summary>
    public static class Utf8Marshal
    {
        /// <summary>
        /// Converts a pointer to an UTF-8 string.
        /// </summary>
        /// <param name="value">
        /// The value to marshal.
        /// </param>
        /// <returns>
        /// A <see cref="string"/> which represents the underlying string.
        /// </returns>
        public static string PtrToStringUtf8(IntPtr value)
        {
            return PtrToStringUtf8(value, LibiMobileDevice.Instance);
        }

        /// <summary>
        /// Converts a pointer to an UTF-8 string.
        /// </summary>
        /// <param name="value">
        /// The value to marshal.
        /// </param>
        /// <param name="api">
        /// The libimobiledevice API to use when marshalling.
        /// </param>
        /// <returns>
        /// A <see cref="string"/> which represents the underlying string.
        /// </returns>
        public static string PtrToStringUtf8(IntPtr value, ILibiMobileDevice api)
        {
            if (value == IntPtr.Zero)
            {
                return null;
            }

            if (api == null)
            {
                throw new ArgumentNullException(nameof(api));
            }

            ulong size = 0;
            api.Pinvoke.pinvoke_get_string_length(value, out size);
            byte[] data = new byte[size];
            Marshal.Copy(value, data, 0, (int)size);
            string result = Encoding.UTF8.GetString(data);
            return result;
        }

        /// <summary>
        /// Converts a value to an UTF-8 string, in native memory.
        /// </summary>
        /// <param name="value">
        /// The value to encode.
        /// </param>
        /// <returns>
        /// A pointer to the UTF-8 string in unmanaged memory. You must free this pointer
        /// using <see cref="Marshal.FreeHGlobal(IntPtr)"/>.
        /// </returns>
        public static IntPtr StringToHGlobalUtf8(string value)
        {
            if (value == null)
            {
                return IntPtr.Zero;
            }

            byte[] data = Encoding.UTF8.GetBytes(value + '\0');

            // Make sure the string is 0-terminated
            IntPtr handle = Marshal.AllocHGlobal(data.Length);
            Marshal.Copy(data, 0, handle, data.Length);

            return handle;
        }
    }
}

{'fraction_non_alphanumeric': 0.11406743940990516, 'fraction_numerical': 0.007376185458377239, 'mean_word_length': 1.5432016075016746, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5264403', 'n_tokens_mistral': 1042, 'n_tokens_neox': 1010, 'n_words': 277}