<commit_before>using System;
using System.Runtime.InteropServices;
using System.Collections.Generic;

using bh_bool = System.Boolean;
using bh_int8 = System.SByte;
using bh_uint8 = System.Byte;
using bh_int16 = System.Int16;
using bh_uint16 = System.UInt16;
using bh_int32 = System.Int32;
using bh_uint32 = System.UInt32;
using bh_int64 = System.Int64;
using bh_uint64 = System.UInt64;
using bh_float32 = System.Single;
using bh_float64 = System.Double;
using bh_complex64 = NumCIL.Complex64.DataType;
using bh_complex128 = System.Numerics.Complex;

namespace NumCIL.Bohrium2
{
    /// <summary>
    /// Utility class that keeps track of all pinned memory allocations,
    /// which is used for non-bohrium tracked arrays
    /// </summary>
    public static class PinnedArrayTracker
    {
        /// <summary>
        /// The lookup table with all pinned items
        /// </summary>
        private static Dictionary<Array, Tuple<IntPtr, GCHandle, IDisposable>> _allocations = new Dictionary<Array, Tuple<IntPtr, GCHandle, IDisposable>>();

        /// <summary>
        /// The lock object that protects access to the pinner
        /// </summary>
        private static object _lock = new object();
    
        /// <summary>
        /// Pins the array.
        /// </summary>
        /// <param name="item">Item.</param>
        /// <typeparam name="T">The 1st type parameter.</typeparam>
        public static IntPtr PinArray<T>(T[] item)
        {
            lock (_lock)
            {
                Tuple<IntPtr, GCHandle, IDisposable> r;
                if (_allocations.TryGetValue(item, out r))
                    return r.Item1;

                var handle = GCHandle.Alloc(item, GCHandleType.Pinned);
                if (typeof(T) == typeof(bh_bool))
                {
                    var p = new PInvoke.bh_base_bool8_p(handle.AddrOfPinnedObject(), item.Length);
                    r = new Tuple<IntPtr, GCHandle, IDisposable>(
                        p.ToPointer(),
                        handle,
                        p
                    );
                }
                else if (typeof(T) == typeof(bh_int8))
                {
                    var p = new PInvoke.bh_base_int8_p(handle.AddrOfPinnedObject(), item.Length);
                    r = new Tuple<IntPtr, GCHandle, IDisposable>(<commit_msg>Added support for uint8 in CIL<commit_after>using System;
using System.Runtime.InteropServices;
using System.Collections.Generic;

using bh_bool = System.Boolean;
using bh_int8 = System.SByte;
using bh_uint8 = System.Byte;
using bh_int16 = System.Int16;
using bh_uint16 = System.UInt16;
using bh_int32 = System.Int32;
using bh_uint32 = System.UInt32;
using bh_int64 = System.Int64;
using bh_uint64 = System.UInt64;
using bh_float32 = System.Single;
using bh_float64 = System.Double;
using bh_complex64 = NumCIL.Complex64.DataType;
using bh_complex128 = System.Numerics.Complex;

namespace NumCIL.Bohrium2
{
    /// <summary>
    /// Utility class that keeps track of all pinned memory allocations,
    /// which is used for non-bohrium tracked arrays
    /// </summary>
    public static class PinnedArrayTracker
    {
        /// <summary>
        /// The lookup table with all pinned items
        /// </summary>
        private static Dictionary<Array, Tuple<IntPtr, GCHandle, IDisposable>> _allocations = new Dictionary<Array, Tuple<IntPtr, GCHandle, IDisposable>>();

        /// <summary>
        /// The lock object that protects access to the pinner
        /// </summary>
        private static object _lock = new object();
    
        /// <summary>
        /// Pins the array.
        /// </summary>
        /// <param name="item">Item.</param>
        /// <typeparam name="T">The 1st type parameter.</typeparam>
        public static IntPtr PinArray<T>(T[] item)
        {
            lock (_lock)
            {
                Tuple<IntPtr, GCHandle, IDisposable> r;
                if (_allocations.TryGetValue(item, out r))
                    return r.Item1;

                var handle = GCHandle.Alloc(item, GCHandleType.Pinned);
                if (typeof(T) == typeof(bh_bool))
                {
                    var p = new PInvoke.bh_base_bool8_p(handle.AddrOfPinnedObject(), item.Length);
                    r = new Tuple<IntPtr, GCHandle, IDisposable>(
                        p.ToPointer(),
                        handle,
                        p
                    );
                }
                else if (typeof(T) == typeof(bh_uint8))
                {
                    var p = new PInvoke.bh_base_uint8_p(handle.AddrOfPinnedObject(), item.Length);
                    r = new Tuple<IntPtr, GCHandle, IDisposable>(
                        p.ToPointer(),
                        handle,
                        p
                    );
                }
                else if (typeof(T) == typeof(bh_int8))
                {
                    var p = new PInvoke.bh_base_int8_p(handle.AddrOfPinnedObject(), item.Length);
                    r = new Tuple<IntPtr, GCHandle, IDisposable>(
{'fraction_non_alphanumeric': 0.10512820512820513, 'fraction_numerical': 0.017554240631163707, 'mean_word_length': 1.8032061912658928, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7384165', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1451, 'n_words': 363}