<commit_before>using System;
using System.Buffers.Text;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.InteropServices;
using System.Reflection;
using System.Security.Authentication;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;

namespace MySqlConnector.Utilities
{
	internal static class Utility
	{
		public static void Dispose<T>(ref T disposable)
			where T : class, IDisposable
		{
			if (disposable != null)
			{
				disposable.Dispose();
				disposable = null;
			}
		}

		public static string FormatInvariant(this string format, params object[] args) =>
			string.Format(CultureInfo.InvariantCulture, format, args);

#if NET45 || NET461 || NET471 || NETSTANDARD1_3 || NETSTANDARD2_0
		public static string GetString(this Encoding encoding, ReadOnlySpan<byte> span)
		{
			if (span.Length == 0)
				return "";
#if NET45
			return encoding.GetString(span.ToArray());
#else
			unsafe
			{
				fixed (byte* ptr = span)
					return encoding.GetString(ptr, span.Length);
			}
#endif
		}

		public static unsafe void GetBytes(this Encoding encoding, ReadOnlySpan<char> chars, Span<byte> bytes)
		{
			fixed (char* charsPtr = chars)
			fixed (byte* bytesPtr = bytes)
			{
				encoding.GetBytes(charsPtr, chars.Length, bytesPtr, bytes.Length);
			}
		}
#endif

#if NET461 || NET471 || NETSTANDARD2_0
		public static unsafe void Convert(this Encoder encoder, ReadOnlySpan<char> chars, Span<byte> bytes, bool flush, out int charsUsed, out int bytesUsed, out bool completed)
		{
			fixed (char* charsPtr = chars)
			fixed (byte* bytesPtr = bytes)
			{
				// MemoryMarshal.GetNonNullPinnableReference is internal, so fake it by using an invalid but non-null pointer; this
				// prevents Convert from throwing an exception when the output buffer is empty
				encoder.Convert(charsPtr, chars.Length, bytesPtr == null ? (byte*) 1 : bytesPtr, bytes.Length, flush, out charsUsed, out bytesUsed, out completed);
			}
		}
#endif
<commit_msg>Use MemoryMarshal.GetReference.<commit_after>using System;
using System.Buffers.Text;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.InteropServices;
using System.Reflection;
using System.Security.Authentication;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;

namespace MySqlConnector.Utilities
{
	internal static class Utility
	{
		public static void Dispose<T>(ref T disposable)
			where T : class, IDisposable
		{
			if (disposable != null)
			{
				disposable.Dispose();
				disposable = null;
			}
		}

		public static string FormatInvariant(this string format, params object[] args) =>
			string.Format(CultureInfo.InvariantCulture, format, args);

#if NET45 || NET461 || NET471 || NETSTANDARD1_3 || NETSTANDARD2_0
		public static string GetString(this Encoding encoding, ReadOnlySpan<byte> span)
		{
			if (span.Length == 0)
				return "";
#if NET45
			return encoding.GetString(span.ToArray());
#else
			unsafe
			{
				fixed (byte* ptr = &MemoryMarshal.GetReference(span))
					return encoding.GetString(ptr, span.Length);
			}
#endif
		}

		public static unsafe void GetBytes(this Encoding encoding, ReadOnlySpan<char> chars, Span<byte> bytes)
		{
			fixed (char* charsPtr = &MemoryMarshal.GetReference(chars))
			fixed (byte* bytesPtr = &MemoryMarshal.GetReference(bytes))
			{
				encoding.GetBytes(charsPtr, chars.Length, bytesPtr, bytes.Length);
			}
		}
#endif

#if NET461 || NET471 || NETSTANDARD2_0
		public static unsafe void Convert(this Encoder encoder, ReadOnlySpan<char> chars, Span<byte> bytes, bool flush, out int charsUsed, out int bytesUsed, out bool completed)
		{
			fixed (char* charsPtr = &MemoryMarshal.GetReference(chars))
			fixed (byte* bytesPtr = &MemoryMarshal.GetReference(bytes))
			{
				// MemoryMarshal.GetNonNullPinnableReference is internal, so fake it by using an invalid but non-null pointer; this
				// prevents Convert from throwing an exception when the output buffer is empty
				encoder.Convert(charsPtr, chars.Length, bytesPtr == null ? (byte*) 1 : bytesPtr, bytes.Length, flush, out charsUsed, out bytesUsed, out completed);
			}
		}
#endif

{'fraction_non_alphanumeric': 0.10106761565836299, 'fraction_numerical': 0.011387900355871887, 'mean_word_length': 4.329962073324905, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2220062', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1258, 'n_words': 381}