<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Sigma.Core.Data
{
	/// <summary>
	/// A data type that can be used for data buffers and mathematical operations. Data types are used to define buffer types to use at runtime.
	/// </summary>
	/// <typeparam name="T"></typeparam>
	public interface IDataType<T>
	{
		/// <summary>
		/// The underlying system type of this data type. 
		/// </summary>
		System.Type UnderlyingType { get; }

		/// <summary>
		/// The size of this type in bytes.
		/// </summary>
		int SizeBytes { get; }

		/// <summary>
		/// Creates a generic array with a given length and the underlying type of this data type as type. 
		/// </summary>
		/// <param name="length">The length of the array to create.</param>
		/// <returns>An array with the underlying type of this data type as type and the given length.</returns>
		T[] CreateArray(int length);
	}

	public class DataType<T> : IDataType<T>
	{
		public static readonly IDataType<double> FLOAT64 = new DataType<double>(8);
		public static readonly IDataType<float> FLOAT32 = new DataType<float>(4);

		public static readonly IDataType<byte> INT8 = new DataType<byte>(1);
		public static readonly IDataType<short> INT16 = new DataType<short>(2);
		public static readonly IDataType<int> INT32 = new DataType<int>(4);
		public static readonly IDataType<long> INT64 = new DataType<long>(8);

		public int SizeBytes
		{
			get
			{
				return SizeBytes;
			}

			private set
			{
				SizeBytes = value;
			}
		}

		public System.Type UnderlyingType
		{
			get;
		} = typeof(T);

		public T[] CreateArray(int length)
		{
			return new T[length];
		}

		public DataType(int sizeBytes)
		{
			this.SizeBytes = sizeBytes;<commit_msg>Fixed DataType access<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Sigma.Core.Data
{
	/// <summary>
	/// A data type that can be used for data buffers and mathematical operations. Data types are used to define buffer types to use at runtime.
	/// </summary>
	/// <typeparam name="T"></typeparam>
	public interface IDataType<T>
	{
		/// <summary>
		/// The underlying system type of this data type. 
		/// </summary>
		System.Type UnderlyingType { get; }

		/// <summary>
		/// The size of this type in bytes.
		/// </summary>
		int SizeBytes { get; }

		/// <summary>
		/// Creates a generic array with a given length and the underlying type of this data type as type. 
		/// </summary>
		/// <param name="length">The length of the array to create.</param>
		/// <returns>An array with the underlying type of this data type as type and the given length.</returns>
		T[] CreateArray(int length);
	}

	public abstract class DataTypes
	{
		public static readonly IDataType<double> FLOAT64 = new DataType<double>(8);
		public static readonly IDataType<float> FLOAT32 = new DataType<float>(4);

		public static readonly IDataType<byte> INT8 = new DataType<byte>(1);
		public static readonly IDataType<short> INT16 = new DataType<short>(2);
		public static readonly IDataType<int> INT32 = new DataType<int>(4);
		public static readonly IDataType<long> INT64 = new DataType<long>(8);
	}

	public class DataType<T> : DataTypes, IDataType<T>
	{
		public int SizeBytes
		{
			get
			{
				return SizeBytes;
			}

			private set
			{
				SizeBytes = value;
			}
		}

		public Type UnderlyingType
		{
			get;
		} = typeof(T);

		public T[] CreateArray(int length)
		{
			return new T[length];
		}

		public DataType(int sizeBytes)
		{
			this.SizeBytes = sizeBytes;
{'fraction_non_alphanumeric': 0.11735673156018646, 'fraction_numerical': 0.009322731011790512, 'mean_word_length': 3.2816901408450705, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3920770', 'n_tokens_mistral': 1242, 'n_tokens_neox': 1116, 'n_words': 384}