<commit_before>/*
	Dataphor
	© Copyright 2000-2008 Alphora
	This file is licensed under a modified BSD-license which can be found here: http://dataphor.org/dataphor_license.txt
*/

#define USEDATATYPESINNATIVEROW // Determines whether or not the native row tracks the data type of the values it contains

using System;
using System.IO;
using System.Text;
using System.Collections;

namespace Alphora.Dataphor.DAE.Runtime.Data
{
	#if USEDATATYPESINNATIVEROW
	using Alphora.Dataphor.DAE.Language.D4;
	using Alphora.Dataphor.DAE.Compiling;
	#endif
	using Alphora.Dataphor.DAE.Server;
	using Alphora.Dataphor.DAE.Streams;

	/*
		Row ->
		
			Rows store data as an array of objects.
			Data values can be stored in one of two ways inside a row.
			If the Data value is a StreamID, it is considered a non-native
			value and enables deferred access values by effectively storing
			only the StreamID of the data.  Otherwise, the value is considered
			a native value and is stored in its host representation.
	*/
	
	public class NativeRow : System.Object
	{
		public NativeRow(int AColumnCount) : base()
		{
			#if USEDATATYPESINNATIVEROW
			FDataTypes = new Schema.IDataType[AColumnCount];
			#endif
			FValues = new object[AColumnCount];
		}
		
		#if USEDATATYPESINNATIVEROW
		private Schema.IDataType[] FDataTypes;
		public Schema.IDataType[] DataTypes { get { return FDataTypes; } }
		#endif
		
		private object[] FValues;
		public object[] Values { get { return FValues; } }
		
		public BitArray ModifiedFlags;
	}
	
	/// <remarks>
	/// Provides a fixed length buffer for cell values with overflow management built in.
	/// Used in conjunction with the CellValueStream, provides transparent variable length
	/// value storage in a fixed length buffer.
	/// </remarks>    
	public class Row : DataValue
	{
		public Row(IValueManager AManager, Schema.IRowType ADataType) : base(AManager, ADataType)
		{
			FRow = new NativeRow(DataType.Columns.Count);
			#if USEDATATYPESINNATIVEROW
			for (int LIndex = 0; LIndex < DataType.Columns.Count; LIndex++)
				FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
			#endif
			ValuesOwned = true;
		}

		// The given object[] is assumed to contains values of the appropriate type for the given data type
		public Row(IValueManager AManager, Schema.IRowType ADataType, NativeRow ARow) : base(AManager, ADataType)
		{
			FRow = ARow;
			ValuesOwned = false;
		}

		protected override void Dispose(bool ADisposing)
		{
			if (FRow != null)
			{
				if (ValuesOwned)
					ClearValues();
				FRow = null;
			}
			base.Dispose(ADisposing);
		}
		
		public new Schema.IRowType DataType { get { return (Schema.IRowType)base.DataType; } }
		
		private NativeRow FRow;
		
		public override object AsNative
		{
			get { return FRow; }
			set 
			{
				if (FRow != null)
					ClearValues();
				FRow = (NativeRow)value; 
			}
		}
		
		private object[] FWriteList;
		private DataValue[] FElementWriteList;

		/*
			Row Value Format ->
			
				00 -> 0 - Indicates that non-native values are stored as a StreamID 1 - Indicates non-native values are stored inline
				01-XX -> Value for each attribute in the row
				
			Row Attribute Format ->

				There are five possibilities for an attribute ->			
					Nil Native
					Nil Non-Native (StreamID.Null)
					Standard Native
					Standard Non-Native
					Specialized Native
					Specialized Non-Native
				
					For non-native values, the value will be expanded or not dependending on the expanded setting for the row value

				00	-> 0 - 5
					0 if the row contains a native nil for this attribute - no data follows
					1 if the row contains a non-native nil for this attribute - no data follows
					2 if the row contains a native value of the data type of the attribute
						01-04 -> Length of the physical representation of this value
						05-XX -> The physical representation of this value
					3 if the row contains a non-native value of the data type of the attribute
						01-04 -> Length of the physical representation of this value
						05-XX -> The physical representation of this value (expanded based on the expanded setting for the list value)
					4 if the row contains a native value of some specialization of the data type of the attribute
						01-XX -> The data type name of this value, stored using a StringConveyor
						XX+1-XX+5 -> The length of the physical representation of this value
						XX+6-YY -> The physical representation of this value
					5 if the row contains a non-native value of some specialization of the data type of the attribute
						01-XX -> The data type name of this value, stored using a StringConveyor
						XX+1-XX+5 -> The length of the physical representation of this value
						XX+6-YY -> The physical representation of this value (expanded based on the expanded setting for the list value)
		*/
		public override int GetPhysicalSize(bool AExpandStreams)
		{
			int LSize = 1; // write the value indicator
			if (!IsNil)
			{
				LSize += 1; // write the extended streams indicator
				FWriteList = new object[DataType.Columns.Count]; // list for saving the sizes or streams of each attribute in the row
				FElementWriteList = new DataValue[DataType.Columns.Count]; // list for saving host representations of values between the GetPhysicalSize and WriteToPhysical calls
				Stream LStream;
				StreamID LStreamID;
				Schema.ScalarType LScalarType;
				Streams.Conveyor LConveyor;
				DataValue LElement;
				int LElementSize;
				for (int LIndex = 0; LIndex < FWriteList.Length; LIndex++)
				{
					LSize += sizeof(byte); // write a value indicator
					if (FRow.Values[LIndex] != null)
					{
						#if USEDATATYPESINNATIVEROW
						if (!DataType.Columns[LIndex].DataType.Equals(FRow.DataTypes[LIndex]))
							LSize += Manager.GetConveyor(Manager.DataTypes.SystemString).GetSize(FRow.DataTypes[LIndex].Name); // write the name of the data type of the value
						#endif
						/*							
						LElement = DataValue.FromNativeRow(Manager, DataType, FRow, LIndex);
						FElementWriteList[LIndex] = LElement;
						LElementSize = LElement.GetPhysicalSize(AExpandStreams);
						FWriteList[LIndex] = LElementSize;
						LSize += sizeof(int) + LElementSize;
						*/
						#if USEDATATYPESINNATIVEROW
						LScalarType = FRow.DataTypes[LIndex] as Schema.ScalarType;
						#else
						LScalarType = DataType.Columns[LIndex].DataType as Schema.ScalarType;
						#endif
						if ((LScalarType != null) && !LScalarType.IsCompound)
						{
							if (FRow.Values[LIndex] is StreamID)
							{
								// If this is a non-native scalar
								LStreamID = (StreamID)FRow.Values[LIndex];
								if (AExpandStreams)
								{
									if (LStreamID != StreamID.Null)
									{
										LStream = Manager.StreamManager.Open((StreamID)FRow.Values[LIndex], LockMode.Exclusive);
										FWriteList[LIndex] = LStream;
										LSize += sizeof(int) + (int)LStream.Length;
									}
								}
								else
								{
									if (LStreamID != StreamID.Null)
									{
										LElementSize = sizeof(long);
										FWriteList[LIndex] = LElementSize;
										LSize += LElementSize;
									}
								}
							}
							else
							{
								LConveyor = Manager.GetConveyor(LScalarType);
								if (LConveyor.IsStreaming)
								{
									LStream = new MemoryStream(64);
									FWriteList[LIndex] = LStream;
									LConveyor.Write(FRow.Values[LIndex], LStream);
									LStream.Position = 0;
									LSize += sizeof(int) + (int)LStream.Length;
								}
								else
								{
									LElementSize = LConveyor.GetSize(FRow.Values[LIndex]);
									FWriteList[LIndex] = LElementSize;
									LSize += sizeof(int) + LElementSize;;
								}
							}
						}
						else
						{
							LElement = DataValue.FromNativeRow(Manager, DataType, FRow, LIndex);
							FElementWriteList[LIndex] = LElement;
							LElementSize = LElement.GetPhysicalSize(AExpandStreams);
							FWriteList[LIndex] = LElementSize;
							LSize += sizeof(int) + LElementSize;
						}						
					}
				}
			}
			return LSize;
		}
		
		public override void WriteToPhysical(byte[] ABuffer, int AOffset, bool AExpandStreams)
		{
			if (FWriteList == null)
				throw new RuntimeException(RuntimeException.Codes.UnpreparedWriteToPhysicalCall);
				
			ABuffer[AOffset] = (byte)(IsNil ? 0 : 1); // Write the value indicator
			AOffset++;
			
			if (!IsNil)
			{
				ABuffer[AOffset] = (byte)(AExpandStreams ? 1 : 0); // Write the expanded streams indicator
				AOffset++;
					
				#if USEDATATYPESINNATIVEROW
				Streams.Conveyor LStringConveyor = null;
				#endif
				Streams.Conveyor LInt64Conveyor = Manager.GetConveyor(Manager.DataTypes.SystemLong);
				Streams.Conveyor LInt32Conveyor = Manager.GetConveyor(Manager.DataTypes.SystemInteger);

				Stream LStream;
				StreamID LStreamID;
				int LElementSize;
				Schema.ScalarType LScalarType;
				Streams.Conveyor LConveyor;
				DataValue LElement;
				for (int LIndex = 0; LIndex < FWriteList.Length; LIndex++)
				{
					if (FRow.Values[LIndex] == null)
					{
						ABuffer[AOffset] = (byte)0; // Write the native nil indicator
						AOffset++;
					}
					else
					{
						#if USEDATATYPESINNATIVEROW
						LScalarType = FRow.DataTypes[LIndex] as Schema.ScalarType;
						#else
						LScalarType = DataType.Columns[LIndex].DataType as Schema.ScalarType;
						#endif
						if ((LScalarType != null) && !LScalarType.IsCompound)
						{
							if (FRow.Values[LIndex] is StreamID)
							{
								// If this is a non-native scalar
								LStreamID = (StreamID)FRow.Values[LIndex];
								if (LStreamID == StreamID.Null)
								{
									ABuffer[AOffset] = (byte)1; // Write the non-native nil indicator
									AOffset++;
								}
								else
								{
									#if USEDATATYPESINNATIVEROW
									if (DataType.Columns[LIndex].DataType.Equals(FRow.DataTypes[LIndex]))
									{
									#endif
										ABuffer[AOffset] = (byte)3; // Write the native standard value indicator
										AOffset++;
									#if USEDATATYPESINNATIVEROW
									}
									else
									{
										ABuffer[AOffset] = (byte)5; // Write the native specialized value indicator
										AOffset++;
										if (LStringConveyor == null)
											LStringConveyor = Manager.GetConveyor(Manager.DataTypes.SystemString);
										LElementSize = LStringConveyor.GetSize(FRow.DataTypes[LIndex].Name);
										LStringConveyor.Write(FRow.DataTypes[LIndex].Name, ABuffer, AOffset); // Write the name of the data type of the value
										AOffset += LElementSize;
									}
									#endif
									
									if (AExpandStreams)
									{
										LStream = (Stream)FWriteList[LIndex];
										LInt32Conveyor.Write(Convert.ToInt32(LStream.Length), ABuffer, AOffset);
										AOffset += sizeof(int);
										LStream.Read(ABuffer, AOffset, (int)LStream.Length);
										AOffset += (int)LStream.Length;
										LStream.Close();
									}
									else
									{
										LInt64Conveyor.Write((long)LStreamID.Value, ABuffer, AOffset);
										AOffset += sizeof(long);
									}
								}
							}
							else
							{
								#if USEDATATYPESINNATIVEROW
								if (DataType.Columns[LIndex].DataType.Equals(FRow.DataTypes[LIndex]))
								{
								#endif
									ABuffer[AOffset] = (byte)2; // Write the native standard value indicator
									AOffset++;
								#if USEDATATYPESINNATIVEROW
								}
								else
								{
									ABuffer[AOffset] = (byte)4; // Write the native specialized value indicator
									AOffset++;
									if (LStringConveyor == null)
										LStringConveyor = Manager.GetConveyor(Manager.DataTypes.SystemString);
									LElementSize = LStringConveyor.GetSize(FRow.DataTypes[LIndex].Name);
									LStringConveyor.Write(FRow.DataTypes[LIndex].Name, ABuffer, AOffset); // Write the name of the data type of the value
									AOffset += LElementSize;
								}
								#endif

								LConveyor = Manager.GetConveyor(LScalarType);
								if (LConveyor.IsStreaming)
								{
									LStream = (Stream)FWriteList[LIndex];
									LInt32Conveyor.Write(Convert.ToInt32(LStream.Length), ABuffer, AOffset); // Write the length of the value
									AOffset += sizeof(int);
									LStream.Read(ABuffer, AOffset, (int)LStream.Length); // Write the value of this scalar
									AOffset += (int)LStream.Length;
								}
								else
								{
									LElementSize = (int)FWriteList[LIndex]; // Write the length of the value
									LInt32Conveyor.Write(LElementSize, ABuffer, AOffset);
									AOffset += sizeof(int);
									LConveyor.Write(FRow.Values[LIndex], ABuffer, AOffset); // Write the value of this scalar
									AOffset += LElementSize;
								}
							}
						}
						else
						{
							#if USEDATATYPESINNATIVEROW
							if (DataType.Columns[LIndex].DataType.Equals(FRow.DataTypes[LIndex]))
							{
							#endif
								ABuffer[AOffset] = (byte)2; // Write the native standard value indicator
								AOffset++;
							#if USEDATATYPESINNATIVEROW
							}
							else
							{
								ABuffer[AOffset] = (byte)4; // Write the native specialized value indicator
								AOffset++;
								if (LStringConveyor == null)
									LStringConveyor = Manager.GetConveyor(Manager.DataTypes.SystemString);
								LElementSize = LStringConveyor.GetSize(FRow.DataTypes[LIndex].Name);
								LStringConveyor.Write(FRow.DataTypes[LIndex].Name, ABuffer, AOffset); // Write the name of the data type of the value
								AOffset += LElementSize;
							}
							#endif

							LElement = FElementWriteList[LIndex];
							LElementSize = (int)FWriteList[LIndex];
							LInt32Conveyor.Write(LElementSize, ABuffer, AOffset);
							AOffset += sizeof(int);
							LElement.WriteToPhysical(ABuffer, AOffset, AExpandStreams); // Write the physical representation of the value;
							AOffset += LElementSize;
							LElement.Dispose();
						}
					}
				}
			}
		}

		public override void ReadFromPhysical(byte[] ABuffer, int AOffset)
		{
			ClearValues(); // Clear the current value of the row
			
			if (ABuffer[AOffset] == 0)
			{
				FRow = null;
			}
			else
			{
				FRow = new NativeRow(DataType.Columns.Count);
				AOffset++;
			
				bool LExpandStreams = ABuffer[AOffset] != 0; // Read the exapnded streams indicator
				AOffset++;
					
				#if USEDATATYPESINNATIVEROW
				Streams.Conveyor LStringConveyor = Manager.GetConveyor(Manager.DataTypes.SystemString);
				string LDataTypeName;
				Schema.IDataType LDataType;
				#endif
				Streams.Conveyor LInt64Conveyor = Manager.GetConveyor(Manager.DataTypes.SystemLong);
				Streams.Conveyor LInt32Conveyor = Manager.GetConveyor(Manager.DataTypes.SystemInteger);

				Stream LStream;
				StreamID LStreamID;
				int LElementSize;
				Schema.ScalarType LScalarType;
				Streams.Conveyor LConveyor;
				for (int LIndex = 0; LIndex < DataType.Columns.Count; LIndex++)
				{
					byte LValueIndicator = ABuffer[AOffset];
					AOffset++;
					
					switch (LValueIndicator)
					{
						case 0 : // native nil
							#if USEDATATYPESINNATIVEROW
							FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
							#endif
							FRow.Values[LIndex] = null;
						break;
						
						case 1 : // non-native nil
							#if USEDATATYPESINNATIVEROW
							FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
							#endif
							FRow.Values[LIndex] = StreamID.Null;
						break;
						
						case 2 : // native standard value
							LScalarType = DataType.Columns[LIndex].DataType as Schema.ScalarType;
							if ((LScalarType != null) && !LScalarType.IsCompound)
							{
								LConveyor = Manager.GetConveyor(LScalarType);
								if (LConveyor.IsStreaming)
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									LStream = new MemoryStream(ABuffer, AOffset, LElementSize, false, true);
									#if USEDATATYPESINNATIVEROW
									FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
									#endif
									FRow.Values[LIndex] = LConveyor.Read(LStream);
									AOffset += LElementSize;
								}
								else
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									#if USEDATATYPESINNATIVEROW
									FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
									#endif
									FRow.Values[LIndex] = LConveyor.Read(ABuffer, AOffset);
									AOffset += LElementSize;
								}
							}
							else
							{
								LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
								AOffset += sizeof(int);
								#if USEDATATYPESINNATIVEROW
								FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
								#endif
								using (DataValue LValue = DataValue.FromPhysical(Manager, DataType.Columns[LIndex].DataType, ABuffer, AOffset))
								{
									FRow.Values[LIndex] = LValue.AsNative;
									LValue.ValuesOwned = false;
								}
								AOffset += LElementSize;
							}
						break;
						
						case 3 : // non-native standard value
							LScalarType = DataType.Columns[LIndex].DataType as Schema.ScalarType;
							if (LScalarType != null)
							{
								if (LExpandStreams)
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									LStreamID = Manager.StreamManager.Allocate();
									LStream = Manager.StreamManager.Open(LStreamID, LockMode.Exclusive);
									LStream.Write(ABuffer, AOffset, LElementSize);
									LStream.Close();
									#if USEDATATYPESINNATIVEROW
									FRow.DataTypes[LIndex] = LScalarType;
									#endif
									FRow.Values[LIndex] = LStreamID;
									AOffset += LElementSize;
								}
								else
								{
									#if USEDATATYPESINNATIVEROW
									FRow.DataTypes[LIndex] = LScalarType;
									#endif
									FRow.Values[LIndex] = new StreamID(Convert.ToUInt64(LInt64Conveyor.Read(ABuffer, AOffset)));
									AOffset += sizeof(long);
								}
							}
							else
							{
								// non-scalar values cannot be non-native
							}
						break;
						
						case 4 : // native specialized value
							#if USEDATATYPESINNATIVEROW
							LDataTypeName = (string)LStringConveyor.Read(ABuffer, AOffset);
							LDataType = Manager.CompileTypeSpecifier(LDataTypeName);
							AOffset += LStringConveyor.GetSize(LDataTypeName);
							LScalarType = LDataType as Schema.ScalarType;
							if ((LScalarType != null) && !LScalarType.IsCompound)
							{
								LConveyor = Manager.GetConveyor(LScalarType);
								if (LConveyor.IsStreaming)
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									LStream = new MemoryStream(ABuffer, AOffset, LElementSize, false, true);
									FRow.DataTypes[LIndex] = LScalarType;
									FRow.Values[LIndex] = LConveyor.Read(LStream);
									AOffset += LElementSize;
								}
								else
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									FRow.DataTypes[LIndex] = LScalarType;
									FRow.Values[LIndex] = LConveyor.Read(ABuffer, AOffset);
									AOffset += LElementSize;
								}
							}
							else
							{
								LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
								AOffset += sizeof(int);
								FRow.DataTypes[LIndex] = LDataType;
								using (DataValue LValue = DataValue.FromPhysical(Manager, LDataType, ABuffer, AOffset))
								{
									FRow.Values[LIndex] = LValue.AsNative;
									LValue.ValuesOwned = false;
								}
								AOffset += LElementSize;
							}
						break;
							#else
							throw new NotSupportedException("Specialized data types in rows are not supported");
							#endif
						
						case 5 : // non-native specialized value
							#if USEDATATYPESINNATIVEROW
							LDataTypeName = (string)LStringConveyor.Read(ABuffer, AOffset);
							LDataType = Manager.CompileTypeSpecifier(LDataTypeName);
							AOffset += LStringConveyor.GetSize(LDataTypeName);
							LScalarType = LDataType as Schema.ScalarType;
							if (LScalarType != null)
							{
								if (LExpandStreams)
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									LStreamID = Manager.StreamManager.Allocate();
									LStream = Manager.StreamManager.Open(LStreamID, LockMode.Exclusive);
									LStream.Write(ABuffer, AOffset, LElementSize);
									LStream.Close();
									FRow.DataTypes[LIndex] = LScalarType;
									FRow.Values[LIndex] = LStreamID;
									AOffset += LElementSize;
								}
								else
								{
									FRow.DataTypes[LIndex] = LScalarType;
									FRow.Values[LIndex] = new StreamID(Convert.ToUInt64(LInt64Conveyor.Read(ABuffer, AOffset)));
									AOffset += sizeof(long);
								}
							}
							else
							{
								// non-scalar values cannot be non-native
							}
						break;
							#else
							throw new NotSupportedException("Specialized data types in rows are not supported");
							#endif
					}
				}
			}
		}
		
		public bool HasNonNativeValue(int AIndex)
		{
			#if USEDATATYPESINNATIVEROW
			return (FRow.DataTypes[AIndex] is Schema.IScalarType) && (FRow.Values[AIndex] is StreamID);
			#else
			return (DataType.Columns[AIndex].DataType is Schema.IScalarType) && (FRow.Values[AIndex] is StreamID);
			#endif
		}
		
		public StreamID GetNonNativeStreamID(int AIndex)
		{
			return (StreamID)FRow.Values[AIndex];
		}
		
		/// <summary>This is a by-reference access of the value, changes made to the resulting DataValue will be refelected in the actual row.</summary>
		public DataValue GetValue(int AIndex)
		{
			return FromNativeRow(Manager, DataType, FRow, AIndex);
		}
		
		public void SetValue(int AIndex, DataValue AValue)
		{
			this[AIndex] = AValue;
		}

		/// <summary>This is a by-reference access of the value, changes made to the resulting DataValue will be refelected in the actual row.</summary>
		public object this[int AIndex]
		{
			get 
			{ 
				if (DataType.Columns[AIndex].DataType is Schema.IScalarType)
					return FRow.Values[AIndex];
				return FromNativeRow(Manager, DataType, FRow, AIndex); 
			}
			set
			{<commit_msg>Row selector was returning RowInternedScalar.<commit_after>/*
	Dataphor
	© Copyright 2000-2008 Alphora
	This file is licensed under a modified BSD-license which can be found here: http://dataphor.org/dataphor_license.txt
*/

#define USEDATATYPESINNATIVEROW // Determines whether or not the native row tracks the data type of the values it contains

using System;
using System.IO;
using System.Text;
using System.Collections;

namespace Alphora.Dataphor.DAE.Runtime.Data
{
	#if USEDATATYPESINNATIVEROW
	using Alphora.Dataphor.DAE.Language.D4;
	using Alphora.Dataphor.DAE.Compiling;
	#endif
	using Alphora.Dataphor.DAE.Server;
	using Alphora.Dataphor.DAE.Streams;

	/*
		Row ->
		
			Rows store data as an array of objects.
			Data values can be stored in one of two ways inside a row.
			If the Data value is a StreamID, it is considered a non-native
			value and enables deferred access values by effectively storing
			only the StreamID of the data.  Otherwise, the value is considered
			a native value and is stored in its host representation.
	*/
	
	public class NativeRow : System.Object
	{
		public NativeRow(int AColumnCount) : base()
		{
			#if USEDATATYPESINNATIVEROW
			FDataTypes = new Schema.IDataType[AColumnCount];
			#endif
			FValues = new object[AColumnCount];
		}
		
		#if USEDATATYPESINNATIVEROW
		private Schema.IDataType[] FDataTypes;
		public Schema.IDataType[] DataTypes { get { return FDataTypes; } }
		#endif
		
		private object[] FValues;
		public object[] Values { get { return FValues; } }
		
		public BitArray ModifiedFlags;
	}
	
	/// <remarks>
	/// Provides a fixed length buffer for cell values with overflow management built in.
	/// Used in conjunction with the CellValueStream, provides transparent variable length
	/// value storage in a fixed length buffer.
	/// </remarks>    
	public class Row : DataValue
	{
		public Row(IValueManager AManager, Schema.IRowType ADataType) : base(AManager, ADataType)
		{
			FRow = new NativeRow(DataType.Columns.Count);
			#if USEDATATYPESINNATIVEROW
			for (int LIndex = 0; LIndex < DataType.Columns.Count; LIndex++)
				FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
			#endif
			ValuesOwned = true;
		}

		// The given object[] is assumed to contains values of the appropriate type for the given data type
		public Row(IValueManager AManager, Schema.IRowType ADataType, NativeRow ARow) : base(AManager, ADataType)
		{
			FRow = ARow;
			ValuesOwned = false;
		}

		protected override void Dispose(bool ADisposing)
		{
			if (FRow != null)
			{
				if (ValuesOwned)
					ClearValues();
				FRow = null;
			}
			base.Dispose(ADisposing);
		}
		
		public new Schema.IRowType DataType { get { return (Schema.IRowType)base.DataType; } }
		
		private NativeRow FRow;
		
		public override object AsNative
		{
			get { return FRow; }
			set 
			{
				if (FRow != null)
					ClearValues();
				FRow = (NativeRow)value; 
			}
		}
		
		private object[] FWriteList;
		private DataValue[] FElementWriteList;

		/*
			Row Value Format ->
			
				00 -> 0 - Indicates that non-native values are stored as a StreamID 1 - Indicates non-native values are stored inline
				01-XX -> Value for each attribute in the row
				
			Row Attribute Format ->

				There are five possibilities for an attribute ->			
					Nil Native
					Nil Non-Native (StreamID.Null)
					Standard Native
					Standard Non-Native
					Specialized Native
					Specialized Non-Native
				
					For non-native values, the value will be expanded or not dependending on the expanded setting for the row value

				00	-> 0 - 5
					0 if the row contains a native nil for this attribute - no data follows
					1 if the row contains a non-native nil for this attribute - no data follows
					2 if the row contains a native value of the data type of the attribute
						01-04 -> Length of the physical representation of this value
						05-XX -> The physical representation of this value
					3 if the row contains a non-native value of the data type of the attribute
						01-04 -> Length of the physical representation of this value
						05-XX -> The physical representation of this value (expanded based on the expanded setting for the list value)
					4 if the row contains a native value of some specialization of the data type of the attribute
						01-XX -> The data type name of this value, stored using a StringConveyor
						XX+1-XX+5 -> The length of the physical representation of this value
						XX+6-YY -> The physical representation of this value
					5 if the row contains a non-native value of some specialization of the data type of the attribute
						01-XX -> The data type name of this value, stored using a StringConveyor
						XX+1-XX+5 -> The length of the physical representation of this value
						XX+6-YY -> The physical representation of this value (expanded based on the expanded setting for the list value)
		*/
		public override int GetPhysicalSize(bool AExpandStreams)
		{
			int LSize = 1; // write the value indicator
			if (!IsNil)
			{
				LSize += 1; // write the extended streams indicator
				FWriteList = new object[DataType.Columns.Count]; // list for saving the sizes or streams of each attribute in the row
				FElementWriteList = new DataValue[DataType.Columns.Count]; // list for saving host representations of values between the GetPhysicalSize and WriteToPhysical calls
				Stream LStream;
				StreamID LStreamID;
				Schema.ScalarType LScalarType;
				Streams.Conveyor LConveyor;
				DataValue LElement;
				int LElementSize;
				for (int LIndex = 0; LIndex < FWriteList.Length; LIndex++)
				{
					LSize += sizeof(byte); // write a value indicator
					if (FRow.Values[LIndex] != null)
					{
						#if USEDATATYPESINNATIVEROW
						if (!DataType.Columns[LIndex].DataType.Equals(FRow.DataTypes[LIndex]))
							LSize += Manager.GetConveyor(Manager.DataTypes.SystemString).GetSize(FRow.DataTypes[LIndex].Name); // write the name of the data type of the value
						#endif
						/*							
						LElement = DataValue.FromNativeRow(Manager, DataType, FRow, LIndex);
						FElementWriteList[LIndex] = LElement;
						LElementSize = LElement.GetPhysicalSize(AExpandStreams);
						FWriteList[LIndex] = LElementSize;
						LSize += sizeof(int) + LElementSize;
						*/
						#if USEDATATYPESINNATIVEROW
						LScalarType = FRow.DataTypes[LIndex] as Schema.ScalarType;
						#else
						LScalarType = DataType.Columns[LIndex].DataType as Schema.ScalarType;
						#endif
						if ((LScalarType != null) && !LScalarType.IsCompound)
						{
							if (FRow.Values[LIndex] is StreamID)
							{
								// If this is a non-native scalar
								LStreamID = (StreamID)FRow.Values[LIndex];
								if (AExpandStreams)
								{
									if (LStreamID != StreamID.Null)
									{
										LStream = Manager.StreamManager.Open((StreamID)FRow.Values[LIndex], LockMode.Exclusive);
										FWriteList[LIndex] = LStream;
										LSize += sizeof(int) + (int)LStream.Length;
									}
								}
								else
								{
									if (LStreamID != StreamID.Null)
									{
										LElementSize = sizeof(long);
										FWriteList[LIndex] = LElementSize;
										LSize += LElementSize;
									}
								}
							}
							else
							{
								LConveyor = Manager.GetConveyor(LScalarType);
								if (LConveyor.IsStreaming)
								{
									LStream = new MemoryStream(64);
									FWriteList[LIndex] = LStream;
									LConveyor.Write(FRow.Values[LIndex], LStream);
									LStream.Position = 0;
									LSize += sizeof(int) + (int)LStream.Length;
								}
								else
								{
									LElementSize = LConveyor.GetSize(FRow.Values[LIndex]);
									FWriteList[LIndex] = LElementSize;
									LSize += sizeof(int) + LElementSize;;
								}
							}
						}
						else
						{
							LElement = DataValue.FromNativeRow(Manager, DataType, FRow, LIndex);
							FElementWriteList[LIndex] = LElement;
							LElementSize = LElement.GetPhysicalSize(AExpandStreams);
							FWriteList[LIndex] = LElementSize;
							LSize += sizeof(int) + LElementSize;
						}						
					}
				}
			}
			return LSize;
		}
		
		public override void WriteToPhysical(byte[] ABuffer, int AOffset, bool AExpandStreams)
		{
			if (FWriteList == null)
				throw new RuntimeException(RuntimeException.Codes.UnpreparedWriteToPhysicalCall);
				
			ABuffer[AOffset] = (byte)(IsNil ? 0 : 1); // Write the value indicator
			AOffset++;
			
			if (!IsNil)
			{
				ABuffer[AOffset] = (byte)(AExpandStreams ? 1 : 0); // Write the expanded streams indicator
				AOffset++;
					
				#if USEDATATYPESINNATIVEROW
				Streams.Conveyor LStringConveyor = null;
				#endif
				Streams.Conveyor LInt64Conveyor = Manager.GetConveyor(Manager.DataTypes.SystemLong);
				Streams.Conveyor LInt32Conveyor = Manager.GetConveyor(Manager.DataTypes.SystemInteger);

				Stream LStream;
				StreamID LStreamID;
				int LElementSize;
				Schema.ScalarType LScalarType;
				Streams.Conveyor LConveyor;
				DataValue LElement;
				for (int LIndex = 0; LIndex < FWriteList.Length; LIndex++)
				{
					if (FRow.Values[LIndex] == null)
					{
						ABuffer[AOffset] = (byte)0; // Write the native nil indicator
						AOffset++;
					}
					else
					{
						#if USEDATATYPESINNATIVEROW
						LScalarType = FRow.DataTypes[LIndex] as Schema.ScalarType;
						#else
						LScalarType = DataType.Columns[LIndex].DataType as Schema.ScalarType;
						#endif
						if ((LScalarType != null) && !LScalarType.IsCompound)
						{
							if (FRow.Values[LIndex] is StreamID)
							{
								// If this is a non-native scalar
								LStreamID = (StreamID)FRow.Values[LIndex];
								if (LStreamID == StreamID.Null)
								{
									ABuffer[AOffset] = (byte)1; // Write the non-native nil indicator
									AOffset++;
								}
								else
								{
									#if USEDATATYPESINNATIVEROW
									if (DataType.Columns[LIndex].DataType.Equals(FRow.DataTypes[LIndex]))
									{
									#endif
										ABuffer[AOffset] = (byte)3; // Write the native standard value indicator
										AOffset++;
									#if USEDATATYPESINNATIVEROW
									}
									else
									{
										ABuffer[AOffset] = (byte)5; // Write the native specialized value indicator
										AOffset++;
										if (LStringConveyor == null)
											LStringConveyor = Manager.GetConveyor(Manager.DataTypes.SystemString);
										LElementSize = LStringConveyor.GetSize(FRow.DataTypes[LIndex].Name);
										LStringConveyor.Write(FRow.DataTypes[LIndex].Name, ABuffer, AOffset); // Write the name of the data type of the value
										AOffset += LElementSize;
									}
									#endif
									
									if (AExpandStreams)
									{
										LStream = (Stream)FWriteList[LIndex];
										LInt32Conveyor.Write(Convert.ToInt32(LStream.Length), ABuffer, AOffset);
										AOffset += sizeof(int);
										LStream.Read(ABuffer, AOffset, (int)LStream.Length);
										AOffset += (int)LStream.Length;
										LStream.Close();
									}
									else
									{
										LInt64Conveyor.Write((long)LStreamID.Value, ABuffer, AOffset);
										AOffset += sizeof(long);
									}
								}
							}
							else
							{
								#if USEDATATYPESINNATIVEROW
								if (DataType.Columns[LIndex].DataType.Equals(FRow.DataTypes[LIndex]))
								{
								#endif
									ABuffer[AOffset] = (byte)2; // Write the native standard value indicator
									AOffset++;
								#if USEDATATYPESINNATIVEROW
								}
								else
								{
									ABuffer[AOffset] = (byte)4; // Write the native specialized value indicator
									AOffset++;
									if (LStringConveyor == null)
										LStringConveyor = Manager.GetConveyor(Manager.DataTypes.SystemString);
									LElementSize = LStringConveyor.GetSize(FRow.DataTypes[LIndex].Name);
									LStringConveyor.Write(FRow.DataTypes[LIndex].Name, ABuffer, AOffset); // Write the name of the data type of the value
									AOffset += LElementSize;
								}
								#endif

								LConveyor = Manager.GetConveyor(LScalarType);
								if (LConveyor.IsStreaming)
								{
									LStream = (Stream)FWriteList[LIndex];
									LInt32Conveyor.Write(Convert.ToInt32(LStream.Length), ABuffer, AOffset); // Write the length of the value
									AOffset += sizeof(int);
									LStream.Read(ABuffer, AOffset, (int)LStream.Length); // Write the value of this scalar
									AOffset += (int)LStream.Length;
								}
								else
								{
									LElementSize = (int)FWriteList[LIndex]; // Write the length of the value
									LInt32Conveyor.Write(LElementSize, ABuffer, AOffset);
									AOffset += sizeof(int);
									LConveyor.Write(FRow.Values[LIndex], ABuffer, AOffset); // Write the value of this scalar
									AOffset += LElementSize;
								}
							}
						}
						else
						{
							#if USEDATATYPESINNATIVEROW
							if (DataType.Columns[LIndex].DataType.Equals(FRow.DataTypes[LIndex]))
							{
							#endif
								ABuffer[AOffset] = (byte)2; // Write the native standard value indicator
								AOffset++;
							#if USEDATATYPESINNATIVEROW
							}
							else
							{
								ABuffer[AOffset] = (byte)4; // Write the native specialized value indicator
								AOffset++;
								if (LStringConveyor == null)
									LStringConveyor = Manager.GetConveyor(Manager.DataTypes.SystemString);
								LElementSize = LStringConveyor.GetSize(FRow.DataTypes[LIndex].Name);
								LStringConveyor.Write(FRow.DataTypes[LIndex].Name, ABuffer, AOffset); // Write the name of the data type of the value
								AOffset += LElementSize;
							}
							#endif

							LElement = FElementWriteList[LIndex];
							LElementSize = (int)FWriteList[LIndex];
							LInt32Conveyor.Write(LElementSize, ABuffer, AOffset);
							AOffset += sizeof(int);
							LElement.WriteToPhysical(ABuffer, AOffset, AExpandStreams); // Write the physical representation of the value;
							AOffset += LElementSize;
							LElement.Dispose();
						}
					}
				}
			}
		}

		public override void ReadFromPhysical(byte[] ABuffer, int AOffset)
		{
			ClearValues(); // Clear the current value of the row
			
			if (ABuffer[AOffset] == 0)
			{
				FRow = null;
			}
			else
			{
				FRow = new NativeRow(DataType.Columns.Count);
				AOffset++;
			
				bool LExpandStreams = ABuffer[AOffset] != 0; // Read the exapnded streams indicator
				AOffset++;
					
				#if USEDATATYPESINNATIVEROW
				Streams.Conveyor LStringConveyor = Manager.GetConveyor(Manager.DataTypes.SystemString);
				string LDataTypeName;
				Schema.IDataType LDataType;
				#endif
				Streams.Conveyor LInt64Conveyor = Manager.GetConveyor(Manager.DataTypes.SystemLong);
				Streams.Conveyor LInt32Conveyor = Manager.GetConveyor(Manager.DataTypes.SystemInteger);

				Stream LStream;
				StreamID LStreamID;
				int LElementSize;
				Schema.ScalarType LScalarType;
				Streams.Conveyor LConveyor;
				for (int LIndex = 0; LIndex < DataType.Columns.Count; LIndex++)
				{
					byte LValueIndicator = ABuffer[AOffset];
					AOffset++;
					
					switch (LValueIndicator)
					{
						case 0 : // native nil
							#if USEDATATYPESINNATIVEROW
							FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
							#endif
							FRow.Values[LIndex] = null;
						break;
						
						case 1 : // non-native nil
							#if USEDATATYPESINNATIVEROW
							FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
							#endif
							FRow.Values[LIndex] = StreamID.Null;
						break;
						
						case 2 : // native standard value
							LScalarType = DataType.Columns[LIndex].DataType as Schema.ScalarType;
							if ((LScalarType != null) && !LScalarType.IsCompound)
							{
								LConveyor = Manager.GetConveyor(LScalarType);
								if (LConveyor.IsStreaming)
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									LStream = new MemoryStream(ABuffer, AOffset, LElementSize, false, true);
									#if USEDATATYPESINNATIVEROW
									FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
									#endif
									FRow.Values[LIndex] = LConveyor.Read(LStream);
									AOffset += LElementSize;
								}
								else
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									#if USEDATATYPESINNATIVEROW
									FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
									#endif
									FRow.Values[LIndex] = LConveyor.Read(ABuffer, AOffset);
									AOffset += LElementSize;
								}
							}
							else
							{
								LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
								AOffset += sizeof(int);
								#if USEDATATYPESINNATIVEROW
								FRow.DataTypes[LIndex] = DataType.Columns[LIndex].DataType;
								#endif
								using (DataValue LValue = DataValue.FromPhysical(Manager, DataType.Columns[LIndex].DataType, ABuffer, AOffset))
								{
									FRow.Values[LIndex] = LValue.AsNative;
									LValue.ValuesOwned = false;
								}
								AOffset += LElementSize;
							}
						break;
						
						case 3 : // non-native standard value
							LScalarType = DataType.Columns[LIndex].DataType as Schema.ScalarType;
							if (LScalarType != null)
							{
								if (LExpandStreams)
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									LStreamID = Manager.StreamManager.Allocate();
									LStream = Manager.StreamManager.Open(LStreamID, LockMode.Exclusive);
									LStream.Write(ABuffer, AOffset, LElementSize);
									LStream.Close();
									#if USEDATATYPESINNATIVEROW
									FRow.DataTypes[LIndex] = LScalarType;
									#endif
									FRow.Values[LIndex] = LStreamID;
									AOffset += LElementSize;
								}
								else
								{
									#if USEDATATYPESINNATIVEROW
									FRow.DataTypes[LIndex] = LScalarType;
									#endif
									FRow.Values[LIndex] = new StreamID(Convert.ToUInt64(LInt64Conveyor.Read(ABuffer, AOffset)));
									AOffset += sizeof(long);
								}
							}
							else
							{
								// non-scalar values cannot be non-native
							}
						break;
						
						case 4 : // native specialized value
							#if USEDATATYPESINNATIVEROW
							LDataTypeName = (string)LStringConveyor.Read(ABuffer, AOffset);
							LDataType = Manager.CompileTypeSpecifier(LDataTypeName);
							AOffset += LStringConveyor.GetSize(LDataTypeName);
							LScalarType = LDataType as Schema.ScalarType;
							if ((LScalarType != null) && !LScalarType.IsCompound)
							{
								LConveyor = Manager.GetConveyor(LScalarType);
								if (LConveyor.IsStreaming)
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									LStream = new MemoryStream(ABuffer, AOffset, LElementSize, false, true);
									FRow.DataTypes[LIndex] = LScalarType;
									FRow.Values[LIndex] = LConveyor.Read(LStream);
									AOffset += LElementSize;
								}
								else
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									FRow.DataTypes[LIndex] = LScalarType;
									FRow.Values[LIndex] = LConveyor.Read(ABuffer, AOffset);
									AOffset += LElementSize;
								}
							}
							else
							{
								LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
								AOffset += sizeof(int);
								FRow.DataTypes[LIndex] = LDataType;
								using (DataValue LValue = DataValue.FromPhysical(Manager, LDataType, ABuffer, AOffset))
								{
									FRow.Values[LIndex] = LValue.AsNative;
									LValue.ValuesOwned = false;
								}
								AOffset += LElementSize;
							}
						break;
							#else
							throw new NotSupportedException("Specialized data types in rows are not supported");
							#endif
						
						case 5 : // non-native specialized value
							#if USEDATATYPESINNATIVEROW
							LDataTypeName = (string)LStringConveyor.Read(ABuffer, AOffset);
							LDataType = Manager.CompileTypeSpecifier(LDataTypeName);
							AOffset += LStringConveyor.GetSize(LDataTypeName);
							LScalarType = LDataType as Schema.ScalarType;
							if (LScalarType != null)
							{
								if (LExpandStreams)
								{
									LElementSize = (int)LInt32Conveyor.Read(ABuffer, AOffset);
									AOffset += sizeof(int);
									LStreamID = Manager.StreamManager.Allocate();
									LStream = Manager.StreamManager.Open(LStreamID, LockMode.Exclusive);
									LStream.Write(ABuffer, AOffset, LElementSize);
									LStream.Close();
									FRow.DataTypes[LIndex] = LScalarType;
									FRow.Values[LIndex] = LStreamID;
									AOffset += LElementSize;
								}
								else
								{
									FRow.DataTypes[LIndex] = LScalarType;
									FRow.Values[LIndex] = new StreamID(Convert.ToUInt64(LInt64Conveyor.Read(ABuffer, AOffset)));
									AOffset += sizeof(long);
								}
							}
							else
							{
								// non-scalar values cannot be non-native
							}
						break;
							#else
							throw new NotSupportedException("Specialized data types in rows are not supported");
							#endif
					}
				}
			}
		}
		
		public bool HasNonNativeValue(int AIndex)
		{
			#if USEDATATYPESINNATIVEROW
			return (FRow.DataTypes[AIndex] is Schema.IScalarType) && (FRow.Values[AIndex] is StreamID);
			#else
			return (DataType.Columns[AIndex].DataType is Schema.IScalarType) && (FRow.Values[AIndex] is StreamID);
			#endif
		}
		
		public StreamID GetNonNativeStreamID(int AIndex)
		{
			return (StreamID)FRow.Values[AIndex];
		}
		
		/// <summary>This is a by-reference access of the value, changes made to the resulting DataValue will be refelected in the actual row.</summary>
		public DataValue GetValue(int AIndex)
		{
			return FromNativeRow(Manager, DataType, FRow, AIndex);
		}
		
		public void SetValue(int AIndex, DataValue AValue)
		{
			this[AIndex] = AValue;
		}

		/// <summary>This is a by-reference access of the value, changes made to the resulting DataValue will be refelected in the actual row.</summary>
		public object this[int AIndex]
		{
			get 
			{
				#if USEDATATYPESINNATIVEROW
				if (FRow.DataTypes[AIndex] is Schema.IScalarType)
				#else
				if (DataType.Columns[AIndex].DataType is Schema.IScalarType)
				#endif
					return FRow.Values[AIndex];
				return FromNativeRow(Manager, DataType, FRow, AIndex); 
			}
			set
			{
{'fraction_non_alphanumeric': 0.09363599513579246, 'fraction_numerical': 0.005494752961311534, 'mean_word_length': 2.4990938460326215, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6596642', 'n_tokens_mistral': 20045, 'n_tokens_neox': 13627, 'n_words': 3304}