<commit_before>﻿using System;
using System.IO;
using I3DShapesTool.Lib.Container;
using I3DShapesTool.Lib.Tools;
using I3DShapesTool.Lib.Tools.Extensions;

namespace I3DShapesTool.Lib.Model
{
    public class I3DShape : I3DPart
    {
        private const int VERSION_WITH_TANGENTS = 4; // Not sure of exact version here

        public float BoundingVolumeX { get; private set; }
        public float BoundingVolumeY { get; private set; }
        public float BoundingVolumeZ { get; private set; }
        public float BoundingVolumeR { get; private set; }

        /// <summary>
        /// Number of triangle corners
        /// </summary>
        public uint CornerCount { get; private set; }
        public uint NumSubsets { get; private set; }
        /// <summary>
        /// Number of unique vertices
        /// </summary>
        public uint VertexCount { get; private set; }
        public I3DShapeOptions Options { get; private set; }
        public I3DShapeSubset[] Subsets { get; private set; }
        public bool ZeroBasedIndicesInRawData { get; private set; }
        public I3DTri[] Triangles { get; private set; }
        public I3DVector[] Positions { get; private set; }
        public I3DVector[]? Normals { get; private set; }
        public I3DVector4[]? Tangents { get; private set; }
        public I3DUV[][] UVSets { get; private set; }
        public I3DVector4[]? VertexColor { get; private set; }
        public float[,]? BlendWeights { get; private set; }
        public byte[,]? BlendIndices { get; private set; }
        public float[]? GenericData { get; private set; }
        public I3DShapeAttachment[] Attachments { get; private set; }

#nullable disable
        public I3DShape(byte[] rawData, Endian endian, int version)
            : base(EntityType.Shape, rawData, endian, version)
        {
        }
#nullable restore

        protected override void ReadContents(BinaryReader reader)
        {
            BoundingVolumeX = reader.ReadSingle();
            BoundingVolumeY = reader.ReadSingle();
            BoundingVolumeZ = reader.ReadSingle();
            BoundingVolumeR = reader.ReadSingle();
            CornerCount = reader.ReadUInt32();
            NumSubsets = reader.ReadUInt32();
            VertexCount = reader.ReadUInt32();
            Options = (I3DShapeOptions)reader.ReadUInt32();
            Subsets = new I3DShapeSubset[NumSubsets];
            for(int i = 0; i < NumSubsets; i++)
            {
                Subsets[i] = new I3DShapeSubset(reader, Version, Options);
            }

            ZeroBasedIndicesInRawData = false;
            Triangles = new I3DTri[CornerCount / 3];
            for(int i = 0; i < CornerCount / 3; i++)
            {
                Triangles[i] = new I3DTri(reader, VertexCount > (ushort.MaxValue + 1));

                if(Triangles[i].P1Idx == 0 || Triangles[i].P2Idx == 0 || Triangles[i].P3Idx == 0)
                    ZeroBasedIndicesInRawData = true;
            }

            // Convert to 1-based indices if it's detected that it is a zero-based index
            if(ZeroBasedIndicesInRawData)
            {
                foreach(I3DTri t in Triangles)
                {
                    t.P1Idx += 1;
                    t.P2Idx += 1;
                    t.P3Idx += 1;
                }
            }

            // TODO: figure out the exact logic for this
            //if (Version < 4) // Could be 5 as well
            reader.BaseStream.Align(4);

            Positions = new I3DVector[VertexCount];
            for(int i = 0; i < VertexCount; i++)
            {
                Positions[i] = new I3DVector(reader);
            }

            if(Options.HasFlag(I3DShapeOptions.HasNormals))
            {
                Normals = new I3DVector[VertexCount];
                for(int i = 0; i < VertexCount; i++)
                {
                    Normals[i] = new I3DVector(reader);
                }
            }

            if(Version >= VERSION_WITH_TANGENTS && Options.HasFlag(I3DShapeOptions.HasTangents))
            {
                Tangents = new I3DVector4[VertexCount];
                for(int i = 0; i < VertexCount; i++)
                {
                    Tangents[i] = new I3DVector4(reader);
                }
            }

            UVSets = new I3DUV[4][];
            for(int uvSet = 0; uvSet < 4; uvSet++)
            {
                if(Options.HasFlag((I3DShapeOptions)((uint)I3DShapeOptions.HasUV1 << uvSet)))
                {
                    I3DUV[] uvs = new I3DUV[VertexCount];
                    for(int i = 0; i < VertexCount; i++)
                    {
                        uvs[i] = new I3DUV(reader, Version);
                    }
                    UVSets[uvSet] = uvs;
                }
            }

            if(Options.HasFlag(I3DShapeOptions.HasVertexColor))
            {
                VertexColor = new I3DVector4[VertexCount];
                for(int i = 0; i < VertexCount; i++)
                {
                    VertexColor[i] = new I3DVector4(reader);
                }
            }

            if(Options.HasFlag(I3DShapeOptions.HasSkinningInfo))
            {
                bool singleBlendWeights = Options.HasFlag(I3DShapeOptions.SingleBlendWeights);

                // based on how 3D engines usually do skeletal/skinned meshes, you always have 4 weights and 4 indices per vertex

                BlendWeights = new float[VertexCount, 4];

                int numIndices;

                // Load blend weights first, if necessary
                // usually all weights per vertex sum up to 1.0, which means, if you only have one index per vertex, the first weight would always have to be 1
                if (singleBlendWeights)
                {
                    numIndices = 1;

                    for(int i = 0; i < VertexCount; i++)
                    {
                        BlendWeights[i, 0] = 1.0f;
                    }
                }
                else
                {
                    numIndices = 4;

                    for(int i = 0; i < VertexCount; i++)
                    {
                        for(int j = 0; j < 4; j++)
                        {
                            BlendWeights[i, j] = reader.ReadSingle();
                        }
                    }
                }

                // Load blend indices
                BlendIndices = new byte[VertexCount, 4];

                for(int i = 0; i < VertexCount; i++)
                {
                    for(int j = 0; j < numIndices; j++)<commit_msg>ensure BlendIndices array has the correct size<commit_after>﻿using System;
using System.IO;
using I3DShapesTool.Lib.Container;
using I3DShapesTool.Lib.Tools;
using I3DShapesTool.Lib.Tools.Extensions;

namespace I3DShapesTool.Lib.Model
{
    public class I3DShape : I3DPart
    {
        private const int VERSION_WITH_TANGENTS = 4; // Not sure of exact version here

        public float BoundingVolumeX { get; private set; }
        public float BoundingVolumeY { get; private set; }
        public float BoundingVolumeZ { get; private set; }
        public float BoundingVolumeR { get; private set; }

        /// <summary>
        /// Number of triangle corners
        /// </summary>
        public uint CornerCount { get; private set; }
        public uint NumSubsets { get; private set; }
        /// <summary>
        /// Number of unique vertices
        /// </summary>
        public uint VertexCount { get; private set; }
        public I3DShapeOptions Options { get; private set; }
        public I3DShapeSubset[] Subsets { get; private set; }
        public bool ZeroBasedIndicesInRawData { get; private set; }
        public I3DTri[] Triangles { get; private set; }
        public I3DVector[] Positions { get; private set; }
        public I3DVector[]? Normals { get; private set; }
        public I3DVector4[]? Tangents { get; private set; }
        public I3DUV[][] UVSets { get; private set; }
        public I3DVector4[]? VertexColor { get; private set; }
        public float[,]? BlendWeights { get; private set; }
        public byte[,]? BlendIndices { get; private set; }
        public float[]? GenericData { get; private set; }
        public I3DShapeAttachment[] Attachments { get; private set; }

#nullable disable
        public I3DShape(byte[] rawData, Endian endian, int version)
            : base(EntityType.Shape, rawData, endian, version)
        {
        }
#nullable restore

        protected override void ReadContents(BinaryReader reader)
        {
            BoundingVolumeX = reader.ReadSingle();
            BoundingVolumeY = reader.ReadSingle();
            BoundingVolumeZ = reader.ReadSingle();
            BoundingVolumeR = reader.ReadSingle();
            CornerCount = reader.ReadUInt32();
            NumSubsets = reader.ReadUInt32();
            VertexCount = reader.ReadUInt32();
            Options = (I3DShapeOptions)reader.ReadUInt32();
            Subsets = new I3DShapeSubset[NumSubsets];
            for(int i = 0; i < NumSubsets; i++)
            {
                Subsets[i] = new I3DShapeSubset(reader, Version, Options);
            }

            ZeroBasedIndicesInRawData = false;
            Triangles = new I3DTri[CornerCount / 3];
            for(int i = 0; i < CornerCount / 3; i++)
            {
                Triangles[i] = new I3DTri(reader, VertexCount > (ushort.MaxValue + 1));

                if(Triangles[i].P1Idx == 0 || Triangles[i].P2Idx == 0 || Triangles[i].P3Idx == 0)
                    ZeroBasedIndicesInRawData = true;
            }

            // Convert to 1-based indices if it's detected that it is a zero-based index
            if(ZeroBasedIndicesInRawData)
            {
                foreach(I3DTri t in Triangles)
                {
                    t.P1Idx += 1;
                    t.P2Idx += 1;
                    t.P3Idx += 1;
                }
            }

            // TODO: figure out the exact logic for this
            //if (Version < 4) // Could be 5 as well
            reader.BaseStream.Align(4);

            Positions = new I3DVector[VertexCount];
            for(int i = 0; i < VertexCount; i++)
            {
                Positions[i] = new I3DVector(reader);
            }

            if(Options.HasFlag(I3DShapeOptions.HasNormals))
            {
                Normals = new I3DVector[VertexCount];
                for(int i = 0; i < VertexCount; i++)
                {
                    Normals[i] = new I3DVector(reader);
                }
            }

            if(Version >= VERSION_WITH_TANGENTS && Options.HasFlag(I3DShapeOptions.HasTangents))
            {
                Tangents = new I3DVector4[VertexCount];
                for(int i = 0; i < VertexCount; i++)
                {
                    Tangents[i] = new I3DVector4(reader);
                }
            }

            UVSets = new I3DUV[4][];
            for(int uvSet = 0; uvSet < 4; uvSet++)
            {
                if(Options.HasFlag((I3DShapeOptions)((uint)I3DShapeOptions.HasUV1 << uvSet)))
                {
                    I3DUV[] uvs = new I3DUV[VertexCount];
                    for(int i = 0; i < VertexCount; i++)
                    {
                        uvs[i] = new I3DUV(reader, Version);
                    }
                    UVSets[uvSet] = uvs;
                }
            }

            if(Options.HasFlag(I3DShapeOptions.HasVertexColor))
            {
                VertexColor = new I3DVector4[VertexCount];
                for(int i = 0; i < VertexCount; i++)
                {
                    VertexColor[i] = new I3DVector4(reader);
                }
            }

            if(Options.HasFlag(I3DShapeOptions.HasSkinningInfo))
            {
                bool singleBlendWeights = Options.HasFlag(I3DShapeOptions.SingleBlendWeights);

                // based on how 3D engines usually do skeletal/skinned meshes, you always have 4 weights and 4 indices per vertex

                BlendWeights = new float[VertexCount, 4];

                int numIndices;

                // Load blend weights first, if necessary
                // usually all weights per vertex sum up to 1.0, which means, if you only have one index per vertex, the first weight would always have to be 1
                if (singleBlendWeights)
                {
                    numIndices = 1;

                    for(int i = 0; i < VertexCount; i++)
                    {
                        BlendWeights[i, 0] = 1.0f;
                    }
                }
                else
                {
                    numIndices = 4;

                    for(int i = 0; i < VertexCount; i++)
                    {
                        for(int j = 0; j < 4; j++)
                        {
                            BlendWeights[i, j] = reader.ReadSingle();
                        }
                    }
                }

                // Load blend indices
                BlendIndices = new byte[VertexCount, numIndices];

                for(int i = 0; i < VertexCount; i++)
                {
                    for(int j = 0; j < numIndices; j++)
{'fraction_non_alphanumeric': 0.09415337889141989, 'fraction_numerical': 0.015869400151860287, 'mean_word_length': 1.4153676875114616, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2213344', 'n_tokens_mistral': 3835, 'n_tokens_neox': 3563, 'n_words': 945}