<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using ValveResourceFormat.ResourceTypes.NTROSerialization;

namespace ValveResourceFormat.ResourceTypes
{
    public class EntityLump : NTRO
    {
        public List<List<Tuple<uint, uint, object>>> Datas { get; private set; }

        public override void Read(BinaryReader reader, Resource resource)
        {
            base.Read(reader, resource);

            // Output is PermEntityLumpData_t we need to iterate m_entityKeyValues inside it.
            var entityKeyValues = (NTROArray)Output["m_entityKeyValues"];

            Datas = new List<List<Tuple<uint, uint, object>>>();

            foreach (var entityKV in entityKeyValues)
            {
                // entity is EntityKeyValueData_t
                var entity = ((NTROValue<NTROStruct>)entityKV).Value;
                var dataArray = (NTROArray)entity["m_keyValuesData"];
                var data = new List<byte>();
                foreach (NTROValue<byte> entry in dataArray)
                {
                    data.Add(entry.Value);
                }

                using (var dataStream = new MemoryStream(data.ToArray()))
                using (var dataReader = new BinaryReader(dataStream))
                {
                    var a = dataReader.ReadUInt32(); // always 1?
                    var valuesCount = dataReader.ReadUInt32();
                    var c = dataReader.ReadUInt32(); // always 0?

                    var values = new List<Tuple<uint, uint, object>>();
                    while (dataStream.Position != dataStream.Length)
                    {
                        var miscType = dataReader.ReadUInt32(); //Stuff before type, some pointer?
                        var type = dataReader.ReadUInt32();
                        switch (type)
                        {
                            case 0x06:
                                values.Add(new Tuple<uint, uint, object>(type, miscType, dataReader.ReadByte())); //1
                                break;<commit_msg>More info can be found in vents_c now, stop crashing for now vr/vr_command_room.vpk in dota forced this<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using ValveResourceFormat.ResourceTypes.NTROSerialization;

namespace ValveResourceFormat.ResourceTypes
{
    public class EntityLump : NTRO
    {
        public List<List<Tuple<uint, uint, object>>> Datas { get; private set; }

        public override void Read(BinaryReader reader, Resource resource)
        {
            base.Read(reader, resource);

            // Output is PermEntityLumpData_t we need to iterate m_entityKeyValues inside it.
            var entityKeyValues = (NTROArray)Output["m_entityKeyValues"];

            Datas = new List<List<Tuple<uint, uint, object>>>();

            foreach (var entityKV in entityKeyValues)
            {
                // entity is EntityKeyValueData_t
                var entity = ((NTROValue<NTROStruct>)entityKV).Value;
                var dataArray = (NTROArray)entity["m_keyValuesData"];
                var data = new List<byte>();
                foreach (NTROValue<byte> entry in dataArray)
                {
                    data.Add(entry.Value);
                }

                using (var dataStream = new MemoryStream(data.ToArray()))
                using (var dataReader = new BinaryReader(dataStream))
                {
                    var a = dataReader.ReadUInt32(); // always 1?
                    var valuesCount = dataReader.ReadUInt32();
                    var c = dataReader.ReadUInt32(); // always 0? (Its been seen to be 1, footer count?)

                    var values = new List<Tuple<uint, uint, object>>();
                    while (dataStream.Position != dataStream.Length)
                    {
                        if (values.Count == valuesCount)
                        {
                            Console.WriteLine("We hit our values target without reading every byte?!");
                            break;
                        }
                        var miscType = dataReader.ReadUInt32(); //Stuff before type, some pointer?
                        var type = dataReader.ReadUInt32();
                        switch (type)
                        {
                            case 0x06:
                                values.Add(new Tuple<uint, uint, object>(type, miscType, dataReader.ReadByte())); //1
                                break;
{'fraction_non_alphanumeric': 0.08876772082878953, 'fraction_numerical': 0.007197382769901854, 'mean_word_length': 1.6012478729438457, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2648963', 'n_tokens_mistral': 1169, 'n_tokens_neox': 1095, 'n_words': 300}