<commit_before>﻿// Original Work Copyright (c) <NAME> 2014-2016
// This file is subject to the GPL v2 License
// For additional details, see the LICENSE file

using System.Diagnostics;
using System.IO;
using System.Text;
using EOLib.IO.Services;

namespace EOLib.IO.Pub
{
    public class ESFFile : BasePubFile<ESFRecord>
    {
        public override string FileType
        {
            get { return "ESF"; }
        }

        public override void DeserializeFromByteArray(byte[] bytes, INumberEncoderService numberEncoderService)
        {
            using (var mem = new MemoryStream(bytes))
                ReadFromStream(mem, numberEncoderService);
        }

        private void ReadFromStream(Stream mem, INumberEncoderService numberEncoderService)
        {
            mem.Seek(3, SeekOrigin.Begin);

            var checksum = new byte[4];
            mem.Read(checksum, 0, 4);
            CheckSum = numberEncoderService.DecodeNumber(checksum);

            var lenBytes = new byte[2];
            mem.Read(lenBytes, 0, 2);
            var recordsInFile = (short)numberEncoderService.DecodeNumber(lenBytes);

            mem.Seek(1, SeekOrigin.Current);

            var rawData = new byte[ESFRecord.DATA_SIZE];
            for (int i = 0; i < recordsInFile && mem.Position < mem.Length; ++i)
            {
                var nameLength = numberEncoderService.DecodeNumber((byte)mem.ReadByte());
                var shoutLength = numberEncoderService.DecodeNumber((byte)mem.ReadByte());
                var rawName = new byte[nameLength];
                var rawShout = new byte[shoutLength];
                mem.Read(rawName, 0, nameLength);
                mem.Read(rawShout, 0, shoutLength);
                mem.Read(rawData, 0, ESFRecord.DATA_SIZE);

                var record = new ESFRecord { ID = i, Name = Encoding.ASCII.GetString(rawName) };
                record.DeserializeFromByteArray(rawData, numberEncoderService);

                _data.Add(record);
            }

            if (recordsInFile != Length)
                throw new IOException("Mismatch between expected length and actual length!");
        }
    }
}
<commit_msg>Fix bug in ESFFile uncovered by unit testing<commit_after>﻿// Original Work Copyright (c) <NAME> 2014-2016
// This file is subject to the GPL v2 License
// For additional details, see the LICENSE file

using System.Diagnostics;
using System.IO;
using System.Text;
using EOLib.IO.Services;

namespace EOLib.IO.Pub
{
    public class ESFFile : BasePubFile<ESFRecord>
    {
        public override string FileType
        {
            get { return "ESF"; }
        }

        public override void DeserializeFromByteArray(byte[] bytes, INumberEncoderService numberEncoderService)
        {
            using (var mem = new MemoryStream(bytes))
                ReadFromStream(mem, numberEncoderService);
        }

        private void ReadFromStream(Stream mem, INumberEncoderService numberEncoderService)
        {
            mem.Seek(3, SeekOrigin.Begin);

            var checksum = new byte[4];
            mem.Read(checksum, 0, 4);
            CheckSum = numberEncoderService.DecodeNumber(checksum);

            var lenBytes = new byte[2];
            mem.Read(lenBytes, 0, 2);
            var recordsInFile = (short)numberEncoderService.DecodeNumber(lenBytes);

            mem.Seek(1, SeekOrigin.Current);

            var rawData = new byte[ESFRecord.DATA_SIZE];
            for (int i = 0; i < recordsInFile && mem.Position < mem.Length; ++i)
            {
                var nameLength = numberEncoderService.DecodeNumber((byte)mem.ReadByte());
                var shoutLength = numberEncoderService.DecodeNumber((byte)mem.ReadByte());
                var rawName = new byte[nameLength];
                var rawShout = new byte[shoutLength];
                mem.Read(rawName, 0, nameLength);
                mem.Read(rawShout, 0, shoutLength);
                mem.Read(rawData, 0, ESFRecord.DATA_SIZE);

                var record = new ESFRecord
                {
                    ID = i,
                    Name = Encoding.ASCII.GetString(rawName),
                    Shout = Encoding.ASCII.GetString(rawShout)
                };
                record.DeserializeFromByteArray(rawData, numberEncoderService);

                _data.Add(record);
            }

            if (recordsInFile != Length)
                throw new IOException("Mismatch between expected length and actual length!");
        }
    }
}

{'fraction_non_alphanumeric': 0.09161663331109629, 'fraction_numerical': 0.009339559706470981, 'mean_word_length': 2.0412440838404327, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6164436', 'n_tokens_mistral': 1251, 'n_tokens_neox': 1175, 'n_words': 301}