<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Text;

namespace HedgeLib.IO
{
    // This class was purposely written to avoid unnecessary method
    // calls for performance, hence its extreme length.
    public class ExtendedBinaryReader : BinaryReader
    {
        // Variables/Constants
        public uint Offset = 0;
        public bool IsBigEndian = false;

        protected Stream stream;
        protected byte[] buffer;

        protected const string EndOfStream =
            "Cannot read; the end of the Stream has been reached!";

        protected const string StreamClosed =
            "Cannot read; the Stream has been closed!";

        protected const int MinBufferSize = 16;
        protected const char NullChar = '\0';

        // Constructors
        public ExtendedBinaryReader(Stream input, bool isBigEndian = false) :
            this(input, Encoding.ASCII, isBigEndian) { }

        public ExtendedBinaryReader(Stream input, Encoding encoding,
            bool isBigEndian = false) : base(input, encoding, false)
        {
            stream = input;
            IsBigEndian = isBigEndian;

            int bufferSize = encoding.GetMaxByteCount(1);
            if (bufferSize < MinBufferSize)
                bufferSize = MinBufferSize;

            buffer = new byte[bufferSize];
        }

        // Methods
        public void JumpTo(long position, bool absolute = true)
        {
            BaseStream.Position = (absolute) ? position : position + Offset;
        }

        public void JumpAhead(long amount = 1)
        {
            BaseStream.Position += amount;
        }

        public void JumpBehind(long amount = 1)
        {
            BaseStream.Position -= amount;
        }

        public void FixPadding(uint amount = 4)
        {
            if (amount < 1) return;

            long jumpAmount = 0;
            while ((BaseStream.Position + jumpAmount) % amount != 0) ++jumpAmount;
            JumpAhead(jumpAmount);
        }

        public string GetString(bool isAbsolute = false, bool isNullTerminated = true)
        {
            uint offset = (isAbsolute) ? ReadUInt32() : ReadUInt32() + Offset;
            return GetString(offset, isNullTerminated);
        }

        public string GetString(uint offset, bool isNullTerminated = true)
        {
            long curPos = BaseStream.Position;
            BaseStream.Position = offset;

            string str = (isNullTerminated) ?
                ReadNullTerminatedString() : ReadString();

            BaseStream.Position = curPos;
            return str;
        }

        public string ReadSignature(int length = 4)
        {
            var chars = ReadChars(length);
            return new string(chars);
        }

        public string ReadNullTerminatedString()
        {
            var sb = new StringBuilder();
            var fs = BaseStream;
            long len = fs.Length;
            char curChar;

            do
            {
                curChar = ReadChar();
                if (curChar == NullChar) break;
                sb.Append(curChar);
            }
            while (fs.Position < len);
            return sb.ToString();
        }

        public string ReadNullTerminatedStringUTF16()
        {<commit_msg>Changed ReadSignature to use ASCII Should've done this from the start lol. Should fix BINA v1 Header reading<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Text;

namespace HedgeLib.IO
{
    // This class was purposely written to avoid unnecessary method
    // calls for performance, hence its extreme length.
    public class ExtendedBinaryReader : BinaryReader
    {
        // Variables/Constants
        public uint Offset = 0;
        public bool IsBigEndian = false;

        protected Stream stream;
        protected byte[] buffer;

        protected const string EndOfStream =
            "Cannot read; the end of the Stream has been reached!";

        protected const string StreamClosed =
            "Cannot read; the Stream has been closed!";

        protected const int MinBufferSize = 16;
        protected const char NullChar = '\0';

        // Constructors
        public ExtendedBinaryReader(Stream input, bool isBigEndian = false) :
            this(input, Encoding.ASCII, isBigEndian) { }

        public ExtendedBinaryReader(Stream input, Encoding encoding,
            bool isBigEndian = false) : base(input, encoding, false)
        {
            stream = input;
            IsBigEndian = isBigEndian;

            int bufferSize = encoding.GetMaxByteCount(1);
            if (bufferSize < MinBufferSize)
                bufferSize = MinBufferSize;

            buffer = new byte[bufferSize];
        }

        // Methods
        public void JumpTo(long position, bool absolute = true)
        {
            BaseStream.Position = (absolute) ? position : position + Offset;
        }

        public void JumpAhead(long amount = 1)
        {
            BaseStream.Position += amount;
        }

        public void JumpBehind(long amount = 1)
        {
            BaseStream.Position -= amount;
        }

        public void FixPadding(uint amount = 4)
        {
            if (amount < 1) return;

            long jumpAmount = 0;
            while ((BaseStream.Position + jumpAmount) % amount != 0) ++jumpAmount;
            JumpAhead(jumpAmount);
        }

        public string GetString(bool isAbsolute = false, bool isNullTerminated = true)
        {
            uint offset = (isAbsolute) ? ReadUInt32() : ReadUInt32() + Offset;
            return GetString(offset, isNullTerminated);
        }

        public string GetString(uint offset, bool isNullTerminated = true)
        {
            long curPos = BaseStream.Position;
            BaseStream.Position = offset;

            string str = (isNullTerminated) ?
                ReadNullTerminatedString() : ReadString();

            BaseStream.Position = curPos;
            return str;
        }

        public string ReadSignature(int length = 4)
        {
            return Encoding.ASCII.GetString(ReadBytes(length));
        }

        public string ReadNullTerminatedString()
        {
            var sb = new StringBuilder();
            var fs = BaseStream;
            long len = fs.Length;
            char curChar;

            do
            {
                curChar = ReadChar();
                if (curChar == NullChar) break;
                sb.Append(curChar);
            }
            while (fs.Position < len);
            return sb.ToString();
        }

        public string ReadNullTerminatedStringUTF16()
        {
{'fraction_non_alphanumeric': 0.07717618905174993, 'fraction_numerical': 0.005533951540532456, 'mean_word_length': 1.7586633663366336, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7632641', 'n_tokens_mistral': 1725, 'n_tokens_neox': 1609, 'n_words': 501}