<commit_before>﻿using kh.common;
using kh.Imaging;
using System;
using System.Drawing;
using System.IO;
using System.Linq;

namespace kh.kh2
{
    public partial class Imgd : IImageRead
	{
		private const uint MagicCode = 0x44474D49U;
		private int unk04 = 0x100;
		private int unk18 = -1;
		private short unk20 = 9;
		private short unk22 = 9;
		private short unk24 = 8;
		private short format;
		private int unk28 = -1;
		private short unk2c;
		private short unk2e;
		private int unk30;

        public static bool IsValid(Stream stream) =>
            stream.Length >= 4 && new BinaryReader(stream).PeekInt32() == MagicCode;

        private short unk34;
		private short unk36;
		private int unk38;
		private int swizzled;

		public Imgd(Stream stream)
		{
			if (!stream.CanRead || !stream.CanSeek)
				throw new InvalidDataException($"Read or seek must be supported.");

			var reader = new BinaryReader(stream);
			if (stream.Length < 16L || reader.ReadUInt32() != MagicCode)
				throw new InvalidDataException("Invalid header");

			unk04 = reader.ReadInt32();
			var dataOffset = reader.ReadInt32();
			var dataLength = reader.ReadInt32();
			var palOffset = reader.ReadInt32();
			var palLength = reader.ReadInt32();
			unk18 = reader.ReadInt32();
			var width = reader.ReadInt16();
			var height = reader.ReadInt16();
			unk20 = reader.ReadInt16();
			unk22 = reader.ReadInt16();
			unk24 = reader.ReadInt16();
			format = reader.ReadInt16();
			unk28 = reader.ReadInt32();
			unk2c = reader.ReadInt16();
			unk2e = reader.ReadInt16();
			unk30 = reader.ReadInt32();
			unk34 = reader.ReadInt16();
			unk36 = reader.ReadInt16();
			unk38 = reader.ReadInt32();
			swizzled = reader.ReadInt32();

			Size = new Size(width, height);
			Data = reader.ReadBytes(dataLength);
			Clut = reader.ReadBytes(palLength);
		}

		public void Save(Stream stream)
		{
			if (!stream.CanWrite || !stream.CanSeek)
				throw new InvalidDataException($"Write or seek must be supported.");

			var writer = new BinaryWriter(stream);
			var baseOffset = (int)stream.Position;

			var dataOffset = baseOffset + 0x40;
			var palOffset = dataOffset + Data.Length;

			writer.Write(MagicCode);
			writer.Write(unk04);
			writer.Write(dataOffset);
			writer.Write(Data.Length);
			writer.Write(palOffset);
			writer.Write(Clut.Length);
			writer.Write(unk18);
			writer.Write((short)Size.Width);
			writer.Write((short)Size.Height);
			writer.Write(unk20);
			writer.Write(unk22);
			writer.Write(unk24);
			writer.Write(format);
			writer.Write(unk28);
			writer.Write(unk2c);
			writer.Write(unk2e);
			writer.Write(unk30);
			writer.Write(unk34);
			writer.Write(unk36);
			writer.Write(unk38);
			writer.Write(swizzled);

			writer.Write(Data, 0, Data.Length);
			writer.Write(Clut, 0, Clut.Length);
		}

		public Size Size { get; }

		public byte[] Data { get; }

		public byte[] Clut { get; }

        public PixelFormat PixelFormat
        {
            get
            {
                switch (format)
                {
                    case 0x13: return PixelFormat.Indexed8;
                    case 0x14: return PixelFormat.Indexed4;
                    default: return PixelFormat.Undefined;
                }
            }
        }

        public byte[] GetData()
		{
			switch (format)
			{
				case 0x13:
                    return swizzled == 7 ? Ps2.Decode8(Ps2.Encode32(Data, Size.Width / 128, Size.Height / 64), Size.Width / 128, Size.Height / 64) : Data;
				case 0x14:
					return swizzled == 7 ? Ps2.Decode4(Ps2.Encode32(Data, Size.Width / 128, Size.Height / 128), Size.Width / 128, Size.Height / 128) : Data;
				default:
					throw new NotSupportedException($"The format {format} is not supported.");
			}
        }

        public byte[] GetClut()
        {
            switch (format)
            {
                case 0x13: return GetClut8();
                case 0x14: return GetClut4();
                default:
                    throw new NotSupportedException($"The format {format} is not supported.");
            }
        }

        public byte[] GetClut4()
        {
            var data = new byte[16 * 4];
            for (var i = 0; i < 16; i++)
            {
                data[i * 4 + 0] = Clut[(i & 15) * 4 + 0];
                data[i * 4 + 1] = Clut[(i & 15) * 4 + 1];
                data[i * 4 + 2] = Clut[(i & 15) * 4 + 2];
                data[i * 4 + 3] = FromPs2Alpha(Clut[(i & 15) * 4 + 3]);
            }

            return data;
        }

        public byte[] GetClut8()
        {
            var data = new byte[256 * 4];
            for (var i = 0; i < 256; i++)
            {
                var srcIndex = Ps2.Repl(i);
                data[i * 4 + 0] = Clut[srcIndex * 4 + 0];
                data[i * 4 + 1] = Clut[srcIndex * 4 + 1];
                data[i * 4 + 2] = Clut[srcIndex * 4 + 2];
                data[i * 4 + 3] = FromPs2Alpha(Clut[srcIndex * 4 + 3]);
            }

            return data;
        }

        public byte FromPs2Alpha(byte alpha) => (byte)Math.Min(alpha * 2, byte.MaxValue);
    }
}<commit_msg>Write back the unknown fields<commit_after>﻿using kh.common;
using kh.Imaging;
using System;
using System.Drawing;
using System.IO;
using System.Linq;

namespace kh.kh2
{
    public partial class Imgd : IImageRead
	{
		private const uint MagicCode = 0x44474D49U;
        private const int Format8bpp = 0x13;
        private const int Format4bpp = 0x14;
        private short format;

        public static bool IsValid(Stream stream) =>
            stream.Length >= 4 && new BinaryReader(stream).PeekInt32() == MagicCode;

		private int swizzled;

		public Imgd(Stream stream)
		{
			if (!stream.CanRead || !stream.CanSeek)
				throw new InvalidDataException($"Read or seek must be supported.");

			var reader = new BinaryReader(stream);
			if (stream.Length < 16L || reader.ReadUInt32() != MagicCode)
				throw new InvalidDataException("Invalid header");

			var unk04 = reader.ReadInt32();
			var dataOffset = reader.ReadInt32();
			var dataLength = reader.ReadInt32();
			var palOffset = reader.ReadInt32();
			var palLength = reader.ReadInt32();
			var unk18 = reader.ReadInt32();
			var width = reader.ReadInt16();
			var height = reader.ReadInt16();
			var powWidth = reader.ReadInt16();
			var powHeight = reader.ReadInt16();
			var widthDiv64 = reader.ReadInt16();
			format = reader.ReadInt16();
			var unk28 = reader.ReadInt32();
			var unk2c = reader.ReadInt16();
			var unk2e = reader.ReadInt16();
			var unk30 = reader.ReadInt32();
            var unk34 = reader.ReadInt16();
			var unk36 = reader.ReadInt16();
			var unk38 = reader.ReadInt32();
			swizzled = reader.ReadInt32();

			Size = new Size(width, height);
			Data = reader.ReadBytes(dataLength);
			Clut = reader.ReadBytes(palLength);
		}

		public void Save(Stream stream)
		{
			if (!stream.CanWrite || !stream.CanSeek)
				throw new InvalidDataException($"Write or seek must be supported.");

			var writer = new BinaryWriter(stream);
			var baseOffset = (int)stream.Position;

			var dataOffset = baseOffset + 0x40;
			var palOffset = dataOffset + Data.Length;

			writer.Write(MagicCode);
			writer.Write(256);
			writer.Write(dataOffset);
			writer.Write(Data.Length);
			writer.Write(palOffset);
			writer.Write(Clut.Length);
			writer.Write(-1);
			writer.Write((short)Size.Width);
			writer.Write((short)Size.Height);
			writer.Write(GetPow((short)Size.Width));
			writer.Write(GetPow((short)Size.Height));
			writer.Write((short)(Size.Width / 64));
			writer.Write(format);
			writer.Write(-1);
			writer.Write((short)(format == Format8bpp ? 16 : 8));
			writer.Write((short)(format == Format8bpp ? 16 : 2));
			writer.Write(1);
			writer.Write((short)(format == Format8bpp ? 5 : 4));
			writer.Write((short)3);
			writer.Write(0);
			writer.Write(swizzled);

            writer.Write(Data, 0, Data.Length);
			writer.Write(Clut, 0, Clut.Length);
		}

		public Size Size { get; }

		public byte[] Data { get; }

		public byte[] Clut { get; }

        public bool IsSwizzled => (swizzled & 4) != 0;

        public PixelFormat PixelFormat
        {
            get
            {
                switch (format)
                {
                    case Format8bpp: return PixelFormat.Indexed8;
                    case Format4bpp: return PixelFormat.Indexed4;
                    default: return PixelFormat.Undefined;
                }
            }
        }

        public byte[] GetData()
		{
			switch (format)
			{
				case Format8bpp:
                    return IsSwizzled ? Ps2.Decode8(Ps2.Encode32(Data, Size.Width / 128, Size.Height / 64), Size.Width / 128, Size.Height / 64) : Data;
				case Format4bpp:
					return IsSwizzled ? Ps2.Decode4(Ps2.Encode32(Data, Size.Width / 128, Size.Height / 128), Size.Width / 128, Size.Height / 128) : Data;
				default:
					throw new NotSupportedException($"The format {format} is not supported.");
			}
        }

        public byte[] GetClut()
        {
            switch (format)
            {
                case Format8bpp: return GetClut8();
                case Format4bpp: return GetClut4();
                default:
                    throw new NotSupportedException($"The format {format} is not supported.");
            }
        }

        public byte[] GetClut4()
        {
            var data = new byte[16 * 4];
            for (var i = 0; i < 16; i++)
            {
                data[i * 4 + 0] = Clut[(i & 15) * 4 + 0];
                data[i * 4 + 1] = Clut[(i & 15) * 4 + 1];
                data[i * 4 + 2] = Clut[(i & 15) * 4 + 2];
                data[i * 4 + 3] = FromPs2Alpha(Clut[(i & 15) * 4 + 3]);
            }

            return data;
        }

        public byte[] GetClut8()
        {
            var data = new byte[256 * 4];
            for (var i = 0; i < 256; i++)
            {
                var srcIndex = Ps2.Repl(i);
                data[i * 4 + 0] = Clut[srcIndex * 4 + 0];
                data[i * 4 + 1] = Clut[srcIndex * 4 + 1];
                data[i * 4 + 2] = Clut[srcIndex * 4 + 2];
                data[i * 4 + 3] = FromPs2Alpha(Clut[srcIndex * 4 + 3]);
            }

            return data;
        }

        private static short GetPow(short value)
        {
            short pow = 1;
            while (value > (1 << pow))
                pow++;

            return pow;
        }

        public byte FromPs2Alpha(byte alpha) => (byte)Math.Min(alpha * 2, byte.MaxValue);
    }
}
{'fraction_non_alphanumeric': 0.12728315960600556, 'fraction_numerical': 0.04475470976379459, 'mean_word_length': 2.09499852027227, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1722734', 'n_tokens_mistral': 3997, 'n_tokens_neox': 3265, 'n_words': 835}