<commit_before>﻿using System;
using System.Runtime.InteropServices;

namespace LibUxie.DSI {
	[StructLayout(LayoutKind.Sequential, Pack = 1)]
	struct Footer {
		public ushort unknown;
		public ushort checksum;
	}

	public enum Version {
		Unknown,
		BlackWhite,
		Black2White2,
	}
	public enum SaveSlot {
		Main,
		Backup
	}

	public class Save {
		private const int BLOCK_START = 0;
		private const int BLOCK_LENGTH = 1;

		/* footer is immediately after the length and is 4 bytes long */
		private static readonly int[,] BlackWhiteBlockData = {
			{0x00000,0x3E0},
			{0x00400,0xFF0},
			{0x01400,0xFF0},
			{0x02400,0xFF0},
			{0x03400,0xFF0},
			{0x04400,0xFF0},
			{0x05400,0xFF0},
			{0x06400,0xFF0},
			{0x07400,0xFF0},
			{0x08400,0xFF0},
			{0x09400,0xFF0},
			{0x0A400,0xFF0},
			{0x0B400,0xFF0},
			{0x0C400,0xFF0},
			{0x0D400,0xFF0},
			{0x0E400,0xFF0},
			{0x0F400,0xFF0},
			{0x10400,0xFF0},
			{0x11400,0xFF0},
			{0x12400,0xFF0},
			{0x13400,0xFF0},
			{0x14400,0xFF0},
			{0x15400,0xFF0},
			{0x16400,0xFF0},
			{0x17400,0xFF0},
			{0x18400,0x9C0},
			{0x18E00,0x534},
			{0x19400,0x68},
			{0x19500,0x9C},
			{0x19600,0x1338},
			{0x1AA00,0x7C4},
			{0x1B200,0xD54},
			{0x1C000,0x2C},
			{0x1C100,0x658},
			{0x1C800,0xA94},
			{0x1D300,0x1AC},
			{0x1D500,0x3EC},
			{0x1D900,0x5C},
			{0x1DA00,0x1E0},
			{0x1DC00,0xA8},
			{0x1DD00,0x460},
			{0x1E200,0x1400},
			{0x1F700,0x2A4},
			{0x1FA00,0x2DC},
			{0x1FD00,0x34C},
			{0x20100,0x3EC},
			{0x20500,0xF8},
			{0x20600,0x2FC},
			{0x20900,0x94},
			{0x20A00,0x35C},
			{0x20E00,0x1CC},
			{0x21000,0x168},
			{0x21200,0xEC},
			{0x21300,0x1B0},
			{0x21500,0x1C},
			{0x21600,0x4D4},
			{0x21B00,0x34},
			{0x21C00,0x3C},
			{0x21D00,0x1AC},
			{0x21F00,0xB90},
			{0x22B00,0x9C},
			{0x22C00,0x850},
			{0x23500,0x28},
			{0x23600,0x284},
			{0x23900,0x10},
			{0x23A00,0x5C},
			{0x23B00,0x16C},
			{0x23D00,0x40},
			{0x23E00,0xFC}
		};

		/* footer is immediately after the length and is 4 bytes long */
		private static readonly int[,] Black2White2BlockData = {
			{0x00000,0x3E0},
			{0x00400,0xFF0},
			{0x01400,0xFF0},
			{0x02400,0xFF0},
			{0x03400,0xFF0},
			{0x04400,0xFF0},
			{0x05400,0xFF0},
			{0x06400,0xFF0},
			{0x07400,0xFF0},
			{0x08400,0xFF0},
			{0x09400,0xFF0},
			{0x0A400,0xFF0},
			{0x0B400,0xFF0},
			{0x0C400,0xFF0},
			{0x0D400,0xFF0},
			{0x0E400,0xFF0},
			{0x0F400,0xFF0},
			{0x10400,0xFF0},
			{0x11400,0xFF0},
			{0x12400,0xFF0},
			{0x13400,0xFF0},
			{0x14400,0xFF0},
			{0x15400,0xFF0},
			{0x16400,0xFF0},
			{0x17400,0xFF0},
			{0x18400,0x9EC},
			{0x18E00,0x534},
			{0x19400,0xB0},
			{0x19500,0xA8},
			{0x19600,0x1338},
			{0x1AA00,0x7C4},
			{0x1B200,0xD54},
			{0x1C000,0x94},
			{0x1C100,0x658},
			{0x1C800,0xA94},
			{0x1D300,0x1AC},
			{0x1D500,0x3EC},
			{0x1D900,0x5C},
			{0x1DA00,0x1E0},
			{0x1DC00,0xA8},
			{0x1DD00,0x460},
			{0x1E200,0x1400},
			{0x1F700,0x2A4},
			{0x1FA00,0xE0},
			{0x1FB00,0x34C},
			{0x1FF00,0x4E0},
			{0x20400,0xF8},
			{0x20500,0x2FC},
			{0x20800,0x94},
			{0x20900,0x35C},
			{0x20D00,0x1D4},
			{0x20F00,0x1E0},
			{0x21100,0xF0},
			{0x21200,0x1B4},
			{0x21400,0x4DC},
			{0x21900,0x34},
			{0x21A00,0x3C},
			{0x21B00,0x1AC},
			{0x21D00,0xB90},
			{0x22900,0xAC},
			{0x22A00,0x850},
			{0x23300,0x284},
			{0x23600,0x10},
			{0x23700,0xA8},
			{0x23800,0x16C},
			{0x23A00,0x80},
			{0x23B00,0xFC},
			{0x23C00,0x16A8},
			{0x25300,0x498},
        };

		private ushort GetBlockChecksum(int block) {
			if(type == Version.BlackWhite) {
				return Utilities.Checksum.CRC16(savdata, BlackWhiteBlockData[block, BLOCK_START], BlackWhiteBlockData[block, BLOCK_LENGTH]);
			} else if(type == Version.Black2White2) {
				return Utilities.Checksum.CRC16(savdata, Black2White2BlockData[block, BLOCK_START], Black2White2BlockData[block, BLOCK_LENGTH]);
			}
			return 0;
		}

		private const int SAV_SIZE = 0x80000;

		private Version type;
		private byte[] savdata;

		private static Version DetectVersion(byte[] data) {
			if(data.Length != SAV_SIZE) {
				return Version.Unknown;
			}

			const int BW_MARK_OFFSET = 0x23F94;
			const int B2W2_MARK_OFFSET = 0x25F9C;
			const uint MARK =  0x31053527;

			if(BitConverter.ToUInt32(data, BW_MARK_OFFSET) == MARK) {
				return Version.BlackWhite;
			} else if(BitConverter.ToUInt32(data, B2W2_MARK_OFFSET) == MARK) {
				return Version.Black2White2;
			}

			return Version.Unknown;
		}

		public bool Load(byte[] data) {
			Version v = DetectVersion(data);
			if(v != Version.Unknown) {
				//TODO maybe copy this.
				savdata = data;
				return true;
			}
			return false;
		}

		public Version Version {
			get {
				return type;
			}
			set {
				type = value;
			}
		}
	}
}
<commit_msg>Streamline DSI block data a bit more.<commit_after>﻿using System;
using System.Runtime.InteropServices;

namespace LibUxie.DSI {
	[StructLayout(LayoutKind.Sequential, Pack = 1)]
	struct Footer {
		public ushort unknown;
		public ushort checksum;
	}

	public enum Version {
		BlackWhite = 0,
		Black2White2 = 1,
		Unknown
	}
	public enum SaveSlot {
		Main,
		Backup
	}

	public class Save {
		private const int BLOCK_START = 0;
		private const int BLOCK_LENGTH = 1;

		/* [block,version,length] */
		private static readonly int[,,] BlockData = {
			{ {0x00000,0x3E0}, {0x00000,0x3E0} },
			{ {0x00400,0xFF0}, {0x00400,0xFF0} },
			{ {0x01400,0xFF0}, {0x01400,0xFF0} },
			{ {0x02400,0xFF0}, {0x02400,0xFF0} },
			{ {0x03400,0xFF0}, {0x03400,0xFF0} },
			{ {0x04400,0xFF0}, {0x04400,0xFF0} },
			{ {0x05400,0xFF0}, {0x05400,0xFF0} },
			{ {0x06400,0xFF0}, {0x06400,0xFF0} },
			{ {0x07400,0xFF0}, {0x07400,0xFF0} },
			{ {0x08400,0xFF0}, {0x08400,0xFF0} },
			{ {0x09400,0xFF0}, {0x09400,0xFF0} },
			{ {0x0A400,0xFF0}, {0x0A400,0xFF0} },
			{ {0x0B400,0xFF0}, {0x0B400,0xFF0} },
			{ {0x0C400,0xFF0}, {0x0C400,0xFF0} },
			{ {0x0D400,0xFF0}, {0x0D400,0xFF0} },
			{ {0x0E400,0xFF0}, {0x0E400,0xFF0} },
			{ {0x0F400,0xFF0}, {0x0F400,0xFF0} },
			{ {0x10400,0xFF0}, {0x10400,0xFF0} },
			{ {0x11400,0xFF0}, {0x11400,0xFF0} },
			{ {0x12400,0xFF0}, {0x12400,0xFF0} },
			{ {0x13400,0xFF0}, {0x13400,0xFF0} },
			{ {0x14400,0xFF0}, {0x14400,0xFF0} },
			{ {0x15400,0xFF0}, {0x15400,0xFF0} },
			{ {0x16400,0xFF0}, {0x16400,0xFF0} },
			{ {0x17400,0xFF0}, {0x17400,0xFF0} },
			{ {0x18400,0x9C0}, {0x18400,0x9EC} },
			{ {0x18E00,0x534}, {0x18E00,0x534} },
			{ {0x19400,0x68}, {0x19400,0xB0} },
			{ {0x19500,0x9C}, {0x19500,0xA8} },
			{ {0x19600,0x1338}, {0x19600,0x1338} },
			{ {0x1AA00,0x7C4}, {0x1AA00,0x7C4} },
			{ {0x1B200,0xD54}, {0x1B200,0xD54} },
			{ {0x1C000,0x2C}, {0x1C000,0x94} },
			{ {0x1C100,0x658}, {0x1C100,0x658} },
			{ {0x1C800,0xA94}, {0x1C800,0xA94} },
			{ {0x1D300,0x1AC}, {0x1D300,0x1AC} },
			{ {0x1D500,0x3EC}, {0x1D500,0x3EC} },
			{ {0x1D900,0x5C}, {0x1D900,0x5C} },
			{ {0x1DA00,0x1E0}, {0x1DA00,0x1E0} },
			{ {0x1DC00,0xA8}, {0x1DC00,0xA8} },
			{ {0x1DD00,0x460}, {0x1DD00,0x460} },
			{ {0x1E200,0x1400}, {0x1E200,0x1400} },
			{ {0x1F700,0x2A4}, {0x1F700,0x2A4} },
			{ {0x1FA00,0x2DC}, {0x1FA00,0xE0} },
			{ {0x1FD00,0x34C}, {0x1FB00,0x34C} },
			{ {0x20100,0x3EC}, {0x1FF00,0x4E0} },
			{ {0x20500,0xF8}, {0x20400,0xF8} },
			{ {0x20600,0x2FC}, {0x20500,0x2FC} },
			{ {0x20900,0x94}, {0x20800,0x94} },
			{ {0x20A00,0x35C}, {0x20900,0x35C} },
			{ {0x20E00,0x1CC}, {0x20D00,0x1D4} },
			{ {0x21000,0x168}, {0x20F00,0x1E0} },
			{ {0x21200,0xEC}, {0x21100,0xF0} },
			{ {0x21300,0x1B0}, {0x21200,0x1B4} },
			{ {0x21500,0x1C}, {0x21400,0x4DC} },
			{ {0x21600,0x4D4}, {0x21900,0x34} },
			{ {0x21B00,0x34}, {0x21A00,0x3C} },
			{ {0x21C00,0x3C}, {0x21B00,0x1AC} },
			{ {0x21D00,0x1AC}, {0x21D00,0xB90} },
			{ {0x21F00,0xB90}, {0x22900,0xAC} },
			{ {0x22B00,0x9C}, {0x22A00,0x850} },
			{ {0x22C00,0x850}, {0x23300,0x284} },
			{ {0x23500,0x28}, {0x23600,0x10} },
			{ {0x23600,0x284}, {0x23700,0xA8} },
			{ {0x23900,0x10}, {0x23800,0x16C} },
			{ {0x23A00,0x5C}, {0x23A00,0x80} },
			{ {0x23B00,0x16C}, {0x23B00,0xFC} },
			{ {0x23D00,0x40}, {0x23C00,0x16A8} },
			{ {0x23E00,0xFC}, {0x25300,0x498} }
		};

		private ushort GetBlockChecksum(int block) {
			if(type == Version.Unknown) {
				return 0;
			}
			return Utilities.Checksum.CRC16(savdata, BlockData[block, (int)type, BLOCK_START], BlockData[block, (int)type, BLOCK_LENGTH]);
		}

		private const int SAV_SIZE = 0x80000;

		private Version type;
		private byte[] savdata;

		private static Version DetectVersion(byte[] data) {
			if(data.Length != SAV_SIZE) {
				return Version.Unknown;
			}

			const int BW_MARK_OFFSET = 0x23F94;
			const int B2W2_MARK_OFFSET = 0x25F9C;
			const uint MARK =  0x31053527;

			if(BitConverter.ToUInt32(data, BW_MARK_OFFSET) == MARK) {
				return Version.BlackWhite;
			} else if(BitConverter.ToUInt32(data, B2W2_MARK_OFFSET) == MARK) {
				return Version.Black2White2;
			}

			return Version.Unknown;
		}

		public bool Load(byte[] data) {
			Version v = DetectVersion(data);
			if(v != Version.Unknown) {
				//TODO maybe copy this.
				savdata = data;
				return true;
			}
			return false;
		}

		public Version Version {
			get {
				return type;
			}
			set {
				type = value;
			}
		}
	}
}

{'fraction_non_alphanumeric': 0.17513134851138354, 'fraction_numerical': 0.2508756567425569, 'mean_word_length': 3.9150080688542226, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '733353', 'n_tokens_mistral': 6264, 'n_tokens_neox': 4785, 'n_words': 589}