<commit_before>import PMXHeader = require("./PMXHeader");
import PMXVerticies = require("./PMXVerticies");
import PMXMaterial = require("./PMXMaterial");
import PMXBone = require("./PMXBone");
import PMXMorph = require("./PMXMorph");
import PMXDisplayFrame = require("./PMXDisplayFrame");
import PMXRigidBody = require("./PMXRigidBody");
import PMXJoint = require("./PMXJoint");
class PMX {
	private reader: jDataView;

	private header: PMXHeader;

	private verticies: PMXVerticies;

	private surfaces: number[];

	private textures: string[];

	private materials: PMXMaterial[];

	private bones: PMXBone[];

	private morphs: PMXMorph[];

	private displayFrames: PMXDisplayFrame[];

	private rigidBodies: PMXRigidBody[];

	private joints: PMXJoint[];

	public get Verticies() {
		return this.verticies;
	}

	public get Surfaces() {
		return this.surfaces;
	}

	public get Materials() {
		return this.materials;
	}

	public get Textures() {
		return this.textures;
	}

	public get Bones() {
		return this.bones;
	}

	public get Morphs()
	{
		return this.morphs;
	}

	constructor(data: ArrayBuffer) {
		this.reader = new jDataView(data, 0, data.byteLength, true);
		this.loadHeader();
		this.loadVerticies();
		this.loadSurfaces();
		this.loadTextures();
		this.loadMaterials();
		this.loadBones();
		this.loadMorphs();
		this.loadDisplayFrames();
		this.loadRigidBodies();
		this.loadJoints();
	}

	private readTextBuf(): string {
		var length = this.reader.getInt32();
		return this.reader.getString(length, this.reader.tell(), "utf8");
	}

	private toUTF8(str: string) {
		var utf8 = [];
		for (var i = 0; i < str.length; i++) {
			var charcode = str.charCodeAt(i);
			if (charcode < 0x80) utf8.push(charcode);
			else if (charcode < 0x800) {
				utf8.push(0xc0 | (charcode >> 6),
					0x80 | (charcode & 0x3f));
			}
			else if (charcode < 0xd800 || charcode >= 0xe000) {
				utf8.push(0xe0 | (charcode >> 12),
					0x80 | ((charcode >> 6) & 0x3f),
					0x80 | (charcode & 0x3f));
			}
			// surrogate pair
			else {
				i++;
				// UTF-16 encodes 0x10000-0x10FFFF by
				// subtracting 0x10000 and splitting the
				// 20 bits of 0x0-0xFFFFF into two halves
				charcode = 0x10000 + (((charcode & 0x3ff) << 10)
					| (str.charCodeAt(i) & 0x3ff))
				utf8.push(0xf0 | (charcode >> 18),
					0x80 | ((charcode >> 12) & 0x3f),
					0x80 | ((charcode >> 6) & 0x3f),
					0x80 | (charcode & 0x3f));
			}
		}
		var text = "";
		for (var i = 0; i < utf8.length; i++) {
			text += String.fromCharCode(utf8[i]);
		}
		return text;
	}

	private loadHeader() {
		var r = this.reader;
		this.reader.getUint32();//pass magic
		this.header =
		{
			version: r.getFloat32(),
			headerByteSize: r.getUint8(),
			encoding: r.getUint8(),
			uvAddition: r.getUint8(),
			vertexIndexSize: r.getUint8(),
			textureIndexSize: r.getUint8(),
			materialIndexSize: r.getUint8(),
			boneIndexSize: r.getUint8(),
			morphIndexSize: r.getUint8(),<commit_msg>:sparkles: support for utf16le pmx format<commit_after>import PMXHeader = require("./PMXHeader");
import PMXVerticies = require("./PMXVerticies");
import PMXMaterial = require("./PMXMaterial");
import PMXBone = require("./PMXBone");
import PMXMorph = require("./PMXMorph");
import PMXDisplayFrame = require("./PMXDisplayFrame");
import PMXRigidBody = require("./PMXRigidBody");
import PMXJoint = require("./PMXJoint");
class PMX {
	private reader: jDataView;

	private header: PMXHeader;

	private verticies: PMXVerticies;

	private surfaces: number[];

	private textures: string[];

	private materials: PMXMaterial[];

	private bones: PMXBone[];

	private morphs: PMXMorph[];

	private displayFrames: PMXDisplayFrame[];

	private rigidBodies: PMXRigidBody[];

	private joints: PMXJoint[];

	public get Verticies() {
		return this.verticies;
	}

	public get Surfaces() {
		return this.surfaces;
	}

	public get Materials() {
		return this.materials;
	}

	public get Textures() {
		return this.textures;
	}

	public get Bones() {
		return this.bones;
	}

	public get Morphs()
	{
		return this.morphs;
	}

	constructor(data: ArrayBuffer) {
		this.reader = new jDataView(data, 0, data.byteLength, true);
		this.loadHeader();
		this.loadVerticies();
		this.loadSurfaces();
		this.loadTextures();
		this.loadMaterials();
		this.loadBones();
		this.loadMorphs();
		this.loadDisplayFrames();
		this.loadRigidBodies();
		this.loadJoints();
	}

	private readTextBuf(): string {
		var length = this.reader.getInt32();
		if(this.header.encoding == 0)
		{
			//When this was UTF-16LE
			var textArr = new Uint16Array(length);
			for(var i = 0; i < length/2; i++)
			{
				textArr[i] = this.reader.getUint16();
			}
			return String.fromCharCode.apply(null,textArr)
		}
		return this.reader.getString(length, this.reader.tell(), "utf8");
	}

	private loadHeader() {
		var r = this.reader;
		this.reader.getUint32();//pass magic
		this.header =
		{
			version: r.getFloat32(),
			headerByteSize: r.getUint8(),
			encoding: r.getUint8(),
			uvAddition: r.getUint8(),
			vertexIndexSize: r.getUint8(),
			textureIndexSize: r.getUint8(),
			materialIndexSize: r.getUint8(),
			boneIndexSize: r.getUint8(),
			morphIndexSize: r.getUint8(),
{'fraction_non_alphanumeric': 0.15056266977105162, 'fraction_numerical': 0.02852153667054715, 'mean_word_length': 3.854048964218456, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1178458', 'n_tokens_mistral': 2156, 'n_tokens_neox': 1863, 'n_words': 409}