<commit_before>// asset-fbx-binary.ts - FBX binary file parser
// Part of Stardazed TX
// (c) 2016 by <NAME> - @zenmumbler

// Implementation partially based on research done by <NAME> of the Blender Foundation
// See: http://code.blender.org/2013/08/fbx-binary-file-format-specification/

/// <reference path="../defs/inflate.d.ts" />

namespace sd.asset {

	interface FieldHeader {
		offset: number;
		endOffset: number;
		propertyCount: number;
		propertiesSizeBytes: number;
		nameLength: number;
		name: string;
	}


	export class FBXBinaryParser {
		private bytes_: Uint8Array;
		private dataView_: DataView;
		private offset_ = 0;
		private length_ = 0;
		private version_ = 0;
		private stack_: FieldHeader[] = [];

		private twoExp21 = Math.pow(2, 21);
		private twoExp32 = Math.pow(2, 32);

		constructor(data: ArrayBuffer, private delegate_: FBXParserDelegate) {
			this.length_ = data.byteLength;
			this.bytes_ = new Uint8Array(data);
			this.dataView_ = new DataView(data);
		}


		private error(msg: string, offset?: number) {
			if (offset == null) {
				offset = this.offset_;
			}
			this.delegate_.error(msg, offset);
			this.offset_ = this.length_;
		}


		get version() { return this.version_; }

		
		private inflateCompressedArray(dataBlock: TypedArray, outElementType: NumericType): TypedArray {
			// the first 2 bytes are usually 0x7801, which is a DEFLATE marker
			// the final 4 bytes are likely a checksum
			assert(dataBlock.byteLength > 6, "Compressed array data size is too small");
			var compData = new Uint8Array(dataBlock.buffer, dataBlock.byteOffset + 2, dataBlock.byteLength - 6);

			var inf = new Inflater();
			var result = inf.append(compData);
			inf.flush();
			
			assert(result.byteLength % outElementType.byteSize == 0, "Invalid aligned size of output buffer");
			return new (outElementType.arrayType)(result.buffer);
		}


		private checkHeader() {
			var ident = String.fromCharCode.apply(null, this.bytes_.subarray(0, 20));
			if (ident != "Kaydara FBX Binary  ") {
				this.error("Not an FBX binary file");
				return false;
			}
			if (this.dataView_.getUint16(21, true) != 0x001A) {
				this.error("Expected 0x001A marker in header");
				return false;	
			}

			this.version_ = this.dataView_.getUint32(23, true);
			if (this.version_ < 7000 || this.version_ >= 8000) {
				this.error("This parser only supports v7.x files.");
				return false;
			}

			this.offset_ = 27;
			return true;
		}


		private readFieldHeader() {
			// fixed offset header fields
			const endOffset = this.dataView_.getUint32(this.offset_, true);
			const propertyCount = this.dataView_.getUint32(this.offset_ + 4, true);
			const propertiesSizeBytes = this.dataView_.getUint32(this.offset_ + 8, true);
			const nameLength = this.dataView_.getUint8(this.offset_ + 12);

			var result: FieldHeader = {
				offset: this.offset_,
				endOffset: endOffset,
				propertyCount: propertyCount,
				propertiesSizeBytes: propertiesSizeBytes,
				nameLength: nameLength,
				name: nameLength > 0 ? String.fromCharCode.apply(null, this.bytes_.subarray(this.offset_ + 13, this.offset_ + 13 + nameLength)) : ""
			};

			this.offset_ += 13 + nameLength;

			return result;
		}


		private readArrayProperty(element: NumericType): TypedArray {
			const arrayLength = this.dataView_.getUint32(this.offset_, true);
			const encoding = this.dataView_.getUint32(this.offset_ + 4, true);
			const compressedSizeBytes = this.dataView_.getUint32(this.offset_ + 8, true);
			this.offset_ += 12;

			var array: TypedArray = null;
			var dataSizeBytes = 0;

			if (encoding == 0) { 
				dataSizeBytes = element.byteSize * arrayLength;
				assert(dataSizeBytes < (this.length_ - this.offset_), "array length out of bounds");

				let source = this.bytes_.subarray(this.offset_, this.offset_ + dataSizeBytes);
				let dest = new Uint8Array(dataSizeBytes);
				dest.set(source);
				array = new (element.arrayType)(dest.buffer);
			}
			else if (encoding == 1) {
				dataSizeBytes = compressedSizeBytes;
				assert(dataSizeBytes < (this.length_ - this.offset_), "array compressed size out of bounds");

				let source = this.bytes_.subarray(this.offset_, this.offset_ + dataSizeBytes);
				array = this.inflateCompressedArray(source, element);
			}
			else {
				console.warn("Unknown array encoding encountered: " + encoding + ". Skipping array.");
				dataSizeBytes = compressedSizeBytes;
			}

			this.offset_ += dataSizeBytes;
			return array;
		}


		private convertInt64ToDouble(dv: DataView, offset: number): number {
			let vLo = dv.getUint32(offset, true);
			let vHi = dv.getInt32(offset + 4, true);

			if (vHi > this.twoExp21 || vHi < -this.twoExp21) {
				console.warn("A 64-bit int property was larger than (+/-)2^53 so it may not be accurately represented.");
			}

			return vLo + vHi * this.twoExp32;
		}


		private readProperties(field: FieldHeader) {
			var count = field.propertyCount;
			var arrayProp: TypedArray = null;
			var props: FBXFieldProp[] = [];
			var firstPropOffset = this.offset_;

			while (count--) {
				let val: FBXFieldProp;
				let type = String.fromCharCode(this.bytes_[this.offset_]);
				let propLen: number;
				this.offset_ += 1;

				switch (type) {
					// String and data
					case 'S':
						// [[fallthrough]]
					case 'R':
						propLen = this.dataView_.getUint32(this.offset_, true);
						if (propLen > 0) {
							let propData = this.bytes_.subarray(this.offset_ + 4, this.offset_ + 4 + propLen);
							if (type == 'S') {
								// TODO: in WebKit at least, this can cause a stack overflow to occur
								// inside fromCharCode if the data is very long, apparently implemented recursively?
								// This is both slow and can lead to errors. Chop this up in pieces and convert
								// separately.
								val = String.fromCharCode.apply(null, propData);
							}
							else {<commit_msg>replace 0x0001 sequence in binary FBX string props with "::" for consistency<commit_after>// asset-fbx-binary.ts - FBX binary file parser
// Part of Stardazed TX
// (c) 2016 by <NAME> - @zenmumbler

// Implementation partially based on research done by <NAME> of the Blender Foundation
// See: http://code.blender.org/2013/08/fbx-binary-file-format-specification/

/// <reference path="../defs/inflate.d.ts" />

namespace sd.asset {

	interface FieldHeader {
		offset: number;
		endOffset: number;
		propertyCount: number;
		propertiesSizeBytes: number;
		nameLength: number;
		name: string;
	}


	export class FBXBinaryParser {
		private bytes_: Uint8Array;
		private dataView_: DataView;
		private offset_ = 0;
		private length_ = 0;
		private version_ = 0;
		private stack_: FieldHeader[] = [];

		private twoExp21 = Math.pow(2, 21);
		private twoExp32 = Math.pow(2, 32);

		constructor(data: ArrayBuffer, private delegate_: FBXParserDelegate) {
			this.length_ = data.byteLength;
			this.bytes_ = new Uint8Array(data);
			this.dataView_ = new DataView(data);
		}


		private error(msg: string, offset?: number) {
			if (offset == null) {
				offset = this.offset_;
			}
			this.delegate_.error(msg, offset);
			this.offset_ = this.length_;
		}


		get version() { return this.version_; }

		
		private inflateCompressedArray(dataBlock: TypedArray, outElementType: NumericType): TypedArray {
			// the first 2 bytes are usually 0x7801, which is a DEFLATE marker
			// the final 4 bytes are likely a checksum
			assert(dataBlock.byteLength > 6, "Compressed array data size is too small");
			var compData = new Uint8Array(dataBlock.buffer, dataBlock.byteOffset + 2, dataBlock.byteLength - 6);

			var inf = new Inflater();
			var result = inf.append(compData);
			inf.flush();
			
			assert(result.byteLength % outElementType.byteSize == 0, "Invalid aligned size of output buffer");
			return new (outElementType.arrayType)(result.buffer);
		}


		private checkHeader() {
			var ident = String.fromCharCode.apply(null, this.bytes_.subarray(0, 20));
			if (ident != "Kaydara FBX Binary  ") {
				this.error("Not an FBX binary file");
				return false;
			}
			if (this.dataView_.getUint16(21, true) != 0x001A) {
				this.error("Expected 0x001A marker in header");
				return false;	
			}

			this.version_ = this.dataView_.getUint32(23, true);
			if (this.version_ < 7000 || this.version_ >= 8000) {
				this.error("This parser only supports v7.x files.");
				return false;
			}

			this.offset_ = 27;
			return true;
		}


		private readFieldHeader() {
			// fixed offset header fields
			const endOffset = this.dataView_.getUint32(this.offset_, true);
			const propertyCount = this.dataView_.getUint32(this.offset_ + 4, true);
			const propertiesSizeBytes = this.dataView_.getUint32(this.offset_ + 8, true);
			const nameLength = this.dataView_.getUint8(this.offset_ + 12);

			var result: FieldHeader = {
				offset: this.offset_,
				endOffset: endOffset,
				propertyCount: propertyCount,
				propertiesSizeBytes: propertiesSizeBytes,
				nameLength: nameLength,
				name: nameLength > 0 ? String.fromCharCode.apply(null, this.bytes_.subarray(this.offset_ + 13, this.offset_ + 13 + nameLength)) : ""
			};

			this.offset_ += 13 + nameLength;

			return result;
		}


		private readArrayProperty(element: NumericType): TypedArray {
			const arrayLength = this.dataView_.getUint32(this.offset_, true);
			const encoding = this.dataView_.getUint32(this.offset_ + 4, true);
			const compressedSizeBytes = this.dataView_.getUint32(this.offset_ + 8, true);
			this.offset_ += 12;

			var array: TypedArray = null;
			var dataSizeBytes = 0;

			if (encoding == 0) { 
				dataSizeBytes = element.byteSize * arrayLength;
				assert(dataSizeBytes < (this.length_ - this.offset_), "array length out of bounds");

				let source = this.bytes_.subarray(this.offset_, this.offset_ + dataSizeBytes);
				let dest = new Uint8Array(dataSizeBytes);
				dest.set(source);
				array = new (element.arrayType)(dest.buffer);
			}
			else if (encoding == 1) {
				dataSizeBytes = compressedSizeBytes;
				assert(dataSizeBytes < (this.length_ - this.offset_), "array compressed size out of bounds");

				let source = this.bytes_.subarray(this.offset_, this.offset_ + dataSizeBytes);
				array = this.inflateCompressedArray(source, element);
			}
			else {
				console.warn("Unknown array encoding encountered: " + encoding + ". Skipping array.");
				dataSizeBytes = compressedSizeBytes;
			}

			this.offset_ += dataSizeBytes;
			return array;
		}


		private convertInt64ToDouble(dv: DataView, offset: number): number {
			let vLo = dv.getUint32(offset, true);
			let vHi = dv.getInt32(offset + 4, true);

			if (vHi > this.twoExp21 || vHi < -this.twoExp21) {
				console.warn("A 64-bit int property was larger than (+/-)2^53 so it may not be accurately represented.");
			}

			return vLo + vHi * this.twoExp32;
		}


		private readProperties(field: FieldHeader) {
			var count = field.propertyCount;
			var arrayProp: TypedArray = null;
			var props: FBXFieldProp[] = [];
			var firstPropOffset = this.offset_;

			while (count--) {
				let val: FBXFieldProp;
				let type = String.fromCharCode(this.bytes_[this.offset_]);
				let propLen: number;
				this.offset_ += 1;

				switch (type) {
					// String and data
					case 'S':
						// [[fallthrough]]
					case 'R':
						propLen = this.dataView_.getUint32(this.offset_, true);
						if (propLen > 0) {
							let propData = this.bytes_.subarray(this.offset_ + 4, this.offset_ + 4 + propLen);
							if (type == 'S') {
								// TODO: in WebKit at least, this can cause a stack overflow to occur
								// inside fromCharCode if the data is very long, apparently implemented recursively?
								// This is both slow and can lead to errors. Chop this up in pieces and convert
								// separately.
								let str: string = String.fromCharCode.apply(null, propData);

								// In binary FBX, the :: separating a name and a classname is replaced with a 0x0001 sequence
								str = str.replace("\x00\x01", "::");

								val = str;
							}
							else {
{'fraction_non_alphanumeric': 0.11767655662348517, 'fraction_numerical': 0.02173004596740493, 'mean_word_length': 3.3074154067674586, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5272309', 'n_tokens_mistral': 4680, 'n_tokens_neox': 3719, 'n_words': 1034}