<commit_before>/* 
 * Copyright (c) Telecom ParisTech/TSI/MM/GPAC <NAME>
 * License: BSD-3-Clause (see LICENSE file)
 */
BoxParser.parseOneBox = function(stream, headerOnly) {
	var box;
	var start = stream.getPosition();
	var hdr_size = 0;
	var uuid;
	if (stream.getEndPosition() - start < 8) {
		Log.debug("BoxParser", "Not enough data in stream to parse the type and size of the box");
		return { code: BoxParser.ERR_NOT_ENOUGH_DATA };
	}
	var size = stream.readUint32();
	var type = stream.readString(4);
	Log.debug("BoxParser", "Found box of type "+type+" and size "+size+" at position "+start);
	hdr_size = 8;
	if (type == "uuid") {
		uuid = stream.readUint8Array(16);
		hdr_size += 16;
	}
	if (size == 1) {
		if (stream.getEndPosition() - stream.getPosition() < 8) {
			stream.seek(start);
			Log.warn("BoxParser", "Not enough data in stream to parse the extended size of the \""+type+"\" box");
			return { code: BoxParser.ERR_NOT_ENOUGH_DATA };
		}
		size = stream.readUint64();
		hdr_size += 8;
	} else if (size === 0) {
		/* box extends till the end of file */
		if (type !== "mdat") {
			throw "Unlimited box size not supported";
		}
	}
	
	if (start + size > stream.getEndPosition()) {
		stream.seek(start);
		Log.warn("BoxParser", "Not enough data in stream to parse the entire \""+type+"\" box");
		return { code: BoxParser.ERR_NOT_ENOUGH_DATA, type: type, size: size, hdr_size: hdr_size, start: start };
	}
	if (headerOnly) {
		return { code: BoxParser.OK, type: type, size: size, hdr_size: hdr_size, start: start };
	} else {
		if (BoxParser[type+"Box"]) {
			if (BoxParser.parseForWrite && BoxParser[type+"Box"].prototype.write === BoxParser.Box.prototype.write &&
				type !== "mdat" && type !== "skip" && type != "free") {
				Log.warn("BoxParser", type+" box writing not yet implemented, forcing default parsing");
				box = new BoxParser.Box(type, size);
			} else {
				box = new BoxParser[type+"Box"](size);		
			}
		} else {
			if (type !== "uuid") {
				Log.warn("BoxParser", "Unknown box type: "+type);
			}
			box = new BoxParser.Box(type, size);
			if (uuid) {
				box.uuid = uuid;
			}
		}
	}
	/* recording the position of the box in the input stream */
	box.hdr_size = hdr_size;
	box.start = start;
	box.parse(stream);
	return { code: BoxParser.OK, box: box, size: size };
}

BoxParser.Box.prototype.parse = function(stream) {
	if (this.type != "mdat") {<commit_msg>esds is deep parsed, special case<commit_after>/* 
 * Copyright (c) Telecom ParisTech/TSI/MM/GPAC <NAME>
 * License: BSD-3-Clause (see LICENSE file)
 */
BoxParser.parseOneBox = function(stream, headerOnly) {
	var box;
	var start = stream.getPosition();
	var hdr_size = 0;
	var uuid;
	if (stream.getEndPosition() - start < 8) {
		Log.debug("BoxParser", "Not enough data in stream to parse the type and size of the box");
		return { code: BoxParser.ERR_NOT_ENOUGH_DATA };
	}
	var size = stream.readUint32();
	var type = stream.readString(4);
	Log.debug("BoxParser", "Found box of type "+type+" and size "+size+" at position "+start);
	hdr_size = 8;
	if (type == "uuid") {
		uuid = stream.readUint8Array(16);
		hdr_size += 16;
	}
	if (size == 1) {
		if (stream.getEndPosition() - stream.getPosition() < 8) {
			stream.seek(start);
			Log.warn("BoxParser", "Not enough data in stream to parse the extended size of the \""+type+"\" box");
			return { code: BoxParser.ERR_NOT_ENOUGH_DATA };
		}
		size = stream.readUint64();
		hdr_size += 8;
	} else if (size === 0) {
		/* box extends till the end of file */
		if (type !== "mdat") {
			throw "Unlimited box size not supported";
		}
	}
	
	if (start + size > stream.getEndPosition()) {
		stream.seek(start);
		Log.warn("BoxParser", "Not enough data in stream to parse the entire \""+type+"\" box");
		return { code: BoxParser.ERR_NOT_ENOUGH_DATA, type: type, size: size, hdr_size: hdr_size, start: start };
	}
	if (headerOnly) {
		return { code: BoxParser.OK, type: type, size: size, hdr_size: hdr_size, start: start };
	} else {
		if (BoxParser[type+"Box"]) {
			if (BoxParser.parseForWrite && BoxParser[type+"Box"].prototype.write === BoxParser.Box.prototype.write &&
				type !== "mdat" && type !== "skip" && type != "free" && type != "esds") {
				Log.warn("BoxParser", type+" box writing not yet implemented, forcing default parsing");
				box = new BoxParser.Box(type, size);
			} else {
				box = new BoxParser[type+"Box"](size);		
			}
		} else {
			if (type !== "uuid") {
				Log.warn("BoxParser", "Unknown box type: "+type);
			}
			box = new BoxParser.Box(type, size);
			if (uuid) {
				box.uuid = uuid;
			}
		}
	}
	/* recording the position of the box in the input stream */
	box.hdr_size = hdr_size;
	box.start = start;
	box.parse(stream);
	return { code: BoxParser.OK, box: box, size: size };
}

BoxParser.Box.prototype.parse = function(stream) {
	if (this.type != "mdat") {
{'fraction_non_alphanumeric': 0.16425020712510358, 'fraction_numerical': 0.007456503728251864, 'mean_word_length': 3.661196911196911, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3575832', 'n_tokens_mistral': 1837, 'n_tokens_neox': 1680, 'n_words': 485}