<commit_before>/*
 * This class is used to encode ccnb binary elements (blob, type/value pairs).
 * 
 * @author: ucla-cs
 * See COPYING for copyright and distribution information.
 */

var XML_EXT = 0x00; 
	
var XML_TAG = 0x01; 
	
var XML_DTAG = 0x02; 
	
var XML_ATTR = 0x03; 
 
var XML_DATTR = 0x04; 
	
var XML_BLOB = 0x05; 
	
var XML_UDATA = 0x06; 
	
var XML_CLOSE = 0x0;

var XML_SUBTYPE_PROCESSING_INSTRUCTIONS = 16; 


var XML_TT_BITS = 3;
var XML_TT_MASK = ((1 << XML_TT_BITS) - 1);
var XML_TT_VAL_BITS = XML_TT_BITS + 1;
var XML_TT_VAL_MASK = ((1 << (XML_TT_VAL_BITS)) - 1);
var XML_REG_VAL_BITS = 7;
var XML_REG_VAL_MASK = ((1 << XML_REG_VAL_BITS) - 1);
var XML_TT_NO_MORE = (1 << XML_REG_VAL_BITS); // 0x80
var BYTE_MASK = 0xFF;
var LONG_BYTES = 8;
var LONG_BITS = 64;
	
var bits_11 = 0x0000007FF;
var bits_18 = 0x00003FFFF;
var bits_32 = 0x0FFFFFFFF;


var BinaryXMLEncoder = function BinaryXMLEncoder(){
	this.ostream = new Uint8Array(10000);
	this.offset =0;
	this.CODEC_NAME = "Binary";
};


BinaryXMLEncoder.prototype.writeUString = function(/*String*/ utf8Content) {
	this.encodeUString(this.ostream, utf8Content, XML_UDATA);
};


BinaryXMLEncoder.prototype.writeBlob = function(
		/*Uint8Array*/ binaryContent
		//, /*int*/ offset, /*int*/ length
		) {
	
	if(LOG >3) console.log(binaryContent);
	
	this.encodeBlob(this.ostream, binaryContent, this.offset, binaryContent.length);
};


BinaryXMLEncoder.prototype.writeStartElement = function(
	/*String*/ tag, 
	/*TreeMap<String,String>*/ attributes
	) {

	/*Long*/ dictionaryVal = tag; //stringToTag(tag);
	
	if (null == dictionaryVal) {
		this.encodeUString(this.ostream, tag, XML_TAG);
	} else {
		this.encodeTypeAndVal(XML_DTAG, dictionaryVal, this.ostream);
	}
	
	if (null != attributes) {
		this.writeAttributes(attributes); 
	}
};


BinaryXMLEncoder.prototype.writeEndElement = function() {
	this.ostream[this.offset] = XML_CLOSE;
	this.offset += 1;
}


BinaryXMLEncoder.prototype.writeAttributes = function(/*TreeMap<String,String>*/ attributes) {
	if (null == attributes) {
		return;
	}

	// the keySet of a TreeMap is sorted.

	for(var i=0; i<attributes.length;i++){
		var strAttr = attributes[i].k;
		var strValue = attributes[i].v;

		var dictionaryAttr = stringToTag(strAttr);
		if (null == dictionaryAttr) {
			// not in dictionary, encode as attr
			// compressed format wants length of tag represented as length-1
			// to save that extra bit, as tag cannot be 0 length.
			// encodeUString knows to do that.
			this.encodeUString(this.ostream, strAttr, XML_ATTR);
		} else {
			this.encodeTypeAndVal(XML_DATTR, dictionaryAttr, this.ostream);
		}
		// Write value
		this.encodeUString(this.ostream, strValue);
		
	}
}


//returns a string
stringToTag = function(/*long*/ tagVal) {
	if ((tagVal >= 0) && (tagVal < CCNProtocolDTagsStrings.length)) {
		return CCNProtocolDTagsStrings[tagVal];
	} else if (tagVal == CCNProtocolDTags.CCNProtocolDataUnit) {
		return CCNProtocolDTags.CCNPROTOCOL_DATA_UNIT;
	}
	return null;
};

//returns a Long
tagToString =  function(/*String*/ tagName) {
	// the slow way, but right now we don't care.... want a static lookup for the forward direction
	for (var i=0; i < CCNProtocolDTagsStrings.length; ++i) {
		if ((null != CCNProtocolDTagsStrings[i]) && (CCNProtocolDTagsStrings[i] == tagName)) {
			return i;
		}
	}
	if (CCNProtocolDTags.CCNPROTOCOL_DATA_UNIT == tagName) {
		return CCNProtocolDTags.CCNProtocolDataUnit;
	}
	return null;
};


BinaryXMLEncoder.prototype.writeElement = function(
		//long 
		tag, 
		//byte[] 
		Content,
		//TreeMap<String, String> 
		attributes
		) {
	this.writeStartElement(tag, attributes);
	// Will omit if 0-length
	
	if(typeof Content === 'number') {
		if(LOG>4) console.log('GOING TO WRITE THE NUMBER .charCodeAt(0) ' + Content.toString().charCodeAt(0) );
		if(LOG>4) console.log('GOING TO WRITE THE NUMBER ' + Content.toString() );
		if(LOG>4) console.log('type of number is ' + typeof Content.toString() );
		
		this.writeUString(Content.toString());
		//whatever
	}
	else if(typeof Content === 'string'){
		if(LOG>4) console.log('GOING TO WRITE THE STRING  ' + Content );
		if(LOG>4) console.log('type of STRING is ' + typeof Content );
		
		this.writeUString(Content);
	}
	else{
		if(LOG>4) console.log('GOING TO WRITE A BLOB  ' + Content );

		this.writeBlob(Content);
	}
	
	this.writeEndElement();
}



var TypeAndVal = function TypeAndVal(_type,_val) {
	this.type = _type;
	this.val = _val;
	
};


BinaryXMLEncoder.prototype.encodeTypeAndVal = function(
		//int
		type, 
		//long 
		val, 
		//ArrayBufferView 
		ostream
		) {
	
	if(LOG>4) console.log('Encoding type '+ type+ ' and value '+ val);
	
	if(LOG>4) console.log('OFFSET IS ' + this.offset);
	
	if ((type > XML_UDATA) || (type < 0) || (val < 0)) {
		throw new Error("Tag and value must be positive, and tag valid.");
	}
	
	// Encode backwards. Calculate how many bytes we need:
	var numEncodingBytes = this.numEncodingBytes(val);
	
	if ((this.offset + numEncodingBytes) > ostream.length) {
		throw new Error("Buffer space of " + (ostream.length - this.offset) + 
											" bytes insufficient to hold " + 
											numEncodingBytes + " of encoded type and value.");
	}

	// Bottom 4 bits of val go in last byte with tag.
	ostream[this.offset + numEncodingBytes - 1] = 
		//(byte)
			(BYTE_MASK &
					(((XML_TT_MASK & type) | 
					 ((XML_TT_VAL_MASK & val) << XML_TT_BITS))) |
					 XML_TT_NO_MORE); // set top bit for last byte
	val = val >>> XML_TT_VAL_BITS;;
	
	// Rest of val goes into preceding bytes, 7 bits per byte, top bit
	// is "more" flag.
	var i = this.offset + numEncodingBytes - 2;
	while ((0 != val) && (i >= this.offset)) {
		ostream[i] = //(byte)
				(BYTE_MASK & (val & XML_REG_VAL_MASK)); // leave top bit unset
		val = val >>> XML_REG_VAL_BITS;
		--i;
	}
	if (val != 0) {
		throw new Error( "This should not happen: miscalculated encoding");
		//Log.warning(Log.FAC_ENCODING, "This should not happen: miscalculated encoding length, have " + val + " left.");
	}
	this.offset+= numEncodingBytes;
	
	return numEncodingBytes;
};


BinaryXMLEncoder.prototype.encodeUString = function(
		//OutputStream 
		ostream, 
		//String 
		ustring, 
		//byte 
		type) {
	
	if ((null == ustring) || (ustring.length == 0)) {
		return;
	}
	
	if(LOG>3) console.log("The string to write is ");
	if(LOG>3) console.log(ustring);

	//COPY THE STRING TO AVOID PROBLEMS
	strBytes = new Array(ustring.length);

	for (i = 0; i < ustring.length; i++) //in InStr.ToCharArray())
	{
		if(LOG>3) console.log('ustring[' + i + '] = ' + ustring[i]);
		strBytes[i] = ustring.charCodeAt(i);
	}
	
	//strBytes = DataUtils.getBytesFromUTF8String(ustring);
	
	this.encodeTypeAndVal(type, 
						(((type == XML_TAG) || (type == XML_ATTR)) ?
								(strBytes.length-1) :
								strBytes.length), ostream);
	
	if(LOG>3) console.log("THE string to write is ");
	
	if(LOG>3) console.log(strBytes);
	
	this.writeString(strBytes,this.offset);
	this.offset+= strBytes.length;
};



BinaryXMLEncoder.prototype.encodeBlob = function(
		//OutputStream 
		ostream, 
		//Uint8Array 
		blob, 
		//int 
		offset, 
		//int 
		length) {


	if (null == blob)
		return;
	
	if(LOG>4) console.log('LENGTH OF XML_BLOB IS '+length);
	
	/*blobCopy = new Array(blob.Length);
	
	for (i = 0; i < blob.length; i++) //in InStr.ToCharArray())
	{
		blobCopy[i] = blob[i];
	}*/

	this.encodeTypeAndVal(XML_BLOB, length, ostream, offset);

	this.writeBlobArray(blob, this.offset);
	this.offset += length;
};<commit_msg>encodeUString, encodeTypeAndVal and encodeBlob were updated to not take the ostream argument, and to use this.ostream. This closes bug #116.<commit_after>/*
 * This class is used to encode ccnb binary elements (blob, type/value pairs).
 * 
 * @author: ucla-cs
 * See COPYING for copyright and distribution information.
 */

var XML_EXT = 0x00; 
	
var XML_TAG = 0x01; 
	
var XML_DTAG = 0x02; 
	
var XML_ATTR = 0x03; 
 
var XML_DATTR = 0x04; 
	
var XML_BLOB = 0x05; 
	
var XML_UDATA = 0x06; 
	
var XML_CLOSE = 0x0;

var XML_SUBTYPE_PROCESSING_INSTRUCTIONS = 16; 


var XML_TT_BITS = 3;
var XML_TT_MASK = ((1 << XML_TT_BITS) - 1);
var XML_TT_VAL_BITS = XML_TT_BITS + 1;
var XML_TT_VAL_MASK = ((1 << (XML_TT_VAL_BITS)) - 1);
var XML_REG_VAL_BITS = 7;
var XML_REG_VAL_MASK = ((1 << XML_REG_VAL_BITS) - 1);
var XML_TT_NO_MORE = (1 << XML_REG_VAL_BITS); // 0x80
var BYTE_MASK = 0xFF;
var LONG_BYTES = 8;
var LONG_BITS = 64;
	
var bits_11 = 0x0000007FF;
var bits_18 = 0x00003FFFF;
var bits_32 = 0x0FFFFFFFF;


var BinaryXMLEncoder = function BinaryXMLEncoder(){
	this.ostream = new Uint8Array(10000);
	this.offset =0;
	this.CODEC_NAME = "Binary";
};


BinaryXMLEncoder.prototype.writeUString = function(/*String*/ utf8Content) {
	this.encodeUString(utf8Content, XML_UDATA);
};


BinaryXMLEncoder.prototype.writeBlob = function(
		/*Uint8Array*/ binaryContent
		) {
	
	if(LOG >3) console.log(binaryContent);
	
	this.encodeBlob(binaryContent, binaryContent.length);
};


BinaryXMLEncoder.prototype.writeStartElement = function(
	/*String*/ tag, 
	/*TreeMap<String,String>*/ attributes
	) {

	/*Long*/ var dictionaryVal = tag; //stringToTag(tag);
	
	if (null == dictionaryVal) {
		this.encodeUString(tag, XML_TAG);
	} else {
		this.encodeTypeAndVal(XML_DTAG, dictionaryVal);
	}
	
	if (null != attributes) {
		this.writeAttributes(attributes); 
	}
};


BinaryXMLEncoder.prototype.writeEndElement = function() {
	this.ostream[this.offset] = XML_CLOSE;
	this.offset += 1;
}


BinaryXMLEncoder.prototype.writeAttributes = function(/*TreeMap<String,String>*/ attributes) {
	if (null == attributes) {
		return;
	}

	// the keySet of a TreeMap is sorted.

	for(var i=0; i<attributes.length;i++){
		var strAttr = attributes[i].k;
		var strValue = attributes[i].v;

		var dictionaryAttr = stringToTag(strAttr);
		if (null == dictionaryAttr) {
			// not in dictionary, encode as attr
			// compressed format wants length of tag represented as length-1
			// to save that extra bit, as tag cannot be 0 length.
			// encodeUString knows to do that.
			this.encodeUString(strAttr, XML_ATTR);
		} else {
			this.encodeTypeAndVal(XML_DATTR, dictionaryAttr);
		}
		// Write value
		this.encodeUString(strValue);
		
	}
}


//returns a string
stringToTag = function(/*long*/ tagVal) {
	if ((tagVal >= 0) && (tagVal < CCNProtocolDTagsStrings.length)) {
		return CCNProtocolDTagsStrings[tagVal];
	} else if (tagVal == CCNProtocolDTags.CCNProtocolDataUnit) {
		return CCNProtocolDTags.CCNPROTOCOL_DATA_UNIT;
	}
	return null;
};

//returns a Long
tagToString =  function(/*String*/ tagName) {
	// the slow way, but right now we don't care.... want a static lookup for the forward direction
	for (var i=0; i < CCNProtocolDTagsStrings.length; ++i) {
		if ((null != CCNProtocolDTagsStrings[i]) && (CCNProtocolDTagsStrings[i] == tagName)) {
			return i;
		}
	}
	if (CCNProtocolDTags.CCNPROTOCOL_DATA_UNIT == tagName) {
		return CCNProtocolDTags.CCNProtocolDataUnit;
	}
	return null;
};


BinaryXMLEncoder.prototype.writeElement = function(
		//long 
		tag, 
		//byte[] 
		Content,
		//TreeMap<String, String> 
		attributes
		) {
	this.writeStartElement(tag, attributes);
	// Will omit if 0-length
	
	if(typeof Content === 'number') {
		if(LOG>4) console.log('GOING TO WRITE THE NUMBER .charCodeAt(0) ' + Content.toString().charCodeAt(0) );
		if(LOG>4) console.log('GOING TO WRITE THE NUMBER ' + Content.toString() );
		if(LOG>4) console.log('type of number is ' + typeof Content.toString() );
		
		this.writeUString(Content.toString());
		//whatever
	}
	else if(typeof Content === 'string'){
		if(LOG>4) console.log('GOING TO WRITE THE STRING  ' + Content );
		if(LOG>4) console.log('type of STRING is ' + typeof Content );
		
		this.writeUString(Content);
	}
	else{
		if(LOG>4) console.log('GOING TO WRITE A BLOB  ' + Content );

		this.writeBlob(Content);
	}
	
	this.writeEndElement();
}



var TypeAndVal = function TypeAndVal(_type,_val) {
	this.type = _type;
	this.val = _val;
	
};


BinaryXMLEncoder.prototype.encodeTypeAndVal = function(
		//int
		type, 
		//long 
		val
		) {
	
	if(LOG>4) console.log('Encoding type '+ type+ ' and value '+ val);
	
	if(LOG>4) console.log('OFFSET IS ' + this.offset);
	
	if ((type > XML_UDATA) || (type < 0) || (val < 0)) {
		throw new Error("Tag and value must be positive, and tag valid.");
	}
	
	// Encode backwards. Calculate how many bytes we need:
	var numEncodingBytes = this.numEncodingBytes(val);
	
	if ((this.offset + numEncodingBytes) > this.ostream.length) {
		throw new Error("Buffer space of " + (this.ostream.length - this.offset) + 
											" bytes insufficient to hold " + 
											numEncodingBytes + " of encoded type and value.");
	}

	// Bottom 4 bits of val go in last byte with tag.
	this.ostream[this.offset + numEncodingBytes - 1] = 
		//(byte)
			(BYTE_MASK &
					(((XML_TT_MASK & type) | 
					 ((XML_TT_VAL_MASK & val) << XML_TT_BITS))) |
					 XML_TT_NO_MORE); // set top bit for last byte
	val = val >>> XML_TT_VAL_BITS;;
	
	// Rest of val goes into preceding bytes, 7 bits per byte, top bit
	// is "more" flag.
	var i = this.offset + numEncodingBytes - 2;
	while ((0 != val) && (i >= this.offset)) {
		this.ostream[i] = //(byte)
				(BYTE_MASK & (val & XML_REG_VAL_MASK)); // leave top bit unset
		val = val >>> XML_REG_VAL_BITS;
		--i;
	}
	if (val != 0) {
		throw new Error( "This should not happen: miscalculated encoding");
		//Log.warning(Log.FAC_ENCODING, "This should not happen: miscalculated encoding length, have " + val + " left.");
	}
	this.offset+= numEncodingBytes;
	
	return numEncodingBytes;
};


BinaryXMLEncoder.prototype.encodeUString = function(
		//String 
		ustring, 
		//byte 
		type) {
	
	if ((null == ustring) || (ustring.length == 0)) {
		return;
	}
	
	if(LOG>3) console.log("The string to write is ");
	if(LOG>3) console.log(ustring);

	//COPY THE STRING TO AVOID PROBLEMS
	var strBytes = new Array(ustring.length);

	for (i = 0; i < ustring.length; i++) //in InStr.ToCharArray())
	{
		if(LOG>3) console.log('ustring[' + i + '] = ' + ustring[i]);
		strBytes[i] = ustring.charCodeAt(i);
	}
	
	//strBytes = DataUtils.getBytesFromUTF8String(ustring);
	
	this.encodeTypeAndVal(type, 
						(((type == XML_TAG) || (type == XML_ATTR)) ?
								(strBytes.length-1) :
								strBytes.length));
	
	if(LOG>3) console.log("THE string to write is ");
	
	if(LOG>3) console.log(strBytes);
	
	this.writeString(strBytes,this.offset);
	this.offset+= strBytes.length;
};



BinaryXMLEncoder.prototype.encodeBlob = function(
		//Uint8Array 
		blob, 
		//int 
		length) {


	if (null == blob)
		return;
	
	if(LOG>4) console.log('LENGTH OF XML_BLOB IS '+length);
	
	/*blobCopy = new Array(blob.Length);
	
	for (i = 0; i < blob.length; i++) //in InStr.ToCharArray())
	{
		blobCopy[i] = blob[i];
	}*/

	this.encodeTypeAndVal(XML_BLOB, length);

	this.writeBlobArray(blob, this.offset);
	this.offset += length;
};
{'fraction_non_alphanumeric': 0.14246261619291392, 'fraction_numerical': 0.0152903138892631, 'mean_word_length': 3.6498590667084247, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7035172', 'n_tokens_mistral': 5921, 'n_tokens_neox': 5248, 'n_words': 1292}