<commit_before>
/*
 * This class contains utilities to help parse the data
 * author: ucla-cs
 */
 
var DataUtils = function DataUtils(){
	
	
};


/*
 * NOTE THIS IS CURRENTLY NOT BEHING USED
 * 
 */

DataUtils.keyStr = "<KEY>" +
               "<KEY>" +
               "<KEY>" +
               "<KEY> +
               "=";

               
/**
 * Raw String to Base 64
 */
DataUtils.stringtoBase64=function stringtoBase64(input) {
     input = escape(input);
     var output = "";
     var chr1, chr2, chr3 = "";
     var enc1, enc2, enc3, enc4 = "";
     var i = 0;

     do {
        chr1 = input.charCodeAt(i++);
        chr2 = input.charCodeAt(i++);
        chr3 = input.charCodeAt(i++);

        enc1 = chr1 >> 2;
        enc2 = ((chr1 & 3) << 4) | (chr2 >> 4);
        enc3 = ((chr2 & 15) << 2) | (chr3 >> 6);
        enc4 = chr3 & 63;

        if (isNaN(chr2)) {
           enc3 = enc4 = 64;
        } else if (isNaN(chr3)) {
           enc4 = 64;
        }

        output = output +
           DataUtils.keyStr.charAt(enc1) +
           DataUtils.keyStr.charAt(enc2) +
           DataUtils.keyStr.charAt(enc3) +
           DataUtils.keyStr.charAt(enc4);
        chr1 = chr2 = chr3 = "";
        enc1 = enc2 = enc3 = enc4 = "";
     } while (i < input.length);

     return output;
  }

/**
 * Base 64 to Raw String 
 */
DataUtils.base64toString = function base64toString(input) {
     var output = "";
     var chr1, chr2, chr3 = "";
     var enc1, enc2, enc3, enc4 = "";
     var i = 0;

     // remove all characters that are not A-Z, a-z, 0-9, +, /, or =
     var base64test = /[^A-Za-z0-9\+\/\=]/g;
     if (base64test.exec(input)) {
        alert("There were invalid base64 characters in the input text.\n" +
              "Valid base64 characters are A-Z, a-z, 0-9, '+', '/',and '='\n" +
              "Expect errors in decoding.");
     }
     
     input = input.replace(/[^A-Za-z0-9\+\/\=]/g, "");

     do {
        enc1 = DataUtils.keyStr.indexOf(input.charAt(i++));
        enc2 = DataUtils.keyStr.indexOf(input.charAt(i++));
        enc3 = DataUtils.keyStr.indexOf(input.charAt(i++));
        enc4 = DataUtils.keyStr.indexOf(input.charAt(i++));

        chr1 = (enc1 << 2) | (enc2 >> 4);
        chr2 = ((enc2 & 15) << 4) | (enc3 >> 2);
        chr3 = ((enc3 & 3) << 6) | enc4;

        output = output + String.fromCharCode(chr1);

        if (enc3 != 64) {
           output = output + String.fromCharCode(chr2);
        }
        if (enc4 != 64) {
           output = output + String.fromCharCode(chr3);
        }

        chr1 = chr2 = chr3 = "";
        enc1 = enc2 = enc3 = enc4 = "";

     } while (i < input.length);

     return unescape(output);
  };

//byte [] 

/**
 * NOT WORKING!!!!!
 * 
 * Unsiged Long Number to Byte Array
 */
	
 /*
DataUtils.unsignedLongToByteArray= function( value) {
	
	if(LOG>4)console.log('INPUT IS '+value);
	
	if( 0 == value )
		return [0];

	if( 0 <= value && value <= 0x00FF ) {
		//byte [] 
		var bb = new Array(1);
		bb[0] = (value & 0x00FF);
		return bb;
	}

	if(LOG>4) console.log('type of value is '+typeof value);
	if(LOG>4) console.log('value is '+value);
	//byte [] 
	var out = null;
	//int
	var offset = -1;
	for(var i = 7; i >=0; --i) {
		//byte
		console.log(i);
		console.log('value is '+value);
		console.log('(value >> (i * 8)) '+ (value >> (i * 8))  );
		console.log(' ((value >> (i * 8)) & 0xFF) '+ ((value >> (i * 8)) & 0xFF)  );

		var b = ((value >> (i * 8)) & 0xFF)  ;
		
		if(LOG>4) console.log('b is '+b);
		
		if( out == null && b != 0 ) {
			//out = new byte[i+1];
			out = new Array(i+1);
			offset = i;
		}
		
		if( out != null )
			out[ offset - i ] = b;
	}
	if(LOG>4)console.log('OUTPUT IS ');
	if(LOG>4)console.log(out);
	return out;
}
*/
	
/**
 * NOT WORKING!!!!!
 * 
 * Unsiged Long Number to Byte Array
 *//*
DataUtils.byteArrayToUnsignedLong = function(//final byte [] 
	src) {
		if(LOG>4) console.log('INPUT IS ');
		if(LOG>4) console.log(src);
		
		var value = 0;
		for(var i = 0; i < src.length; i++) {
			value = value << 8;
			// Java will assume the byte is signed, so extend it and trim it.
			
			
			var b = ((src[i]) & 0xFF );
			value |= b;
		}
		
		if(LOG>4) console.log('OUTPUT IS ');
		
		if(LOG>4) console.log(value);

		return value;
	}*/


/**
 * Hex String to Byte Array
 */
	//THIS IS NOT WORKING
/*
DataUtils.HexStringtoByteArray = function(str) {
    var byteArray = [];
    for (var i = 0; i < str.length; i++)
        if (str.charCodeAt(i) <= 0x7F)
            byteArray.push(str.charCodeAt(i));
        else {
            var h = encodeURIComponent(str.charAt(i)).substr(1).split('%');
            for (var j = 0; j < h.length; j++)
                byteArray.push(parseInt(h[j], 16));
        }
    return byteArray;
};
*/
	
/**
 * Byte Array to Hex String
 */
DataUtils.byteArrayToHexString = function(byteArray) {
    var str = '';
    for (var i = 0; i < byteArray.length; i++)
        str +=  byteArray[i] <= 0x7F?
                byteArray[i] === 0x25 ? "%25" : // %
                String.fromCharCode(byteArray[i]) :
                "%" + byteArray[i].toString(16).toUpperCase();
    return decodeURIComponent(str);
};


/**
 * Byte array to Hex String
 */
//http://ejohn.org/blog/numbers-hex-and-colors/
DataUtils.toHex = function(arguments){
	if(LOG>4) console.log('ABOUT TO CONVERT '+ arguments);
  //console.log(arguments);
  var ret = "";
  for ( var i = 0; i < arguments.length; i++ )
    ret += (arguments[i] < 16 ? "0" : "") + arguments[i].toString(16);
  return ret; //.toUpperCase();
}

/**
 * Byte array to raw string
 */
//DOES NOT SEEM TO WORK<commit_msg>Added stringToHex<commit_after>
/*
 * This class contains utilities to help parse the data
 * author: ucla-cs
 */
 
var DataUtils = function DataUtils(){
	
	
};


/*
 * NOTE THIS IS CURRENTLY NOT BEHING USED
 * 
 */

DataUtils.keyStr = "<KEY>" +
               "<KEY>" +
               "<KEY>" +
               "wxyz0123456789+/" +
               "=";

               
/**
 * Raw String to Base 64
 */
DataUtils.stringtoBase64=function stringtoBase64(input) {
     input = escape(input);
     var output = "";
     var chr1, chr2, chr3 = "";
     var enc1, enc2, enc3, enc4 = "";
     var i = 0;

     do {
        chr1 = input.charCodeAt(i++);
        chr2 = input.charCodeAt(i++);
        chr3 = input.charCodeAt(i++);

        enc1 = chr1 >> 2;
        enc2 = ((chr1 & 3) << 4) | (chr2 >> 4);
        enc3 = ((chr2 & 15) << 2) | (chr3 >> 6);
        enc4 = chr3 & 63;

        if (isNaN(chr2)) {
           enc3 = enc4 = 64;
        } else if (isNaN(chr3)) {
           enc4 = 64;
        }

        output = output +
           DataUtils.keyStr.charAt(enc1) +
           DataUtils.keyStr.charAt(enc2) +
           DataUtils.keyStr.charAt(enc3) +
           DataUtils.keyStr.charAt(enc4);
        chr1 = chr2 = chr3 = "";
        enc1 = enc2 = enc3 = enc4 = "";
     } while (i < input.length);

     return output;
  }

/**
 * Base 64 to Raw String 
 */
DataUtils.base64toString = function base64toString(input) {
     var output = "";
     var chr1, chr2, chr3 = "";
     var enc1, enc2, enc3, enc4 = "";
     var i = 0;

     // remove all characters that are not A-Z, a-z, 0-9, +, /, or =
     var base64test = /[^A-Za-z0-9\+\/\=]/g;
     if (base64test.exec(input)) {
        alert("There were invalid base64 characters in the input text.\n" +
              "Valid base64 characters are A-Z, a-z, 0-9, '+', '/',and '='\n" +
              "Expect errors in decoding.");
     }
     
     input = input.replace(/[^A-Za-z0-9\+\/\=]/g, "");

     do {
        enc1 = DataUtils.keyStr.indexOf(input.charAt(i++));
        enc2 = DataUtils.keyStr.indexOf(input.charAt(i++));
        enc3 = DataUtils.keyStr.indexOf(input.charAt(i++));
        enc4 = DataUtils.keyStr.indexOf(input.charAt(i++));

        chr1 = (enc1 << 2) | (enc2 >> 4);
        chr2 = ((enc2 & 15) << 4) | (enc3 >> 2);
        chr3 = ((enc3 & 3) << 6) | enc4;

        output = output + String.fromCharCode(chr1);

        if (enc3 != 64) {
           output = output + String.fromCharCode(chr2);
        }
        if (enc4 != 64) {
           output = output + String.fromCharCode(chr3);
        }

        chr1 = chr2 = chr3 = "";
        enc1 = enc2 = enc3 = enc4 = "";

     } while (i < input.length);

     return unescape(output);
  };

//byte [] 

/**
 * NOT WORKING!!!!!
 * 
 * Unsiged Long Number to Byte Array
 */
	
 /*
DataUtils.unsignedLongToByteArray= function( value) {
	
	if(LOG>4)console.log('INPUT IS '+value);
	
	if( 0 == value )
		return [0];

	if( 0 <= value && value <= 0x00FF ) {
		//byte [] 
		var bb = new Array(1);
		bb[0] = (value & 0x00FF);
		return bb;
	}

	if(LOG>4) console.log('type of value is '+typeof value);
	if(LOG>4) console.log('value is '+value);
	//byte [] 
	var out = null;
	//int
	var offset = -1;
	for(var i = 7; i >=0; --i) {
		//byte
		console.log(i);
		console.log('value is '+value);
		console.log('(value >> (i * 8)) '+ (value >> (i * 8))  );
		console.log(' ((value >> (i * 8)) & 0xFF) '+ ((value >> (i * 8)) & 0xFF)  );

		var b = ((value >> (i * 8)) & 0xFF)  ;
		
		if(LOG>4) console.log('b is '+b);
		
		if( out == null && b != 0 ) {
			//out = new byte[i+1];
			out = new Array(i+1);
			offset = i;
		}
		
		if( out != null )
			out[ offset - i ] = b;
	}
	if(LOG>4)console.log('OUTPUT IS ');
	if(LOG>4)console.log(out);
	return out;
}
*/
	
/**
 * NOT WORKING!!!!!
 * 
 * Unsiged Long Number to Byte Array
 *//*
DataUtils.byteArrayToUnsignedLong = function(//final byte [] 
	src) {
		if(LOG>4) console.log('INPUT IS ');
		if(LOG>4) console.log(src);
		
		var value = 0;
		for(var i = 0; i < src.length; i++) {
			value = value << 8;
			// Java will assume the byte is signed, so extend it and trim it.
			
			
			var b = ((src[i]) & 0xFF );
			value |= b;
		}
		
		if(LOG>4) console.log('OUTPUT IS ');
		
		if(LOG>4) console.log(value);

		return value;
	}*/


/**
 * Hex String to Byte Array
 */
	//THIS IS NOT WORKING
/*
DataUtils.HexStringtoByteArray = function(str) {
    var byteArray = [];
    for (var i = 0; i < str.length; i++)
        if (str.charCodeAt(i) <= 0x7F)
            byteArray.push(str.charCodeAt(i));
        else {
            var h = encodeURIComponent(str.charAt(i)).substr(1).split('%');
            for (var j = 0; j < h.length; j++)
                byteArray.push(parseInt(h[j], 16));
        }
    return byteArray;
};
*/
	
/**
 * Byte Array to Hex String
 */
DataUtils.byteArrayToHexString = function(byteArray) {
    var str = '';
    for (var i = 0; i < byteArray.length; i++)
        str +=  byteArray[i] <= 0x7F?
                byteArray[i] === 0x25 ? "%25" : // %
                String.fromCharCode(byteArray[i]) :
                "%" + byteArray[i].toString(16).toUpperCase();
    return decodeURIComponent(str);
};


/**
 * Byte array to Hex String
 */
//http://ejohn.org/blog/numbers-hex-and-colors/
DataUtils.toHex = function(arguments){
	if(LOG>4) console.log('ABOUT TO CONVERT '+ arguments);
  //console.log(arguments);
  var ret = "";
  for ( var i = 0; i < arguments.length; i++ )
    ret += (arguments[i] < 16 ? "0" : "") + arguments[i].toString(16);
  return ret; //.toUpperCase();
}

/**
 * Raw string to hex string.
 */
DataUtils.stringToHex = function(arguments){
	var ret = "";
	for (var i = 0; i < arguments.length; ++i) {
		var value = arguments.charCodeAt(i);
		ret += (value < 16 ? "0" : "") + value.toString(16);
	}
	return ret;
}

/**
 * Byte array to raw string
 */
//DOES NOT SEEM TO WORK
{'fraction_non_alphanumeric': 0.1841095412523984, 'fraction_numerical': 0.034013605442176874, 'mean_word_length': 2.066862797539449, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3361067', 'n_tokens_mistral': 4557, 'n_tokens_neox': 4135, 'n_words': 1012}