<commit_before>var util = require('util');
var crypto = require('crypto');
var child_process = require('child_process');
var os = require('os');
var fs = require('fs');
var url = require('url');
require('follow-redirects').maxRedirects = 12;
var http = require('follow-redirects').http;
http.globalAgent.maxSockets = Infinity;
var https = require('follow-redirects').https;
https.globalAgent.maxSockets = Infinity;
var mkdirp = require('mkdirp');
var path = require('path');

var DOWNLOAD_RETRY_INTERVAL = 2;

// add startsWith/endsWith functions to string
if (typeof String.prototype.startsWith != 'function') {
	String.prototype.startsWith = function(str) {
		return this.slice(0, str.length) == str;
	};
}

if (typeof String.prototype.endsWith != 'function') {
	String.prototype.endsWith = function(str) {
		return this.slice(-str.length) == str;
	};
}

if (typeof String.prototype.md5 != 'function') {
	String.prototype.md5 = function() {
		return crypto.createHash('md5').update(new Buffer(this.toString())).digest('hex');
	};
}

if (typeof String.prototype.exec != 'function') {
	String.prototype.exec = function(callback, errorCallback) {
		var cmd = this;
		var childProcess = child_process.exec(cmd, function (error, stdout, stderr) {
			KalturaLogger.log('Command: ' + cmd);
			KalturaLogger.debug('Standard output: ' + stdout);
			
			if(stderr.length){
				KalturaLogger.log('Standard error: ' + stderr);
			}
			
		    if (error) {
		    	if(errorCallback){
		    		errorCallback(error);
		    	}
		    	else{
		    		var exception = new Error();
		    		KalturaLogger.error('Exec: ' + error + '\n' + exception.stack);
		    	}
		    }
		    else if(callback){
		    	callback(stdout);
		    }
		});

		KalturaLogger.debug('Started cli process [' + childProcess.pid + ']: ' + cmd);
	};
}

if (typeof Array.prototype.unique != 'function') {
	Array.prototype.unique = function() {
	    return this.reduce(function(p, c) {
	        if (p.indexOf(c) < 0)
	        	p.push(c);
	        return p;
	    }, []);
	};
}

if (typeof Array.prototype.diff != 'function') {
	Array.prototype.diff = function(arr) {
		var retArr = [];
		
		thisLoop: for (var k = 0; k < this.length; k++) {
		    for (var i = 0; i < arr.length; i++) {
		    	if(this[k] == arr[i]){
		    		continue thisLoop;
		    	}
		    }
		    retArr.push(this[k]);
		}
		return retArr;
	};
}

if (typeof Buffer.prototype.md5 != 'function') {
	Buffer.prototype.md5 = function() {
		return crypto.createHash('md5').update(this).digest('hex');
	};
}

KalturaUtils = {
	getUniqueId : function(){
		return Math.floor(Math.random() * 10000000000000001).toString(36);
	},

	seconds2npt: function( sec, show_ms ) {
		if ( isNaN( sec ) ) {
			return '0:00:00';
		}
	
		var tm = KalturaUtils.seconds2Measurements( sec );
	
		// Round the number of seconds to the required number of significant
		// digits
		if ( show_ms ) {
			tm.seconds = Math.round( tm.seconds * 1000 ) / 1000;
		} else {
			tm.seconds = Math.round( tm.seconds );
		}
		if ( tm.seconds < 10 ){
			tm.seconds = '0' +	tm.seconds;
		}
		var hoursStr;
		if( tm.hours == 0 ){
			hoursStr = '';
		} else {
			if ( tm.minutes < 10 )
				tm.minutes = '0' + tm.minutes;
	
			hoursStr = tm.hours + ":";
		}
		return hoursStr + tm.minutes + ":" + tm.seconds;
	},

	seconds2Measurements: function ( sec ){
		var tm = {};
		tm.days = Math.floor( sec / ( 3600 * 24 ) );
		tm.hours = Math.floor( Math.round( sec ) / 3600 );
		tm.minutes = Math.floor( ( Math.round( sec ) / 60 ) % 60 );
		tm.seconds = Math.round(sec) % 60;
		return tm;
	},
	
	getHttpModuleByProtocol : function(protocol, urlStr){
		if(!protocol){
			var parsedUrl = url.parse(urlStr);
			protocol = parsedUrl.protocol;
		}
		if(protocol == 'https:'){
			return https;
		}
		return http;
	},
	
	downloadMultiHttpUrls : function(urls, filePaths, successCallback, errorCallback) {
		var missingResults = urls.length;

		if(!filePaths){
			filePaths = [];
			for(var i = 0; i < urls.length; i++){
				filePaths[i] = KalturaConfig.config.cloud.sharedBasePath + '/tmp/' + KalturaUtils.getUniqueId();
			}
		}
		
		var singleSuccessCallback = function(){
			missingResults--;
			
			if(!missingResults){
				successCallback(filePaths);
			}
		};
		
		for(var i = 0; i < urls.length; i++){
			KalturaUtils.downloadHttpUrl(urls[i], {filePath: filePaths[i]}, singleSuccessCallback, errorCallback);
		}
	},

	getHttpUrl : function(urlStr, headers, successCallback, errorCallback) {
		var parsedUrl = url.parse(urlStr);
		var httpModule = KalturaUtils.getHttpModuleByProtocol(parsedUrl.protocol);
		var options = {
			hostname : parsedUrl.hostname,
			port : parsedUrl.port,
			path : parsedUrl.path,
			method : 'GET',
		};
		if (headers){
			options.headers = headers;
		}

		var request = httpModule.request(options, function(response) {
			var fullData = '';
			response.on('data', function(data) {
				fullData += data;
			});
			response.on('end', function() {
				if(response.statusCode != 200){
					if(errorCallback){
						return errorCallback('Invalid http status: ' + response.statusCode);
					}
					else{
						KalturaLogger.error('Invalid http status: ' + response.statusCode + ' while trying to fetch ' + urlStr);
						return;
					}
				}
				if(successCallback){
					successCallback(fullData);
				}
			});
		});

		request.on('error', function(e) {
			if(errorCallback){
				errorCallback(e.message);
			}
		});

		request.end();
	},
	
	getHttpUrlByteLimit : function(urlStr, headers, bytes, successCallback, errorCallback) {
		var parsedUrl = url.parse(urlStr);
		var httpModule = KalturaUtils.getHttpModuleByProtocol(parsedUrl.protocol);
		var options = {
			hostname : parsedUrl.hostname,
			port : parsedUrl.port,
			path : parsedUrl.path,
			method : 'GET',
		};
		if (headers){
			options.headers = headers;
		}

		var request = httpModule.request(options, function(response) {
			var dataBuffers = [];
			var length = 0;
			response.on('data', function(data) {
				  length += data.length;
				  dataBuffers.push(data);
				  KalturaLogger.debug('current length=' + length);
			      if (length >= bytes){
			        response.destroy();
			      }
			});
			response.on('end', function() {
				if(response.statusCode != 200){
					if(errorCallback){
						return errorCallback('Invalid http status: ' + response.statusCode);
					}
					else{
						KalturaLogger.error('Invalid http status: ' + response.statusCode + ' while trying to fetch ' + urlStr);
						return;
					}
				}
				if(successCallback){
					KalturaLogger.log('Downloaded [' + length + '] bytes');
					 var bytesBuffer = Buffer.concat(dataBuffers, bytes);
					successCallback(bytesBuffer);
				}
			});
		});

		request.on('error', function(e) {
			if(errorCallback){
				errorCallback(e.message);
			}
		});

		request.end();
	},
	
	buildFilePath: function(baseDir, fileName){
		
		if(!fileName)
			return;
		var dir1 = fileName.substring(0,2);	
		var dir2 = fileName.substring(2,4);
		return path.join(KalturaConfig.config.cloud.sharedBasePath, baseDir, dir1, dir2, fileName);
	},
	
	createFilePath : function(filePath, successCallback, errorCallback) {
		var dirName= path.dirname(filePath);
		mkdirp(dirName, function (err) {
		    if (err){<commit_msg>#PLAT-4203 concat full buffers lenght to avoid targetStart out of bounds exception<commit_after>var util = require('util');
var crypto = require('crypto');
var child_process = require('child_process');
var os = require('os');
var fs = require('fs');
var url = require('url');
require('follow-redirects').maxRedirects = 12;
var http = require('follow-redirects').http;
http.globalAgent.maxSockets = Infinity;
var https = require('follow-redirects').https;
https.globalAgent.maxSockets = Infinity;
var mkdirp = require('mkdirp');
var path = require('path');

var DOWNLOAD_RETRY_INTERVAL = 2;

// add startsWith/endsWith functions to string
if (typeof String.prototype.startsWith != 'function') {
	String.prototype.startsWith = function(str) {
		return this.slice(0, str.length) == str;
	};
}

if (typeof String.prototype.endsWith != 'function') {
	String.prototype.endsWith = function(str) {
		return this.slice(-str.length) == str;
	};
}

if (typeof String.prototype.md5 != 'function') {
	String.prototype.md5 = function() {
		return crypto.createHash('md5').update(new Buffer(this.toString())).digest('hex');
	};
}

if (typeof String.prototype.exec != 'function') {
	String.prototype.exec = function(callback, errorCallback) {
		var cmd = this;
		var childProcess = child_process.exec(cmd, function (error, stdout, stderr) {
			KalturaLogger.log('Command: ' + cmd);
			KalturaLogger.debug('Standard output: ' + stdout);
			
			if(stderr.length){
				KalturaLogger.log('Standard error: ' + stderr);
			}
			
		    if (error) {
		    	if(errorCallback){
		    		errorCallback(error);
		    	}
		    	else{
		    		var exception = new Error();
		    		KalturaLogger.error('Exec: ' + error + '\n' + exception.stack);
		    	}
		    }
		    else if(callback){
		    	callback(stdout);
		    }
		});

		KalturaLogger.debug('Started cli process [' + childProcess.pid + ']: ' + cmd);
	};
}

if (typeof Array.prototype.unique != 'function') {
	Array.prototype.unique = function() {
	    return this.reduce(function(p, c) {
	        if (p.indexOf(c) < 0)
	        	p.push(c);
	        return p;
	    }, []);
	};
}

if (typeof Array.prototype.diff != 'function') {
	Array.prototype.diff = function(arr) {
		var retArr = [];
		
		thisLoop: for (var k = 0; k < this.length; k++) {
		    for (var i = 0; i < arr.length; i++) {
		    	if(this[k] == arr[i]){
		    		continue thisLoop;
		    	}
		    }
		    retArr.push(this[k]);
		}
		return retArr;
	};
}

if (typeof Buffer.prototype.md5 != 'function') {
	Buffer.prototype.md5 = function() {
		return crypto.createHash('md5').update(this).digest('hex');
	};
}

KalturaUtils = {
	getUniqueId : function(){
		return Math.floor(Math.random() * 10000000000000001).toString(36);
	},

	seconds2npt: function( sec, show_ms ) {
		if ( isNaN( sec ) ) {
			return '0:00:00';
		}
	
		var tm = KalturaUtils.seconds2Measurements( sec );
	
		// Round the number of seconds to the required number of significant
		// digits
		if ( show_ms ) {
			tm.seconds = Math.round( tm.seconds * 1000 ) / 1000;
		} else {
			tm.seconds = Math.round( tm.seconds );
		}
		if ( tm.seconds < 10 ){
			tm.seconds = '0' +	tm.seconds;
		}
		var hoursStr;
		if( tm.hours == 0 ){
			hoursStr = '';
		} else {
			if ( tm.minutes < 10 )
				tm.minutes = '0' + tm.minutes;
	
			hoursStr = tm.hours + ":";
		}
		return hoursStr + tm.minutes + ":" + tm.seconds;
	},

	seconds2Measurements: function ( sec ){
		var tm = {};
		tm.days = Math.floor( sec / ( 3600 * 24 ) );
		tm.hours = Math.floor( Math.round( sec ) / 3600 );
		tm.minutes = Math.floor( ( Math.round( sec ) / 60 ) % 60 );
		tm.seconds = Math.round(sec) % 60;
		return tm;
	},
	
	getHttpModuleByProtocol : function(protocol, urlStr){
		if(!protocol){
			var parsedUrl = url.parse(urlStr);
			protocol = parsedUrl.protocol;
		}
		if(protocol == 'https:'){
			return https;
		}
		return http;
	},
	
	downloadMultiHttpUrls : function(urls, filePaths, successCallback, errorCallback) {
		var missingResults = urls.length;

		if(!filePaths){
			filePaths = [];
			for(var i = 0; i < urls.length; i++){
				filePaths[i] = KalturaConfig.config.cloud.sharedBasePath + '/tmp/' + KalturaUtils.getUniqueId();
			}
		}
		
		var singleSuccessCallback = function(){
			missingResults--;
			
			if(!missingResults){
				successCallback(filePaths);
			}
		};
		
		for(var i = 0; i < urls.length; i++){
			KalturaUtils.downloadHttpUrl(urls[i], {filePath: filePaths[i]}, singleSuccessCallback, errorCallback);
		}
	},

	getHttpUrl : function(urlStr, headers, successCallback, errorCallback) {
		var parsedUrl = url.parse(urlStr);
		var httpModule = KalturaUtils.getHttpModuleByProtocol(parsedUrl.protocol);
		var options = {
			hostname : parsedUrl.hostname,
			port : parsedUrl.port,
			path : parsedUrl.path,
			method : 'GET',
		};
		if (headers){
			options.headers = headers;
		}

		var request = httpModule.request(options, function(response) {
			var fullData = '';
			response.on('data', function(data) {
				fullData += data;
			});
			response.on('end', function() {
				if(response.statusCode != 200){
					if(errorCallback){
						return errorCallback('Invalid http status: ' + response.statusCode);
					}
					else{
						KalturaLogger.error('Invalid http status: ' + response.statusCode + ' while trying to fetch ' + urlStr);
						return;
					}
				}
				if(successCallback){
					successCallback(fullData);
				}
			});
		});

		request.on('error', function(e) {
			if(errorCallback){
				errorCallback(e.message);
			}
		});

		request.end();
	},
	
	getHttpUrlByteLimit : function(urlStr, headers, bytes, successCallback, errorCallback) {
		var parsedUrl = url.parse(urlStr);
		var httpModule = KalturaUtils.getHttpModuleByProtocol(parsedUrl.protocol);
		var options = {
			hostname : parsedUrl.hostname,
			port : parsedUrl.port,
			path : parsedUrl.path,
			method : 'GET',
		};
		if (headers){
			options.headers = headers;
		}

		var request = httpModule.request(options, function(response) {
			var dataBuffers = [];
			var length = 0;
			response.on('data', function(data) {
				  length += data.length;
				  dataBuffers.push(data);
				  KalturaLogger.debug('current length=' + length);
			      if (length >= bytes){
			        response.destroy();
			      }
			});
			response.on('end', function() {
				if(response.statusCode != 200){
					if(errorCallback){
						return errorCallback('Invalid http status: ' + response.statusCode);
					}
					else{
						KalturaLogger.error('Invalid http status: ' + response.statusCode + ' while trying to fetch ' + urlStr);
						return;
					}
				}
				if(successCallback){
					KalturaLogger.log('Downloaded [' + length + '] bytes');
					 var bytesBuffer = Buffer.concat(dataBuffers, length);
					successCallback(bytesBuffer.slice(0, bytes));
				}
			});
		});

		request.on('error', function(e) {
			if(errorCallback){
				errorCallback(e.message);
			}
		});

		request.end();
	},
	
	buildFilePath: function(baseDir, fileName){
		
		if(!fileName)
			return;
		var dir1 = fileName.substring(0,2);	
		var dir2 = fileName.substring(2,4);
		return path.join(KalturaConfig.config.cloud.sharedBasePath, baseDir, dir1, dir2, fileName);
	},
	
	createFilePath : function(filePath, successCallback, errorCallback) {
		var dirName= path.dirname(filePath);
		mkdirp(dirName, function (err) {
		    if (err){
{'fraction_non_alphanumeric': 0.15380382446003577, 'fraction_numerical': 0.012450130691979639, 'mean_word_length': 3.230142566191446, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2100752', 'n_tokens_mistral': 5697, 'n_tokens_neox': 4791, 'n_words': 993}