<commit_before>
var fs = require('fs');
var os = require('os');
var url = require('url');
var util = require('util');
var http = require('follow-redirects').http;

var id3Reader = require('../../bin/TsId3Reader.node');

var kaltura = module.exports = require('../KalturaManager');
kaltura.mediaInfo = require('../media/KalturaMediaInfo');
kaltura.ffmpegParams = require('../media/KalturaFfmpegParams');


/**
 * Stream watcher, instantiated per rendition manifest
 * 
 * @param manager KalturaStreamManager
 * @param params {url, entryId, masterUrl}
 * @param finishCallback called when watching is not required anymore
 */
var KalturaStreamWatcher = function(manager, params, finishCallback){
	KalturaLogger.log('init stream watcher for entry [' + params.entryId +']');
	this.manager = manager;
	this.finishCallback = finishCallback;

	this.renditionId = KalturaCache.getManifestId(params.url);
	this.trackerOutputKeyWithAds = KalturaCache.getManifestContent(this.renditionId);
	this.entryRequiredKey = KalturaCache.getEntryRequired(params.entryId);
	this.cuePointsKey = KalturaCache.getCuePoints(params.entryId);
	this.elapsedTimeKey = KalturaCache.getElapsedTime(params.entryId);
	this.oldestSegmentTimeKey = KalturaCache.getOldestSegmentTime(params.entryId);
	this.elapsedTime = {};
	this.cuePoints = {};
	this.segmentsHistoryLimit = KalturaConfig.config.cloud.segmentsHistoryLimit;
	this.segmentsHistory = {};
	this.urlTranslations = {};

	this.firstTime = true;
	this.startTime = new Date().getTime();
	this.trackerRequired = false;

	this.renditionManifestHandledKey = KalturaCache.getRenditionManifestHandled(params.url);
	
	this.url = params.url;
	this.lowestBitrate = params.lowestBitrate;
	this.entryId = params.entryId;
	this.partnerId = params.partnerId;
	this.masterUrl = params.masterUrl;
	
	var This = this;
	// if the process crashed and restored, the cache key might exist although no one is watching this manifest
	if(params.restored){
		this.getManifest();
	}
	else{
		KalturaCache.get(this.renditionManifestHandledKey, function(data){
			if(!data){
				This.getManifest();
			}
		});
	}
};

KalturaStreamWatcher.MINIMUM_RUN_PERIOD = 60000;
KalturaStreamWatcher.CYCLE_INTERVAL = 2000;
	
KalturaStreamWatcher.prototype = {

	/**
	 * @type KalturaStreamManager
	 */
	manager: null,
	
	/**
	 * @type function
	 */
	finishCallback: null,

	/**
	 * Check cache to see if entry still required
	 */
	verifyTrackingRequired: function(){
		var This = this;
		KalturaCache.get(this.entryRequiredKey, function(data){
			if(data){
				KalturaLogger.debug('Entry [' + This.entryId + '] still required [' + This.entryRequiredKey + ']');
				This.trackerRequired = true;
				var encodingIds = data.split('\n').unique();
				for(var i = 0; i < encodingIds.length; i++){ 
					if(encodingIds[i].trim().length){
						var encodingParamsKey = KalturaCache.getEncodingParams(encodingIds[i]);
						var mediaInfoKey = KalturaCache.getMediaInfo(encodingIds[i]);
						var blackEncodingParamsKey = KalturaCache.getBlackEncodingParams(encodingIds[i]);
						KalturaCache.touch(mediaInfoKey, KalturaConfig.config.cache.encodingParams);
						KalturaCache.touch(encodingParamsKey, KalturaConfig.config.cache.encodingParams);
						KalturaCache.touch(blackEncodingParamsKey, KalturaConfig.config.cache.blackMedia);
					}
				}				
			}
			else{
				KalturaLogger.log('Entry [' + This.entryId + '] not required any more [' + This.entryRequiredKey + ']');
				This.trackerRequired = false;
			}
		});
	},
	
	/**
	 * Load cue-points from cache
	 */
	loadCuePoints: function(callback){
		var This = this;
		KalturaCache.get(this.cuePointsKey, function(data){
			if(data){
				This.cuePoints = data;				
			}
			if(callback){
				callback();
			}
		}, function(err){
			if(callback){
				callback();
			}
		});
	},
	
	applyElapsedTimeToSegments: function(){
		if(this.elapsedTime && this.segmentsHistory[this.elapsedTime.sequence]){
			var timestamp;
			var offset;
			
			timestamp = this.elapsedTime.timestamp;
			offset = this.elapsedTime.offset;
			for (var i = this.elapsedTime.sequence; this.segmentsHistory[i]; i--){
				this.segmentsHistory[i].timestamp = timestamp;
				this.segmentsHistory[i].offset = offset;
				timestamp -= this.segmentsHistory[i].duration;
				offset -= this.segmentsHistory[i].duration;
			}

			timestamp = this.elapsedTime.timestamp;
			offset = this.elapsedTime.offset;
			for (var i = this.elapsedTime.sequence; this.segmentsHistory[i]; i++){
				this.segmentsHistory[i].timestamp = timestamp;
				this.segmentsHistory[i].offset = offset;
				this.elapsedTime = {
						sequence: i,<commit_msg>ads not getting stitched from time to time or stitched a bit early<commit_after>
var fs = require('fs');
var os = require('os');
var url = require('url');
var util = require('util');
var http = require('follow-redirects').http;

var id3Reader = require('../../bin/TsId3Reader.node');

var kaltura = module.exports = require('../KalturaManager');
kaltura.mediaInfo = require('../media/KalturaMediaInfo');
kaltura.ffmpegParams = require('../media/KalturaFfmpegParams');


/**
 * Stream watcher, instantiated per rendition manifest
 * 
 * @param manager KalturaStreamManager
 * @param params {url, entryId, masterUrl}
 * @param finishCallback called when watching is not required anymore
 */
var KalturaStreamWatcher = function(manager, params, finishCallback){
	KalturaLogger.log('init stream watcher for entry [' + params.entryId +']');
	this.manager = manager;
	this.finishCallback = finishCallback;

	this.renditionId = KalturaCache.getManifestId(params.url);
	this.trackerOutputKeyWithAds = KalturaCache.getManifestContent(this.renditionId);
	this.entryRequiredKey = KalturaCache.getEntryRequired(params.entryId);
	this.cuePointsKey = KalturaCache.getCuePoints(params.entryId);
	this.elapsedTimeKey = KalturaCache.getElapsedTime(params.entryId);
	this.oldestSegmentTimeKey = KalturaCache.getOldestSegmentTime(params.entryId);
	this.elapsedTime = {};
	this.cuePoints = {};
	this.segmentsHistoryLimit = KalturaConfig.config.cloud.segmentsHistoryLimit;
	this.segmentsHistory = {};
	this.urlTranslations = {};

	this.firstTime = true;
	this.startTime = new Date().getTime();
	this.trackerRequired = false;

	this.renditionManifestHandledKey = KalturaCache.getRenditionManifestHandled(params.url);
	
	this.url = params.url;
	this.lowestBitrate = params.lowestBitrate;
	this.entryId = params.entryId;
	this.partnerId = params.partnerId;
	this.masterUrl = params.masterUrl;
	
	var This = this;
	// if the process crashed and restored, the cache key might exist although no one is watching this manifest
	if(params.restored){
		this.getManifest();
	}
	else{
		KalturaCache.get(this.renditionManifestHandledKey, function(data){
			if(!data){
				This.getManifest();
			}
		});
	}
};

KalturaStreamWatcher.MINIMUM_RUN_PERIOD = 60000;
KalturaStreamWatcher.CYCLE_INTERVAL = 2000;
	
KalturaStreamWatcher.prototype = {

	/**
	 * @type KalturaStreamManager
	 */
	manager: null,
	
	/**
	 * @type function
	 */
	finishCallback: null,

	/**
	 * Check cache to see if entry still required
	 */
	verifyTrackingRequired: function(){
		var This = this;
		KalturaCache.get(this.entryRequiredKey, function(data){
			if(data){
				KalturaLogger.debug('Entry [' + This.entryId + '] still required [' + This.entryRequiredKey + ']');
				This.trackerRequired = true;
				var encodingIds = data.split('\n').unique();
				for(var i = 0; i < encodingIds.length; i++){ 
					if(encodingIds[i].trim().length){
						var encodingParamsKey = KalturaCache.getEncodingParams(encodingIds[i]);
						var mediaInfoKey = KalturaCache.getMediaInfo(encodingIds[i]);
						var blackEncodingParamsKey = KalturaCache.getBlackEncodingParams(encodingIds[i]);
						KalturaCache.touch(mediaInfoKey, KalturaConfig.config.cache.encodingParams);
						KalturaCache.touch(encodingParamsKey, KalturaConfig.config.cache.encodingParams);
						KalturaCache.touch(blackEncodingParamsKey, KalturaConfig.config.cache.blackMedia);
					}
				}				
			}
			else{
				KalturaLogger.log('Entry [' + This.entryId + '] not required any more [' + This.entryRequiredKey + ']');
				This.trackerRequired = false;
			}
		});
	},
	
	/**
	 * Load cue-points from cache
	 */
	loadCuePoints: function(callback){
		var This = this;
		KalturaCache.get(this.cuePointsKey, function(data){
			if(data){
				This.cuePoints = data;				
			}
			if(callback){
				callback();
			}
		}, function(err){
			if(callback){
				callback();
			}
		});
	},
	
	applyElapsedTimeToSegments: function(){
		if(this.elapsedTime && this.segmentsHistory[this.elapsedTime.sequence]){
			var timestamp;
			var offset;
			
			timestamp = this.elapsedTime.timestamp;
			offset = this.elapsedTime.offset;
			for (var i = this.elapsedTime.sequence; this.segmentsHistory[i]; i--){
				this.segmentsHistory[i].timestamp = timestamp;
				this.segmentsHistory[i].offset = offset;
				if(this.segmentsHistory[i-1]){
					timestamp -= this.segmentsHistory[i-1].duration;
					offset -= this.segmentsHistory[i-1].duration;
				}
				else{
					KalturaLogger.debug("Segment history for index [" + i-1 + "] is not defined setting offset/timestamp from current duration values");
					timestamp -= this.segmentsHistory[i].duration;
					offset -= this.segmentsHistory[i].duration;	
				}
			}

			timestamp = this.elapsedTime.timestamp;
			offset = this.elapsedTime.offset;
			for (var i = this.elapsedTime.sequence; this.segmentsHistory[i]; i++){
				this.segmentsHistory[i].timestamp = timestamp;
				this.segmentsHistory[i].offset = offset;
				this.elapsedTime = {
						sequence: i,
{'fraction_non_alphanumeric': 0.1302897644361059, 'fraction_numerical': 0.002918490723368772, 'mean_word_length': 4.330555555555556, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1541401', 'n_tokens_mistral': 3470, 'n_tokens_neox': 3057, 'n_words': 557}