<commit_before>/**
 * Copyright (c) 2015, <NAME>, Quintelligence d.o.o. and contributors
 * All rights reserved.
 * 
 * This source code is licensed under the FreeBSD license found in the
 * LICENSE file in the root directory of this source tree.
 */

namespace TSignalProc {

/////////////////////////////////////////////////
// Simple Online Moving Variance
void TVarSimple::Update(const double& InVal) {	 
	// See Knuth TAOCP vol 2, 3rd edition, page 232
	N++;
	if (N == 1) {
		OldM = NewM = InVal;
		OldS = 0.0;
	} else {
		NewM = OldM + (InVal - OldM) / N;
		NewS = OldS + (InVal - OldM)*(InVal - NewM);
		// set up for next iteration
		OldM = NewM;
		OldS = NewS;
	}
}

void TVarSimple::Load(TSIn& SIn) {
	*this = TVarSimple(SIn);
}

void TVarSimple::Save(TSOut& SOut) const {
	OldM.Save(SOut);
	NewM.Save(SOut);
	OldS.Save(SOut);
	NewS.Save(SOut);
	N.Save(SOut);
}
    
/////////////////////////////////////////////////
// Online Moving Average 
void TMa::Update(const double& InVal, const uint64& InTmMSecs, 
        const TFltV& OutValV, const TUInt64V& OutTmMSecsV, const int& N){
    
    int tempN = N - 1 + OutValV.Len();
    double delta;    
	// check if all the values are removed
	if (OutValV.Len() >= tempN) {
		Ma = 0;		
		tempN = 0;
	}
	else {
		// remove old values from the mean
		for (int ValN = 0; ValN < OutValV.Len(); ValN++)
		{
			tempN--;
			delta = OutValV[ValN] - Ma;
			Ma = Ma - delta / tempN;
		}
	}
    //add the new value to the resulting mean    
    delta = InVal - Ma;
    Ma = Ma + delta/N;
    TmMSecs = InTmMSecs;
}

TMa::TMa(TSIn& SIn) : Ma(SIn), TmMSecs(SIn) { }

void TMa::Load(TSIn& SIn) {
	*this = TMa(SIn);
}

void TMa::Save(TSOut& SOut) const {
	// parameters
	Ma.Save(SOut);
	TmMSecs.Save(SOut);
}

/////////////////////////////////////////////////
// Online Summa 
void TSum::Update(const double& InVal, const uint64& InTmMSecs,
	const TFltV& OutValV, const TUInt64V& OutTmMSecsV){

	// remove old values from the sum
	for (int ValN = 0; ValN < OutValV.Len(); ValN++)
	{
		Sum -= OutValV[ValN];
	}
	//add the new value to the resulting sum    
	Sum += InVal;
	TmMSecs = InTmMSecs;
}

void TSum::Load(TSIn& SIn) {
	*this = TSum(SIn);
}

void TSum::Save(TSOut& SOut) const {
	// parameters
	Sum.Save(SOut); 
	TmMSecs.Save(SOut);
}

/////////////////////////////////////////////////
// Online Summa 

/// Packs sparse vector
void TSumSpVec::Update(const TVec<TIntFltKd>& InVal, const uint64& InTmMSecs, const TVec<TIntFltKdV>& OutValV, const TUInt64V& OutTmMSecsV) {

	// remove old values from the sum
	TIntFltKdV Tmp;
	for (int i = 0; i < OutValV.Len(); i++) {
		TLinAlg::LinComb(1, Sum, -1, OutValV[i], Tmp);
		Sum = Tmp;
	}
	TLinAlg::LinComb(1, Sum, 1, InVal, Tmp);
	Sum = Tmp;
	TmMSecs = InTmMSecs;
}

void TSumSpVec::Load(TSIn& SIn) {
	*this = TSumSpVec(SIn);
}

void TSumSpVec::Save(TSOut& SOut) const {
	// parameters
	Sum.Save(SOut);
	TmMSecs.Save(SOut);
}

PJsonVal TSumSpVec::GetJson() const {
	PJsonVal arr = TJsonVal::NewArr();
	for (int i = 0; i < Sum.Len(); i++) {
		PJsonVal tmp = TJsonVal::NewObj();
		tmp->AddToObj("Idx", Sum[i].Key);
		tmp->AddToObj("Val", Sum[i].Dat);
		arr->AddToArr(tmp);
	}
	PJsonVal res = TJsonVal::NewObj();
	res->AddToObj("Sum", arr);
	res->AddToObj("Tm", TmMSecs);
	return res;
}

/////////////////////////////////////////////////
// Online Min 
void TMin::Update(const double& InVal, const uint64& InTmMSecs,
	const TFltV& OutValV, const TUInt64V& OutTmMSecsV){
		
	while (!AllValV.Empty() && AllValV[AllValV.Len() - 1].Val1 >= InVal) {
		// pop back
		AllValV.DelLast();
	}
	// push back
	AllValV.Add(TFltUInt64Pr(InVal, InTmMSecs));
	

	if (!OutTmMSecsV.Empty()) {
		// find maximum timestamp of outgoing measurements
		uint64 MaxOutTm = OutTmMSecsV.Last();

		while (AllValV[0].Val2 <= MaxOutTm) {
			// pop front
			AllValV.Del(0);
		}
	}

	TmMSecs = InTmMSecs;
	Min = AllValV[0].Val1;
}

void TMin::Load(TSIn& SIn) {
	*this = TMin(SIn);
}

void TMin::Save(TSOut& SOut) const {
	// parameters
	Min.Save(SOut);
	TmMSecs.Save(SOut);
	AllValV.Save(SOut);
}

/////////////////////////////////////////////////
// Online Max 
void TMax::Update(const double& InVal, const uint64& InTmMSecs,
	const TFltV& OutValV, const TUInt64V& OutTmMSecsV){

	
	while (!AllValV.Empty() && AllValV[AllValV.Len() - 1].Val1 <= InVal) {
		// pop back
		AllValV.DelLast();
	}
	// push back
	AllValV.Add(TFltUInt64Pr(InVal, InTmMSecs));


	if (!OutTmMSecsV.Empty()) {
		// find maximum timestamp of outgoing measurements
		uint64 MaxOutTm = OutTmMSecsV.Last();
		while (AllValV[0].Val2 <= MaxOutTm) {
			// pop front
			AllValV.Del(0);
		}
	}

	TmMSecs = InTmMSecs;
	Max = AllValV[0].Val1;
}

void TMax::Load(TSIn& SIn) {
	*this = TMax(SIn);
}

void TMax::Save(TSOut& SOut) const {
	// parameters
	Max.Save(SOut);
	TmMSecs.Save(SOut);
	AllValV.Save(SOut);
}

/////////////////////////////////////////////////
// Exponential Moving Average
double TEma::GetNi(const double& Alpha, const double& Mi) {
	switch (Type) {
	case etPreviousPoint: return 1.0;
	case etLinear: return (1 - Mi) / Alpha;
	case etNextPoint: return Mi;
	}
	throw TExcept::New("Unknown EMA interpolation type");
}

//TODO: compute InitMinMSecs initialization time window from decay factor
TEma::TEma(const double& _Decay, const TEmaType& _Type, const uint64& _InitMinMSecs, 
    const double& _TmInterval): Decay(_Decay), Type(_Type), LastVal(TFlt::Mn),
    TmInterval(_TmInterval), InitP(false), InitMinMSecs(_InitMinMSecs) { }

//TODO: compute InitMinMSecs initialization time window from decay factor
TEma::TEma(const TEmaType& _Type, const uint64& _InitMinMSecs,const double& _TmInterval):
    Type(_Type), LastVal(TFlt::Mn), TmInterval(_TmInterval), InitP(false), 
    InitMinMSecs(_InitMinMSecs) { }

TEma::TEma(const PJsonVal& ParamVal) : LastVal(TFlt::Mn), InitP(false) {
    // type
    TStr TypeStr = ParamVal->GetObjStr("emaType");
    if (TypeStr == "previous") { 
        Type = etPreviousPoint;
    } else if (TypeStr == "linear") {
        Type = etLinear;
    } else if (TypeStr == "next") {
        Type = etNextPoint;
    } else {
        throw TExcept::New("Unknown ema type " + TypeStr);
    }
    // rest
    TmInterval = ParamVal->GetObjNum("interval");
    InitMinMSecs = ParamVal->GetObjInt("initWindow", 0);
}

TEma::TEma(TSIn& SIn) : Decay(SIn), LastVal(SIn), Ema(SIn), TmMSecs(SIn), InitP(SIn),
InitMinMSecs(SIn), InitValV(SIn), InitMSecsV(SIn) {

	TInt TypeI; TypeI.Load(SIn);
	Type = static_cast<TEmaType>((int)TypeI);
	TFlt TmIntervalFlt; TmIntervalFlt.Load(SIn); TmInterval = TmIntervalFlt;
}

void TEma::Load(TSIn& SIn) {
	*this = TEma(SIn);
}

void TEma::Save(TSOut& SOut) const {
	// parameters
	Decay.Save(SOut);
	LastVal.Save(SOut);
	Ema.Save(SOut);
	TmMSecs.Save(SOut);
	InitP.Save(SOut);
	InitMinMSecs.Save(SOut);
	InitValV.Save(SOut);
	InitMSecsV.Save(SOut);
	// TODO: Use macro for saving enum (SaveEnum, LoadEnum)
	// TODO: change TmInterval from double to TFlt
	// PROBLEM: After changing TmInterval from double to TFlt Qminer crashes hard!
	TInt TypeI = Type; // TEmaType 
	TypeI.Save(SOut);
	TFlt TmIntervalFlt = TmInterval; // double
	TmIntervalFlt.Save(SOut);;
}

void TEma::Update(const double& Val, const uint64& NewTmMSecs) {
	double TmInterval1;
	// EMA(first_point) = first_point (no smoothing is possible)
	if (InitMinMSecs == 0) {
		if (LastVal == TFlt::Mn) { LastVal = Val; Ema = Val; TmMSecs = NewTmMSecs; InitP = true;  return; }
	}
	if(NewTmMSecs == TmMSecs) {
		TmInterval1 = 1.0;
    } else{
		TmInterval1 = (double)(NewTmMSecs - TmMSecs);
    }
	if (InitP) {
		// computer parameters for EMA
		double Alpha;
		if (Decay == 0.0) {			
			Alpha = TmInterval1 / TmInterval;
		} else {			
			Alpha = TmInterval1 / TmInterval  * (-1.0) * TMath::Log(Decay);
		}
		const double Mi = exp(-Alpha);
		const double Ni = GetNi(Alpha, Mi);
		// compute new ema
		Ema = Mi*Ema + (Ni - Mi)*LastVal + (1.0 - Ni)*Val;
	} else {
		// update buffers
		InitValV.Add(Val);
		InitMSecsV.Add(NewTmMSecs);
		// initialize when enough data
		const uint64 StartInitMSecs = InitMSecsV[0] + InitMinMSecs;
		if (StartInitMSecs < NewTmMSecs) {
			// Initialize using "buildup time interval",
			//TODO: check how interpolation type influences this code
			const int Vals = InitMSecsV.Len();	
			// compute weights for each value in buffer
			TFltV WeightV(Vals, 0);
			for (int ValN = 0; ValN < Vals; ValN++) {
				const double Alpha = (double)(TmInterval1) / Decay;
				WeightV.Add(exp(-Alpha));
			}
			// normalize weights so they sum to 1.0
			TLinAlg::NormalizeL1(WeightV);
			// compute initial value of EMA as weighted sum
			Ema = TLinAlg::DotProduct(WeightV, InitValV);
			// mark that we are done and clean up after us
			InitP = true; InitValV.Clr(); InitMSecsV.Clr();
		}
	}
	// update last value
	LastVal = Val;
	// update curret time
	TmMSecs = NewTmMSecs;
}

void TEma::Reset() {
	InitP = false;
	LastVal = TFlt::Mn;
	Ema = 0.0;
	TmMSecs = 0;
	InitValV.Gen(0);
	InitMSecsV.Gen(0);
}

/////////////////////////////////////////////////
// Exponential Moving Average - for sparse vectors

double TEmaSpVec::GetNi(const double& Alpha, const double& Mi) {
	switch (Type) {
	case etPreviousPoint: return 1.0;
	case etLinear: return (1 - Mi) / Alpha;
	case etNextPoint: return Mi;
	}
	throw TExcept::New("Unknown EMA interpolation type");
}

TEmaSpVec::TEmaSpVec(const TEmaType& _Type, const uint64& _InitMinMSecs, const double& _TmInterval, const double& _Cutoff) :
	Type(_Type), LastVal(TFlt::Mn), TmInterval(_TmInterval), Cutoff(_Cutoff), InitP(false),
	InitMinMSecs(_InitMinMSecs) {}

TEmaSpVec::TEmaSpVec(const PJsonVal& ParamVal) : LastVal(), InitP(false) {
	// type
	TStr TypeStr = ParamVal->GetObjStr("emaType");
	if (TypeStr == "previous") {
		Type = etPreviousPoint;
	} else if (TypeStr == "linear") {
		Type = etLinear;
	} else if (TypeStr == "next") {
		Type = etNextPoint;
	} else {
		throw TExcept::New("Unknown ema type " + TypeStr);
	}
	// rest
	TmInterval = ParamVal->GetObjNum("interval");
	Cutoff = ParamVal->GetObjNum("cutoff", 0.0001);
	InitMinMSecs = ParamVal->GetObjInt("initWindow", 0);
}

TEmaSpVec::TEmaSpVec(TSIn& SIn) : LastVal(SIn), Ema(SIn), TmMSecs(SIn), 
	TmInterval(SIn), Cutoff(SIn), InitP(SIn),
	InitMinMSecs(SIn), InitValV(SIn), InitMSecsV(SIn) {

	TInt TypeI; TypeI.Load(SIn);
	Type = static_cast<TEmaType>((int)TypeI);
	//TFlt TmIntervalFlt; TmIntervalFlt.Load(SIn); TmInterval = TmIntervalFlt;
	//TFlt CutoffFlt; CutoffFlt.Load(SIn); Cutoff = CutoffFlt;
}

void TEmaSpVec::Load(TSIn& SIn) {
	*this = TEmaSpVec(SIn);
}

void TEmaSpVec::Save(TSOut& SOut) const {
	// parameters
	LastVal.Save(SOut);
	Ema.Save(SOut);
	TmMSecs.Save(SOut);
	TmInterval.Save(SOut);
	Cutoff.Save(SOut);
	InitP.Save(SOut);
	InitMinMSecs.Save(SOut);
	InitValV.Save(SOut);
	InitMSecsV.Save(SOut);
	// TODO: Use macro for saving enum (SaveEnum, LoadEnum)
	// TODO: change TmInterval from double to TFlt
	// PROBLEM: After changing TmInterval from double to TFlt Qminer crashes hard!
	TInt TypeI = Type; // TEmaType 
	TypeI.Save(SOut);
	//TFlt TmIntervalFlt = TmInterval; // double
	//TmIntervalFlt.Save(SOut);
	//TFlt CutoffFlt = Cutoff; // double
	//CutoffFlt.Save(SOut);
}

void TEmaSpVec::Update(const TIntFltKdV& Val, const uint64& NewTmMSecs) {
	double TmInterval1;
	// EMA(first_point) = first_point (no smoothing is possible)
	if (InitMinMSecs == 0) {
		if (LastVal.Empty()) { 
			LastVal = Val; 
			Ema = Val; 
			TmMSecs = NewTmMSecs; 
			InitP = true;  
			return; 
		}
	}
	if (NewTmMSecs == TmMSecs) {
		TmInterval1 = 1.0;
	} else {
		TmInterval1 = (double)(NewTmMSecs - TmMSecs);
	}
	if (InitP) {
		// compute parameters for EMA
		double Alpha = TmInterval1 / TmInterval;
		const double Mi = exp(-Alpha);
		const double Ni = GetNi(Alpha, Mi);
		// compute new ema
		//Ema = Mi*Ema + (Ni - Mi)*LastVal + (1.0 - Ni)*Val;
		TIntFltKdV Tmp;
		TLinAlg::LinComb(Mi, Ema, Ni - Mi, LastVal, Tmp);		
		TLinAlg::LinComb(1, Tmp, 1.0 - Ni, Val, Ema);
	} else {
		// update buffers
		InitValV.Add(Val);
		InitMSecsV.Add(NewTmMSecs);
		// initialize when enough data
		const uint64 StartInitMSecs = InitMSecsV[0] + InitMinMSecs;
		if (StartInitMSecs < NewTmMSecs) {
			// Initialize using "buildup time interval",
			//TODO: check how interpolation type influences this code
			const int Vals = InitMSecsV.Len();
			// compute weights for each value in buffer
			TFltV WeightV(Vals, 0);
			for (int ValN = 0; ValN < Vals; ValN++) {
				const double Alpha = (double)(TmInterval1);
				WeightV.Add(exp(-Alpha));
			}
			// normalize weights so they sum to 1.0
			TLinAlg::NormalizeL1(WeightV);
			// compute initial value of EMA as weighted sum
			//Ema = TLinAlg::DotProduct(WeightV, InitValV);
			TIntFltKdV Tmp;
			for (int i = 0; i < WeightV.Len(); i++) {
				TIntFltKdV Tmp2;
				TLinAlg::LinComb(1, Tmp, WeightV[i], InitValV[i], Tmp2);
				Tmp = Tmp2;
			}
			Ema = Tmp;

			// mark that we are done and clean up after us
			InitP = true; 
			InitValV.Clr();
			InitMSecsV.Clr();
		}
	}
	//// remove dimensions bellow cutoff
	//TIntFltKdV TmpEma;
	//for (int i = 0; i < Ema.Len(); i++) {
	//	if (abs(Ema[i].Dat) >= Cutoff) {
	//		TmpEma.Add(Ema[i]);
	//	}
	//}
	//Ema = TmpEma;

	// update last value
	LastVal = Val;
	// update current time
	TmMSecs = NewTmMSecs;
}

void TEmaSpVec::Reset() {
	InitP = false;<commit_msg>More tests<commit_after>/**
 * Copyright (c) 2015, <NAME>, Quintelligence d.o.o. and contributors
 * All rights reserved.
 * 
 * This source code is licensed under the FreeBSD license found in the
 * LICENSE file in the root directory of this source tree.
 */

namespace TSignalProc {

/////////////////////////////////////////////////
// Simple Online Moving Variance
void TVarSimple::Update(const double& InVal) {	 
	// See Knuth TAOCP vol 2, 3rd edition, page 232
	N++;
	if (N == 1) {
		OldM = NewM = InVal;
		OldS = 0.0;
	} else {
		NewM = OldM + (InVal - OldM) / N;
		NewS = OldS + (InVal - OldM)*(InVal - NewM);
		// set up for next iteration
		OldM = NewM;
		OldS = NewS;
	}
}

void TVarSimple::Load(TSIn& SIn) {
	*this = TVarSimple(SIn);
}

void TVarSimple::Save(TSOut& SOut) const {
	OldM.Save(SOut);
	NewM.Save(SOut);
	OldS.Save(SOut);
	NewS.Save(SOut);
	N.Save(SOut);
}
    
/////////////////////////////////////////////////
// Online Moving Average 
void TMa::Update(const double& InVal, const uint64& InTmMSecs, 
        const TFltV& OutValV, const TUInt64V& OutTmMSecsV, const int& N){
    
    int tempN = N - 1 + OutValV.Len();
    double delta;    
	// check if all the values are removed
	if (OutValV.Len() >= tempN) {
		Ma = 0;		
		tempN = 0;
	}
	else {
		// remove old values from the mean
		for (int ValN = 0; ValN < OutValV.Len(); ValN++)
		{
			tempN--;
			delta = OutValV[ValN] - Ma;
			Ma = Ma - delta / tempN;
		}
	}
    //add the new value to the resulting mean    
    delta = InVal - Ma;
    Ma = Ma + delta/N;
    TmMSecs = InTmMSecs;
}

TMa::TMa(TSIn& SIn) : Ma(SIn), TmMSecs(SIn) { }

void TMa::Load(TSIn& SIn) {
	*this = TMa(SIn);
}

void TMa::Save(TSOut& SOut) const {
	// parameters
	Ma.Save(SOut);
	TmMSecs.Save(SOut);
}

/////////////////////////////////////////////////
// Online Summa 
void TSum::Update(const double& InVal, const uint64& InTmMSecs,
	const TFltV& OutValV, const TUInt64V& OutTmMSecsV){

	// remove old values from the sum
	for (int ValN = 0; ValN < OutValV.Len(); ValN++)
	{
		Sum -= OutValV[ValN];
	}
	//add the new value to the resulting sum    
	Sum += InVal;
	TmMSecs = InTmMSecs;
}

void TSum::Load(TSIn& SIn) {
	*this = TSum(SIn);
}

void TSum::Save(TSOut& SOut) const {
	// parameters
	Sum.Save(SOut); 
	TmMSecs.Save(SOut);
}

/////////////////////////////////////////////////
// Online Summa 

/// Packs sparse vector
void TSumSpVec::Update(const TVec<TIntFltKd>& InVal, const uint64& InTmMSecs, const TVec<TIntFltKdV>& OutValV, const TUInt64V& OutTmMSecsV) {

	// remove old values from the sum
	TIntFltKdV Tmp;
	for (int i = 0; i < OutValV.Len(); i++) {
		TLinAlg::LinComb(1, Sum, -1, OutValV[i], Tmp);
		Sum = Tmp;
	}
	TLinAlg::LinComb(1, Sum, 1, InVal, Tmp);
	Sum = Tmp;
	TmMSecs = InTmMSecs;
}

void TSumSpVec::Load(TSIn& SIn) {
	*this = TSumSpVec(SIn);
}

void TSumSpVec::Save(TSOut& SOut) const {
	// parameters
	Sum.Save(SOut);
	TmMSecs.Save(SOut);
}

PJsonVal TSumSpVec::GetJson() const {
	PJsonVal arr = TJsonVal::NewArr();
	for (int i = 0; i < Sum.Len(); i++) {
		PJsonVal tmp = TJsonVal::NewObj();
		tmp->AddToObj("Idx", Sum[i].Key);
		tmp->AddToObj("Val", Sum[i].Dat);
		arr->AddToArr(tmp);
	}
	PJsonVal res = TJsonVal::NewObj();
	res->AddToObj("Sum", arr);
	res->AddToObj("Tm", TmMSecs);
	return res;
}

/////////////////////////////////////////////////
// Online Min 
void TMin::Update(const double& InVal, const uint64& InTmMSecs,
	const TFltV& OutValV, const TUInt64V& OutTmMSecsV){
		
	while (!AllValV.Empty() && AllValV[AllValV.Len() - 1].Val1 >= InVal) {
		// pop back
		AllValV.DelLast();
	}
	// push back
	AllValV.Add(TFltUInt64Pr(InVal, InTmMSecs));
	

	if (!OutTmMSecsV.Empty()) {
		// find maximum timestamp of outgoing measurements
		uint64 MaxOutTm = OutTmMSecsV.Last();

		while (AllValV[0].Val2 <= MaxOutTm) {
			// pop front
			AllValV.Del(0);
		}
	}

	TmMSecs = InTmMSecs;
	Min = AllValV[0].Val1;
}

void TMin::Load(TSIn& SIn) {
	*this = TMin(SIn);
}

void TMin::Save(TSOut& SOut) const {
	// parameters
	Min.Save(SOut);
	TmMSecs.Save(SOut);
	AllValV.Save(SOut);
}

/////////////////////////////////////////////////
// Online Max 
void TMax::Update(const double& InVal, const uint64& InTmMSecs,
	const TFltV& OutValV, const TUInt64V& OutTmMSecsV){

	
	while (!AllValV.Empty() && AllValV[AllValV.Len() - 1].Val1 <= InVal) {
		// pop back
		AllValV.DelLast();
	}
	// push back
	AllValV.Add(TFltUInt64Pr(InVal, InTmMSecs));


	if (!OutTmMSecsV.Empty()) {
		// find maximum timestamp of outgoing measurements
		uint64 MaxOutTm = OutTmMSecsV.Last();
		while (AllValV[0].Val2 <= MaxOutTm) {
			// pop front
			AllValV.Del(0);
		}
	}

	TmMSecs = InTmMSecs;
	Max = AllValV[0].Val1;
}

void TMax::Load(TSIn& SIn) {
	*this = TMax(SIn);
}

void TMax::Save(TSOut& SOut) const {
	// parameters
	Max.Save(SOut);
	TmMSecs.Save(SOut);
	AllValV.Save(SOut);
}

/////////////////////////////////////////////////
// Exponential Moving Average
double TEma::GetNi(const double& Alpha, const double& Mi) {
	switch (Type) {
	case etPreviousPoint: return 1.0;
	case etLinear: return (1 - Mi) / Alpha;
	case etNextPoint: return Mi;
	}
	throw TExcept::New("Unknown EMA interpolation type");
}

//TODO: compute InitMinMSecs initialization time window from decay factor
TEma::TEma(const double& _Decay, const TEmaType& _Type, const uint64& _InitMinMSecs, 
    const double& _TmInterval): Decay(_Decay), Type(_Type), LastVal(TFlt::Mn),
    TmInterval(_TmInterval), InitP(false), InitMinMSecs(_InitMinMSecs) { }

//TODO: compute InitMinMSecs initialization time window from decay factor
TEma::TEma(const TEmaType& _Type, const uint64& _InitMinMSecs,const double& _TmInterval):
    Type(_Type), LastVal(TFlt::Mn), TmInterval(_TmInterval), InitP(false), 
    InitMinMSecs(_InitMinMSecs) { }

TEma::TEma(const PJsonVal& ParamVal) : LastVal(TFlt::Mn), InitP(false) {
    // type
    TStr TypeStr = ParamVal->GetObjStr("emaType");
    if (TypeStr == "previous") { 
        Type = etPreviousPoint;
    } else if (TypeStr == "linear") {
        Type = etLinear;
    } else if (TypeStr == "next") {
        Type = etNextPoint;
    } else {
        throw TExcept::New("Unknown ema type " + TypeStr);
    }
    // rest
    TmInterval = ParamVal->GetObjNum("interval");
    InitMinMSecs = ParamVal->GetObjInt("initWindow", 0);
}

TEma::TEma(TSIn& SIn) : Decay(SIn), LastVal(SIn), Ema(SIn), TmMSecs(SIn), InitP(SIn),
InitMinMSecs(SIn), InitValV(SIn), InitMSecsV(SIn) {

	TInt TypeI; TypeI.Load(SIn);
	Type = static_cast<TEmaType>((int)TypeI);
	TFlt TmIntervalFlt; TmIntervalFlt.Load(SIn); TmInterval = TmIntervalFlt;
}

void TEma::Load(TSIn& SIn) {
	*this = TEma(SIn);
}

void TEma::Save(TSOut& SOut) const {
	// parameters
	Decay.Save(SOut);
	LastVal.Save(SOut);
	Ema.Save(SOut);
	TmMSecs.Save(SOut);
	InitP.Save(SOut);
	InitMinMSecs.Save(SOut);
	InitValV.Save(SOut);
	InitMSecsV.Save(SOut);
	// TODO: Use macro for saving enum (SaveEnum, LoadEnum)
	// TODO: change TmInterval from double to TFlt
	// PROBLEM: After changing TmInterval from double to TFlt Qminer crashes hard!
	TInt TypeI = Type; // TEmaType 
	TypeI.Save(SOut);
	TFlt TmIntervalFlt = TmInterval; // double
	TmIntervalFlt.Save(SOut);;
}

void TEma::Update(const double& Val, const uint64& NewTmMSecs) {
	double TmInterval1;
	// EMA(first_point) = first_point (no smoothing is possible)
	if (InitMinMSecs == 0) {
		if (LastVal == TFlt::Mn) { LastVal = Val; Ema = Val; TmMSecs = NewTmMSecs; InitP = true;  return; }
	}
	if(NewTmMSecs == TmMSecs) {
		TmInterval1 = 1.0;
    } else{
		TmInterval1 = (double)(NewTmMSecs - TmMSecs);
    }
	if (InitP) {
		// computer parameters for EMA
		double Alpha;
		if (Decay == 0.0) {			
			Alpha = TmInterval1 / TmInterval;
		} else {			
			Alpha = TmInterval1 / TmInterval  * (-1.0) * TMath::Log(Decay);
		}
		const double Mi = exp(-Alpha);
		const double Ni = GetNi(Alpha, Mi);
		// compute new ema
		Ema = Mi*Ema + (Ni - Mi)*LastVal + (1.0 - Ni)*Val;
	} else {
		// update buffers
		InitValV.Add(Val);
		InitMSecsV.Add(NewTmMSecs);
		// initialize when enough data
		const uint64 StartInitMSecs = InitMSecsV[0] + InitMinMSecs;
		if (StartInitMSecs < NewTmMSecs) {
			// Initialize using "buildup time interval",
			//TODO: check how interpolation type influences this code
			const int Vals = InitMSecsV.Len();	
			// compute weights for each value in buffer
			TFltV WeightV(Vals, 0);
			for (int ValN = 0; ValN < Vals; ValN++) {
				const double Alpha = (double)(TmInterval1) / Decay;
				WeightV.Add(exp(-Alpha));
			}
			// normalize weights so they sum to 1.0
			TLinAlg::NormalizeL1(WeightV);
			// compute initial value of EMA as weighted sum
			Ema = TLinAlg::DotProduct(WeightV, InitValV);
			// mark that we are done and clean up after us
			InitP = true; InitValV.Clr(); InitMSecsV.Clr();
		}
	}
	// update last value
	LastVal = Val;
	// update curret time
	TmMSecs = NewTmMSecs;
}

void TEma::Reset() {
	InitP = false;
	LastVal = TFlt::Mn;
	Ema = 0.0;
	TmMSecs = 0;
	InitValV.Gen(0);
	InitMSecsV.Gen(0);
}

/////////////////////////////////////////////////
// Exponential Moving Average - for sparse vectors

double TEmaSpVec::GetNi(const double& Alpha, const double& Mi) {
	switch (Type) {
	case etPreviousPoint: return 1.0;
	case etLinear: return (1 - Mi) / Alpha;
	case etNextPoint: return Mi;
	}
	throw TExcept::New("Unknown EMA interpolation type");
}

TEmaSpVec::TEmaSpVec(const TEmaType& _Type, const uint64& _InitMinMSecs, const double& _TmInterval, const double& _Cutoff) :
	Type(_Type), LastVal(TFlt::Mn), TmInterval(_TmInterval), Cutoff(_Cutoff), InitP(false),
	InitMinMSecs(_InitMinMSecs) {}

TEmaSpVec::TEmaSpVec(const PJsonVal& ParamVal) : LastVal(), InitP(false) {
	// type
	TStr TypeStr = ParamVal->GetObjStr("emaType");
	if (TypeStr == "previous") {
		Type = etPreviousPoint;
	} else if (TypeStr == "linear") {
		Type = etLinear;
	} else if (TypeStr == "next") {
		Type = etNextPoint;
	} else {
		throw TExcept::New("Unknown ema type " + TypeStr);
	}
	// rest
	TmInterval = ParamVal->GetObjNum("interval");
	Cutoff = ParamVal->GetObjNum("cutoff", 0.0001);
	InitMinMSecs = ParamVal->GetObjInt("initWindow", 0);
}

TEmaSpVec::TEmaSpVec(TSIn& SIn) : LastVal(SIn), Ema(SIn), TmMSecs(SIn), 
	TmInterval(SIn), Cutoff(SIn), InitP(SIn),
	InitMinMSecs(SIn), InitValV(SIn), InitMSecsV(SIn) {

	TInt TypeI; TypeI.Load(SIn);
	Type = static_cast<TEmaType>((int)TypeI);
	//TFlt TmIntervalFlt; TmIntervalFlt.Load(SIn); TmInterval = TmIntervalFlt;
	//TFlt CutoffFlt; CutoffFlt.Load(SIn); Cutoff = CutoffFlt;
}

void TEmaSpVec::Load(TSIn& SIn) {
	*this = TEmaSpVec(SIn);
}

void TEmaSpVec::Save(TSOut& SOut) const {
	// parameters
	LastVal.Save(SOut);
	Ema.Save(SOut);
	TmMSecs.Save(SOut);
	TmInterval.Save(SOut);
	Cutoff.Save(SOut);
	InitP.Save(SOut);
	InitMinMSecs.Save(SOut);
	InitValV.Save(SOut);
	InitMSecsV.Save(SOut);
	// TODO: Use macro for saving enum (SaveEnum, LoadEnum)
	// TODO: change TmInterval from double to TFlt
	// PROBLEM: After changing TmInterval from double to TFlt Qminer crashes hard!
	TInt TypeI = Type; // TEmaType 
	TypeI.Save(SOut);
	//TFlt TmIntervalFlt = TmInterval; // double
	//TmIntervalFlt.Save(SOut);
	//TFlt CutoffFlt = Cutoff; // double
	//CutoffFlt.Save(SOut);
}

void TEmaSpVec::Update(const TIntFltKdV& Val, const uint64& NewTmMSecs) {
	double TmInterval1;
	// EMA(first_point) = first_point (no smoothing is possible)
	if (InitMinMSecs == 0) {
		if (LastVal.Empty()) { 
			LastVal = Val; 
			Ema = Val; 
			TmMSecs = NewTmMSecs; 
			InitP = true;  
			return; 
		}
	}
	if (NewTmMSecs == TmMSecs) {
		TmInterval1 = 1.0;
	} else {
		TmInterval1 = (double)(NewTmMSecs - TmMSecs);
	}
	if (InitP) {
		// compute parameters for EMA
		double Alpha = TmInterval1 / TmInterval;
		const double Mi = exp(-Alpha);
		const double Ni = GetNi(Alpha, Mi);
		// compute new ema
		//Ema = Mi*Ema + (Ni - Mi)*LastVal + (1.0 - Ni)*Val;
		TIntFltKdV Tmp;
		TLinAlg::LinComb(Mi, Ema, Ni - Mi, LastVal, Tmp);		
		TLinAlg::LinComb(1, Tmp, 1.0 - Ni, Val, Ema);
	} else {
		// update buffers
		InitValV.Add(Val);
		InitMSecsV.Add(NewTmMSecs);
		// initialize when enough data
		const uint64 StartInitMSecs = InitMSecsV[0] + InitMinMSecs;
		if (StartInitMSecs < NewTmMSecs) {
			// Initialize using "buildup time interval",
			//TODO: check how interpolation type influences this code
			const int Vals = InitMSecsV.Len();
			// compute weights for each value in buffer
			TFltV WeightV(Vals, 0);
			for (int ValN = 0; ValN < Vals; ValN++) {
				const double Alpha = (double)(TmInterval1);
				WeightV.Add(exp(-Alpha));
			}
			// normalize weights so they sum to 1.0
			TLinAlg::NormalizeL1(WeightV);
			// compute initial value of EMA as weighted sum
			//Ema = TLinAlg::DotProduct(WeightV, InitValV);
			TIntFltKdV Tmp;
			for (int i = 0; i < WeightV.Len(); i++) {
				TIntFltKdV Tmp2;
				TLinAlg::LinComb(1, Tmp, WeightV[i], InitValV[i], Tmp2);
				Tmp = Tmp2;
			}
			Ema = Tmp;

			// mark that we are done and clean up after us
			InitP = true; 
			InitValV.Clr();
			InitMSecsV.Clr();
		}
	}
	// remove dimensions bellow cutoff
	TIntFltKdV TmpEma;
	for (int i = 0; i < Ema.Len(); i++) {
		if (abs(Ema[i].Dat) >= Cutoff) {
			TmpEma.Add(Ema[i]);
		}
	}
	Ema = TmpEma;

	// update last value
	LastVal = Val;
	// update current time
	TmMSecs = NewTmMSecs;
}

void TEmaSpVec::Reset() {
	InitP = false;
{'fraction_non_alphanumeric': 0.1734782118284017, 'fraction_numerical': 0.01075309245403617, 'mean_word_length': 3.1979166666666665, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4413018', 'n_tokens_mistral': 11351, 'n_tokens_neox': 10695, 'n_words': 2476}