<commit_before>/**
 * Copyright (c) 2015, <NAME>, Quintelligence d.o.o. and contributors
 * All rights reserved.
 * 
 * This source code is licensed under the FreeBSD license found in the
 * LICENSE file in the root directory of this source tree.
 */

#include "qminer_ftr.h"

namespace TQm {

///////////////////////////////////////////////
// QMiner-Feature-Extractor
TFunRouter<PFtrExt, TFtrExt::TNewF> TFtrExt::NewRouter;
TFunRouter<PFtrExt, TFtrExt::TLoadF> TFtrExt::LoadRouter;

void TFtrExt::Init() {
    Register<TFtrExts::TRandom>();
	Register<TFtrExts::TConstant>();
    Register<TFtrExts::TNumeric>();
    Register<TFtrExts::TNumSpV>();
    Register<TFtrExts::TCategorical>();
    Register<TFtrExts::TMultinomial>();
    Register<TFtrExts::TBagOfWords>();
    Register<TFtrExts::TJoin>();
    Register<TFtrExts::TPair>();
    Register<TFtrExts::TDateWnd>();
}
    
TRec TFtrExt::DoSingleJoin(const TRec& Rec) const {
	QmAssertR(IsStartStore(Rec.GetStoreId()), "Start store not supported!");
	return Rec.DoSingleJoin(Base, GetJoinIdV(Rec.GetStoreId()));
}

TFtrExt::TFtrExt(const TWPt<TBase>& _Base, const TJoinSeqV& JoinSeqV): Base(_Base) {
	QmAssertR(!JoinSeqV.Empty(), "At least one join sequence must be supplied!");
	FtrStore = JoinSeqV[0].GetEndStore(Base);
    // get mapping from stores to join sequence
	for (int JoinSeqN = 0; JoinSeqN < JoinSeqV.Len(); JoinSeqN++) {
		const TJoinSeq& JoinSeq = JoinSeqV[JoinSeqN];
        QmAssertR(JoinSeq.GetEndStoreId(Base) == FtrStore->GetStoreId(), "Mismatching feature stores");
		JoinSeqH.AddDat(JoinSeq.GetStartStoreId(), JoinSeq);
	}
}

TFtrExt::TFtrExt(const TWPt<TBase>& _Base, const PJsonVal& ParamVal): Base(_Base) {
    QmAssert(ParamVal->IsObjKey("source"));
    PJsonVal SourceVal = ParamVal->GetObjKey("source");
    if (SourceVal->IsStr()) {
        // we have just store name
        TStr StoreNm = SourceVal->GetStr();
        FtrStore = Base->GetStoreByStoreNm(StoreNm);
        JoinSeqH.AddDat(FtrStore->GetStoreId(), TJoinSeq(FtrStore->GetStoreId()));
    } else if (SourceVal->IsObj()) {
        // get store
        TStr StoreNm = SourceVal->GetObjStr("store");
        FtrStore = Base->GetStoreByStoreNm(StoreNm);       
        // get joins if any given
        if (SourceVal->IsObjKey("join")) {
            TJoinSeq JoinSeq = TJoinSeq(Base, FtrStore->GetStoreId(), SourceVal->GetObjKey("join"));
            FtrStore = JoinSeq.GetEndStore(Base);
            JoinSeqH.AddDat(JoinSeq.GetStartStoreId(), JoinSeq);
        } else {
            JoinSeqH.AddDat(FtrStore->GetStoreId(), TJoinSeq(FtrStore->GetStoreId()));            
        }
    } else if (SourceVal->IsArr()) {
        // parse join sequence
        for (int SourceN = 0; SourceN < SourceVal->GetArrVals(); SourceN++) {
            PJsonVal JoinVal = SourceVal->GetArrVal(SourceN);
            // get store
            TStr StoreNm = JoinVal->GetObjStr("store");
            TWPt<TStore> Store = Base->GetStoreByStoreNm(StoreNm);       
            // get joins if any give
            if (JoinVal->IsObjKey("join")) {                
                TJoinSeq JoinSeq = TJoinSeq(Base, Store->GetStoreId(), JoinVal->GetObjKey("join"));
                Store = JoinSeq.GetEndStore(Base);
        		JoinSeqH.AddDat(JoinSeq.GetStartStoreId(), JoinSeq);
            } else {
                // no join, just remember the store
                JoinSeqH.AddDat(Store->GetStoreId(), TJoinSeq(Store->GetStoreId()));
            }
            // update feature store or assert still consistent
            if (SourceN == 0) { FtrStore = Store; } 
            QmAssertR(FtrStore->GetStoreId() == Store->GetStoreId(), "Mismatch of feature stores");
        }
    }
}

TFtrExt::TFtrExt(const TWPt<TBase>& _Base, TSIn& SIn): Base(_Base) {
    JoinSeqH.Load(SIn);
    FtrStore = TStore::LoadById(Base, SIn);
}

PFtrExt TFtrExt::New(const TWPt<TBase>& Base, const TStr& TypeNm, const PJsonVal& ParamVal) {
    return NewRouter.Fun(TypeNm)(Base, ParamVal);
}

PFtrExt TFtrExt::Load(const TWPt<TBase>& Base, TSIn& SIn) {
	TStr TypeNm(SIn);
    return LoadRouter.Fun(TypeNm)(Base, SIn);
}

void TFtrExt::Save(TSOut& SOut) const { 
    JoinSeqH.Save(SOut); 
    FtrStore->SaveId(SOut);
}

void TFtrExt::AddFullV(const TRec& Rec, TFltV& FullV, int& Offset) const {
    // get sparse vector
    TIntFltKdV SpV; AddSpV(Rec, SpV, Offset);
    // add to full vector
    for (int SpN = 0; SpN < SpV.Len(); SpN++) {
        FullV[SpV[SpN].Key] = SpV[SpN].Dat; }
}

void TFtrExt::ExtractStrV(const TRec& FtrRec, TStrV& StrV) const { 
	throw TQmExcept::New("ExtractStrV not implemented!"); 
}

void TFtrExt::ExtractFltV(const TRec& FtrRec, TFltV& FltV) const {
	throw TQmExcept::New("ExtractFltV not implemented!"); 
}

void TFtrExt::ExtractTmV(const TRec& FtrRec, TTmV& TmV) const {
	throw TQmExcept::New("ExtractTmV not implemented!"); 
}

///////////////////////////////////////////////
// QMiner-Feature-Space
void TFtrSpace::Init() {
    // we start with empty space
	DimV.Gen(FtrExtV.Len(), 0); Dim = 0;
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
        const PFtrExt& FtrExt = FtrExtV[FtrExtN];
        // get dimensionality
		const int FtrExtDim = FtrExt->GetDim();
        // update counts
		DimV.Add(FtrExtDim); Dim += FtrExtDim;
	}   
}
	
TFtrSpace::TFtrSpace(const TWPt<TBase>& _Base, const PFtrExt& FtrExt): 
    Base(_Base), FtrExtV(TFtrExtV::GetV(FtrExt)) { Init(); }

TFtrSpace::TFtrSpace(const TWPt<TBase>& _Base, const TFtrExtV& _FtrExtV): 
    Base(_Base), FtrExtV(_FtrExtV) { Init(); }

TFtrSpace::TFtrSpace(const TWPt<TBase>& _Base, TSIn& SIn): 
        Base(_Base), Dim(SIn), DimV(SIn), VarDimFtrExtNV(SIn) {

    const int FtrExts = TInt(SIn).Val;
    FtrExtV.Gen(FtrExts, 0);
    for (int FtrExtN = 0; FtrExtN < FtrExts; FtrExtN++) {
        PFtrExt FtrExt = TFtrExt::Load(Base, SIn);
        FtrExtV.Add(FtrExt);
    }
}

TPt<TFtrSpace> TFtrSpace::New(const TWPt<TBase>& Base, const PFtrExt& FtrExt) { 
    return new TFtrSpace(Base, FtrExt); 
}

TPt<TFtrSpace> TFtrSpace::New(const TWPt<TBase>& Base, const TFtrExtV& FtrExtV) { 
    return new TFtrSpace(Base, FtrExtV); 
}

TPt<TFtrSpace> TFtrSpace::Load(const TWPt<TBase>& Base, TSIn& SIn) { 
    return new TFtrSpace(Base, SIn); 
}

void TFtrSpace::Save(TSOut& SOut) const {
    Dim.Save(SOut); DimV.Save(SOut); VarDimFtrExtNV.Save(SOut);
    TInt(FtrExtV.Len()).Save(SOut);
    for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
        FtrExtV[FtrExtN]->Save(SOut);
    }
}

void TFtrSpace::AddFtrExt(const PFtrExt& FtrExt) {
	const int FtrExtDim = FtrExt->GetDim();
	DimV.Add(FtrExtDim);
	Dim += FtrExtDim;
	FtrExtV.Add(FtrExt);
}

TStr TFtrSpace::GetNm() const {
	TChA NmChA = "Space:";
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
		NmChA += "\n - ";
        NmChA += FtrExtV[FtrExtN]->GetNm();
	}
	return NmChA;
}

void TFtrSpace::Clr() {
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
		FtrExtV[FtrExtN]->Clr();
	}
}

bool TFtrSpace::Update(const TRec& Rec) {
    bool UpdateDimP = false;
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
        const PFtrExt& FtrExt = FtrExtV[FtrExtN];       
		const bool FtrExtUpdateDimP = FtrExt->Update(Rec);
        if (FtrExtUpdateDimP) {
            // get new dimensionality
            const int NewDim = FtrExt->GetDim();
            // update the dimensionality sum (existing - old + new)
            Dim = Dim - DimV[FtrExtN] + NewDim;
            // update the feature space dimensionality
            DimV[FtrExtN] = NewDim;
            // remember we did this
            UpdateDimP = true;
        }
	}
    return UpdateDimP;
}

bool TFtrSpace::Update(const PRecSet& RecSet) {
    TEnv::Logger->OnStatusFmt("Updating feature space with %d records", RecSet->GetRecs());
    bool UpdateDimP = false;
	for (int RecN = 0; RecN < RecSet->GetRecs(); RecN++) {
		if (RecN % 10000 == 0) { TEnv::Logger->OnStatusFmt("%d\r", RecN); }
        // update according to the record
        const bool RecUpdateDimP = Update(RecSet->GetRec(RecN));
        // check if we did a dimensionality update
		UpdateDimP = UpdateDimP || RecUpdateDimP;
	}
    return UpdateDimP;
}

void TFtrSpace::GetSpV(const TRec& Rec, TIntFltKdV& SpV) const {
	int Offset = 0;
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
		FtrExtV[FtrExtN]->AddSpV(Rec, SpV, Offset);
	}
}

void TFtrSpace::GetFullV(const TRec& Rec, TFltV& FullV) const {
    // create empty full vector
    FullV.Gen(GetDim()); FullV.PutAll(0.0);
	int Offset = 0;
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
		FtrExtV[FtrExtN]->AddFullV(Rec, FullV, Offset);
	}    
}

void TFtrSpace::InvertFullV(const TFltV& FullV, TFltV& InvertV) const {
	int Offset = 0;
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
		const PFtrExt FtrExt = FtrExtV[FtrExtN];

		TFltV InvV;	FtrExt->InvFullV(FullV, Offset, InvV);
		InvertV.AddV(InvV);
	}
}

double TFtrSpace::InvertFtr(const int& FtrExtN, const TFlt& FtrVal) const {
	const PFtrExt FtrExt = FtrExtV[FtrExtN];

	TFltV FtrV, InvV;
	FtrV.Add(FtrVal);

	int Offset = 0;
	FtrExt->InvFullV(FtrV, Offset, InvV);

	return InvV[0];
}

void TFtrSpace::GetSpVV(const PRecSet& RecSet, TVec<TIntFltKdV>& SpVV) const {
    TEnv::Logger->OnStatusFmt("Creating sparse feature vectors from %d records", RecSet->GetRecs());
	for (int RecN = 0; RecN < RecSet->GetRecs(); RecN++) {
		if (RecN % 10000 == 0) { TEnv::Logger->OnStatusFmt("%d\r", RecN); }
		SpVV.Add(TIntFltKdV()); GetSpV(RecSet->GetRec(RecN), SpVV.Last());
	}
}

void TFtrSpace::GetFullVV(const PRecSet& RecSet, TVec<TFltV>& FullVV) const {
    TEnv::Logger->OnStatusFmt("Creating full feature vectors from %d records", RecSet->GetRecs());
	for (int RecN = 0; RecN < RecSet->GetRecs(); RecN++) {
		if (RecN % 10000 == 0) { TEnv::Logger->OnStatusFmt("%d\r", RecN); }
		FullVV.Add(TFltV()); GetFullV(RecSet->GetRec(RecN), FullVV.Last());
	}
}

void TFtrSpace::GetFullVV(const PRecSet& RecSet, TFltVV& FullVV) const {
    TEnv::Logger->OnStatusFmt("Creating full feature vectors from %d records", RecSet->GetRecs());
	FullVV.Gen(GetDim(), RecSet->GetRecs());
	TFltV Temp(GetDim());
	for (int RecN = 0; RecN < RecSet->GetRecs(); RecN++) {
		if (RecN % 10000 == 0) { TEnv::Logger->OnStatusFmt("%d\r", RecN); }
		GetFullV(RecSet->GetRec(RecN), Temp);
		FullVV.SetCol(RecN, Temp);
	}
}
	
void TFtrSpace::GetCentroidSpV(const PRecSet& RecSet, 
		TIntFltKdV& CentroidSpV, const bool& NormalizeP) const {

	for (int RecStrN = 0; RecStrN < RecSet->GetRecs(); RecStrN++) { 
		// get record sparse vector
		TIntFltKdV RecSpV; GetSpV(RecSet->GetRec(RecStrN), RecSpV);
		// add it to the centroid
		TIntFltKdV SumSpV;
		TLinAlg::AddVec(RecSpV, CentroidSpV, SumSpV);
		CentroidSpV = SumSpV;
	}
	if (NormalizeP) { TLinAlg::Normalize(CentroidSpV); }
}

void TFtrSpace::GetCentroidV(const PRecSet& RecSet,
		TFltV& CentroidV, const bool& NormalizeP) const {

	CentroidV.Gen(GetDim()); CentroidV.PutAll(0.0);
	for (int RecStrN = 0; RecStrN < RecSet->GetRecs(); RecStrN++) { 
		// get record sparse vector
		TIntFltKdV RecSpV; GetSpV(RecSet->GetRec(RecStrN), RecSpV);
		// add it to the centroid
		TLinAlg::AddVec(1.0, RecSpV, CentroidV, CentroidV);
	}
	if (NormalizeP) { TLinAlg::Normalize(CentroidV); }
}

int TFtrSpace::GetDim() const {
	return Dim;
}

TStr TFtrSpace::GetFtr(const int& FtrN) const {
	int SumDim = 0;
	for (int DimN = 0; DimN < DimV.Len(); DimN++) {
		SumDim += DimV[DimN];
		if (SumDim > FtrN) { 
			const int LocalFtrN = FtrN - (SumDim - DimV[DimN]);
			return FtrExtV[DimN]->GetFtr(LocalFtrN);
		}
	}
	throw TQmExcept::New("Feature number out of bounds!");
	return TStr();
}

int TFtrSpace::GetFtrExts() const {
    return FtrExtV.Len();
}

PFtrExt TFtrSpace::GetFtrExt(const int& FtrExtN) const {
    QmAssert(0 <= FtrExtN && FtrExtN < FtrExtV.Len());
    return FtrExtV[FtrExtN];    
}

int TFtrSpace::GetFtrExtDim(const int& FtrExtN) const {
    QmAssert(0 <= FtrExtN && FtrExtN < FtrExtV.Len());
    return DimV[FtrExtN];
}

int TFtrSpace::GetMnFtrN(const int& FtrExtN) const {
    QmAssert(0 <= FtrExtN && FtrExtN < FtrExtV.Len());
    int MnFtrN = 0, _FtrExtN = 0;
    while (_FtrExtN < FtrExtN) { 
        MnFtrN += DimV[_FtrExtN];
        _FtrExtN++;
    }
    return MnFtrN;
}

int TFtrSpace::GetMxFtrN(const int& FtrExtN) const {
    QmAssert(0 <= FtrExtN && FtrExtN < FtrExtV.Len());
    int MxFtrN = 0, _FtrExtN = 0;
    while (_FtrExtN <= FtrExtN) { 
        MxFtrN += DimV[_FtrExtN];
        _FtrExtN++;
    }
    return MxFtrN;
}

void TFtrSpace::ExtractStrV(const int& DimN, const PJsonVal& RecVal, TStrV &StrV) const {
    QmAssertR(DimN < DimV.Len(), "Dimension out of bounds!");
    PStore Store = FtrExtV[DimN]->GetFtrStore();
    TRec Rec(Store, RecVal);
    FtrExtV[DimN]->ExtractStrV(Rec, StrV);
}

PBowDocBs TFtrSpace::MakeBowDocBs(const PRecSet& FtrRecSet) {
	// prepare documents
	PBowDocBs BowDocBs = TBowDocBs::New();
    for (int RecN = 0; RecN < FtrRecSet->GetRecs(); RecN++) { 
        TStr DocNm =  FtrRecSet->GetRec(RecN).GetRecNm();
        TStrV WdStrV;
        for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
            FtrExtV[FtrExtN]->ExtractStrV(FtrRecSet->GetRec(RecN), WdStrV);
        }
        BowDocBs->AddDoc(DocNm, TStrV(), WdStrV);
    }
    return BowDocBs;
}

namespace TFtrExts {

///////////////////////////////////////////////
// Constant Feature Extractor
TConstant::TConstant(const TWPt<TBase>& Base, const PJsonVal& ParamVal): 
	TFtrExt(Base, ParamVal), Constant(ParamVal->GetObjNum("const", 1.0)) { }

TConstant:: TConstant(const TWPt<TBase>& Base, TSIn& SIn): TFtrExt(Base, SIn), Constant(SIn) { }

PFtrExt TConstant::New(const TWPt<TBase>& Base, const PJsonVal& ParamVal) {
    return new TConstant(Base, ParamVal); 
}

PFtrExt TConstant::Load(const TWPt<TBase>& Base, TSIn& SIn) {
    return new TConstant(Base, SIn);
}

void TConstant::Save(TSOut& SOut) const {
    GetType().Save(SOut);
    TFtrExt::Save(SOut);
    
    Constant.Save(SOut);
}

void TConstant::AddSpV(const TRec& FtrRec, TIntFltKdV& SpV, int& Offset) const {
	SpV.Add(TIntFltKd(Offset, Constant.Val)); Offset++;
}

void TConstant::AddFullV(const TRec& Rec, TFltV& FullV, int& Offset) const {
    FullV[Offset] = Constant.Val; Offset++;
}

void TConstant::InvFullV(const TFltV& FullV, int& Offset, TFltV& InvV) const {
	InvV.Add(Constant.Val);
	Offset++;
}

void TConstant::ExtractFltV(const TRec& FtrRec, TFltV& FltV) const {
	FltV.Add(Constant.Val);
}
    
///////////////////////////////////////////////
// Random Feature Extractor
TRandom::TRandom(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, const int& RndSeed): 
    TFtrExt(Base, JoinSeqV), Rnd(RndSeed) { }

TRandom::TRandom(const TWPt<TBase>& Base, const PJsonVal& ParamVal): 
  TFtrExt(Base, ParamVal), Rnd(ParamVal->GetObjInt("seed", 0)) { }

TRandom:: TRandom(const TWPt<TBase>& Base, TSIn& SIn): TFtrExt(Base, SIn), Rnd(SIn) { }
    
PFtrExt TRandom::New(const TWPt<TBase>& Base, const TWPt<TStore>& Store, const int& RndSeed) {
	return new TRandom(Base, TJoinSeqV::GetV(TJoinSeq(Store->GetStoreId())), RndSeed); 
}

PFtrExt TRandom::New(const TWPt<TBase>& Base, const TJoinSeq& JoinSeq, const int& RndSeed) { 
	return new TRandom(Base, TJoinSeqV::GetV(JoinSeq), RndSeed); 
}

PFtrExt TRandom::New(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, const int& RndSeed) { 
	return new TRandom(Base, JoinSeqV, RndSeed);
}

PFtrExt TRandom::New(const TWPt<TBase>& Base, const PJsonVal& ParamVal) {
    return new TRandom(Base, ParamVal); 
}

PFtrExt TRandom::Load(const TWPt<TBase>& Base, TSIn& SIn) {
    return new TRandom(Base, SIn);
}

void TRandom::Save(TSOut& SOut) const {
    GetType().Save(SOut);
    TFtrExt::Save(SOut);
    
    Rnd.Save(SOut);
}

void TRandom::AddSpV(const TRec& FtrRec, TIntFltKdV& SpV, int& Offset) const {
	SpV.Add(TIntFltKd(Offset, Rnd.GetUniDev())); Offset++;
}

void TRandom::AddFullV(const TRec& Rec, TFltV& FullV, int& Offset) const {
    FullV[Offset] = Rnd.GetUniDev(); Offset++;
}

void TRandom::InvFullV(const TFltV& FullV, int& Offset, TFltV& InvV) const {
	throw TExcept::New("Not implemented yet!!", "TRandom::InvFullV");
}

void TRandom::ExtractFltV(const TRec& FtrRec, TFltV& FltV) const {
	FltV.Add(Rnd.GetUniDev());
}

///////////////////////////////////////////////
// Numeric Feature Extractor
double TNumeric::_GetVal(const TRec& FtrRec) const {
    // assert store
    Assert(FtrRec.GetStoreId() == GetFtrStore()->GetStoreId());
    // extract feature value
    if (FtrRec.IsFieldNull(FieldId)) {
        return 0.0;
    } else if (FieldDesc.IsInt()) {
        return (double)FtrRec.GetFieldInt(FieldId);
    } else if (FieldDesc.IsFlt()) {
        return FtrRec.GetFieldFlt(FieldId);
    } else if (FieldDesc.IsUInt64()) {
        return (double)FtrRec.GetFieldUInt64(FieldId);
    } else if (FieldDesc.IsBool()) {
        return FtrRec.GetFieldBool(FieldId) ? 1.0 : 0.0;
    }
    throw TQmExcept::New("Field type " + FieldDesc.GetFieldTypeStr() + 
        " not supported by Numeric Feature Extractor!");
}

double TNumeric::GetVal(const TRec& FtrRec) const {
	Assert(IsStartStore(FtrRec.GetStoreId()));
	if (IsJoin(FtrRec.GetStoreId())) {
        // do the join
        TRec JoinRec = FtrRec.DoSingleJoin(GetBase(), GetJoinIdV(FtrRec.GetStoreId()));
        // get feature value
		return _GetVal(JoinRec);
    } else {
        // get feature value
        return _GetVal(FtrRec);
	}
}

TNumeric::TNumeric(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, 
    const int& _FieldId, const bool& NormalizeP): TFtrExt(Base, JoinSeqV), 
        FtrGen(NormalizeP), FieldId(_FieldId), 
        FieldDesc(GetFtrStore()->GetFieldDesc(FieldId)) { Clr(); }

TNumeric::TNumeric(const TWPt<TBase>& Base, const PJsonVal& ParamVal): TFtrExt(Base, ParamVal) {       
    // parse out parameters and initialize feature generator
    if (ParamVal->IsObjKey("min") && ParamVal->IsObjKey("max")) {
        const double MnVal = ParamVal->GetObjNum("min");
        const double MxVal = ParamVal->GetObjNum("max");
        FtrGen = TFtrGen::TNumeric(MnVal, MxVal);
    } else {
        const bool NormalizeP = ParamVal->GetObjBool("normalize", false);
        FtrGen = TFtrGen::TNumeric(NormalizeP);
    }
    // parse out input parameters  
    TStr FieldNm = ParamVal->GetObjStr("field");
    QmAssertR(GetFtrStore()->IsFieldNm(FieldNm), "Unknown field '" + 
        FieldNm + "' in store '" + GetFtrStore()->GetStoreNm() + "'");
    FieldId = GetFtrStore()->GetFieldId(FieldNm);
    FieldDesc = GetFtrStore()->GetFieldDesc(FieldId);
}

TNumeric::TNumeric(const TWPt<TBase>& Base, TSIn& SIn): 
    TFtrExt(Base, SIn), FtrGen(SIn), FieldId(SIn), FieldDesc(SIn) { }

PFtrExt TNumeric::New(const TWPt<TBase>& Base, const TWPt<TStore>& Store, 
        const int& FieldId, const bool& NormalizeP) { 

    return new TNumeric(Base, TJoinSeqV::GetV(TJoinSeq(Store)), FieldId, NormalizeP); 
}

PFtrExt TNumeric::New(const TWPt<TBase>& Base, const TJoinSeq& JoinSeq, 
        const int& FieldId, const bool& NormalizeP) { 
    
    return new TNumeric(Base, TJoinSeqV::GetV(JoinSeq), FieldId, NormalizeP); 
}

PFtrExt TNumeric::New(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, 
        const int& FieldId, const bool& NormalizeP) { 
    
    return new TNumeric(Base, JoinSeqV, FieldId, NormalizeP); 
}

PFtrExt TNumeric::New(const TWPt<TBase>& Base, const PJsonVal& ParamVal) { 
    return new TNumeric(Base, ParamVal);
}

PFtrExt TNumeric::Load(const TWPt<TBase>& Base, TSIn& SIn) {
    return new TNumeric(Base, SIn);
}

void TNumeric::Save(TSOut& SOut) const {
    GetType().Save(SOut);
    TFtrExt::Save(SOut);
    
    FtrGen.Save(SOut);
    FieldId.Save(SOut);
    FieldDesc.Save(SOut);
}

bool TNumeric::Update(const TRec& Rec) {
    FtrGen.Update(GetVal(Rec));
    return false;
}

void TNumeric::AddSpV(const TRec& Rec, TIntFltKdV& SpV, int& Offset) const {
    FtrGen.AddFtr(GetVal(Rec), SpV, Offset);
}

void TNumeric::AddFullV(const TRec& Rec, TFltV& FullV, int& Offset) const {
    FtrGen.AddFtr(GetVal(Rec), FullV, Offset);
}

void TNumeric::InvFullV(const TFltV& FullV, int& Offset, TFltV& InvV) const {
	InvV.Add(FtrGen.InvFtr(FullV, Offset));
}

void TNumeric::ExtractFltV(const TRec& Rec, TFltV& FltV) const {
	FltV.Add(FtrGen.GetFtr(GetVal(Rec)));   
}

///////////////////////////////////////////////
// Sparse Vector Feature Extractor
void TNumSpV::_GetVal(const TRec& FtrRec, TIntFltKdV& NumSpV) const {
    // assert store
    Assert(FtrRec.GetStoreId() == GetFtrStore()->GetStoreId());
    // extract feature value
    if (FtrRec.IsFieldNull(FieldId)) {
        NumSpV.Clr();
    } else if (FieldDesc.IsNumSpV()) {
        FtrRec.GetFieldNumSpV(FieldId, NumSpV);
    }
    throw TQmExcept::New("Field type " + FieldDesc.GetFieldTypeStr() + 
        " not supported by Numeric Feature Extractor!");
}

void TNumSpV::GetVal(const TRec& FtrRec, TIntFltKdV& NumSpV) const {
	Assert(IsStartStore(FtrRec.GetStoreId()));
	if (IsJoin(FtrRec.GetStoreId())) {
        // do the join
        TRec JoinRec = FtrRec.DoSingleJoin(GetBase(), GetJoinIdV(FtrRec.GetStoreId()));
        // get feature value
		return _GetVal(JoinRec);
    } else {
        // get feature value
        return _GetVal(FtrRec);
	}
}

TNumSpV::TNumSpV(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, const int& _FieldId,
    const int& _Dim, const bool& _NormalizeP): TFtrExt(Base, JoinSeqV),
        Dim(_Dim), NormalizeP(_NormalizeP), FieldId(_FieldId),
        FieldDesc(GetFtrStore()->GetFieldDesc(FieldId)) { }

TNumSpV::TNumSpV(const TWPt<TBase>& Base, const PJsonVal& ParamVal): TFtrExt(Base, ParamVal) {       
    // parse out parameters and initialize feature generator
    const bool Dim = ParamVal->GetObjBool("dimension", false);
    const bool NormalizeP = ParamVal->GetObjBool("normalize", false);
    // parse out input parameters
    TStr FieldNm = ParamVal->GetObjStr("field");
    QmAssertR(GetFtrStore()->IsFieldNm(FieldNm), "Unknown field '" + 
        FieldNm + "' in store '" + GetFtrStore()->GetStoreNm() + "'");
    FieldId = GetFtrStore()->GetFieldId(FieldNm);
    FieldDesc = GetFtrStore()->GetFieldDesc(FieldId);
}

TNumSpV::TNumSpV(const TWPt<TBase>& Base, TSIn& SIn):
    TFtrExt(Base, SIn), Dim(SIn), NormalizeP(SIn), FieldId(SIn), FieldDesc(SIn) { }

PFtrExt TNumSpV::New(const TWPt<TBase>& Base, const TWPt<TStore>& Store, 
        const int& FieldId, const int& Dim, const bool& NormalizeP) {

    return new TNumSpV(Base, TJoinSeqV::GetV(TJoinSeq(Store)), FieldId, Dim, NormalizeP);
}

PFtrExt TNumSpV::New(const TWPt<TBase>& Base, const TJoinSeq& JoinSeq, 
        const int& FieldId, const int& Dim, const bool& NormalizeP) {
    
    return new TNumSpV(Base, TJoinSeqV::GetV(JoinSeq), FieldId, Dim, NormalizeP);
}

PFtrExt TNumSpV::New(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, 
        const int& FieldId, const int& Dim, const bool& NormalizeP) {
    
    return new TNumSpV(Base, JoinSeqV, FieldId, Dim, NormalizeP);
}

PFtrExt TNumSpV::New(const TWPt<TBase>& Base, const PJsonVal& ParamVal) { 
    return new TNumSpV(Base, ParamVal);
}

PFtrExt TNumSpV::Load(const TWPt<TBase>& Base, TSIn& SIn) {
    return new TNumSpV(Base, SIn);
}

void TNumSpV::Save(TSOut& SOut) const {
    GetType().Save(SOut);
    TFtrExt::Save(SOut);
    
    Dim.Save(SOut);
    NormalizeP.Save(SOut);
    FieldId.Save(SOut);
    FieldDesc.Save(SOut);
}

TStr TNumSpV::GetNm() const {
    return "SpVec[" + GetFtrStore()->GetFieldNm(FieldId) + "]";
};

TStr TNumSpV::GetFtr(const int& FtrN) const {
    return TStr::Fmt("SpVec[%s:%d/%d]", GetFtrStore()->GetFieldNm(FieldId).CStr(), FtrN, Dim.Val);
}
    
bool TNumSpV::Update(const TRec& Rec) {
    // we only need to update dimensionality, if new record makes it out-of-bounds
    TIntFltKdV NumSpV; GetVal(Rec, NumSpV);
    const int NewDim = TLAMisc::GetMaxDimIdx(NumSpV);
    if (NewDim > Dim) { Dim = NewDin; return true; }
    return false;
}

void TNumSpV::AddSpV(const TRec& Rec, TIntFltKdV& SpV, int& Offset) const {
    TIntFltKdV NumSpV; GetVal(Rec, NumSpV);
    for (int NumSpN = 0; NumSpN < NumSpV.Len(); NumSpN++) {
        const TIntFltKd& NumSp = NumSpV[NumspN];
        SpV.Add(TIntFldKdV(Offset + NumSp.Key, NumSp.Dat));
    }
    Offset += GetDim();
}

void TNumSpV::AddFullV(const TRec& Rec, TFltV& FullV, int& Offset) const {
    TIntFltKdV NumSpV; GetVal(Rec, NumSpV);
    for (int NumSpN = 0; NumSpN < NumSpV.Len(); NumSpN++) {
        const TIntFltKd& NumSp = NumSpV[NumspN];
        FullV[Offset + NumSp.Key] = NumSp.Dat;
    }
    Offset += GetDim();
}

void TNumSpV::InvFullV(const TFltV& FullV, int& Offset, TFltV& InvV) const {
	throw TExcept::New("Not implemented yet!!", "TCategorical::InvFullV");
}

///////////////////////////////////////////////
// Categorical Feature Extractor
TStr TCategorical::_GetVal(const TRec& FtrRec) const {
	// assert store
	Assert(FtrRec.GetStoreId() == GetFtrStore()->GetStoreId());<commit_msg>Fixed compile errors that were not commited last time for some strange reason...<commit_after>/**
 * Copyright (c) 2015, <NAME>, Quintelligence d.o.o. and contributors
 * All rights reserved.
 * 
 * This source code is licensed under the FreeBSD license found in the
 * LICENSE file in the root directory of this source tree.
 */

#include "qminer_ftr.h"

namespace TQm {

///////////////////////////////////////////////
// QMiner-Feature-Extractor
TFunRouter<PFtrExt, TFtrExt::TNewF> TFtrExt::NewRouter;
TFunRouter<PFtrExt, TFtrExt::TLoadF> TFtrExt::LoadRouter;

void TFtrExt::Init() {
    Register<TFtrExts::TRandom>();
	Register<TFtrExts::TConstant>();
    Register<TFtrExts::TNumeric>();
    Register<TFtrExts::TNumSpV>();
    Register<TFtrExts::TCategorical>();
    Register<TFtrExts::TMultinomial>();
    Register<TFtrExts::TBagOfWords>();
    Register<TFtrExts::TJoin>();
    Register<TFtrExts::TPair>();
    Register<TFtrExts::TDateWnd>();
}
    
TRec TFtrExt::DoSingleJoin(const TRec& Rec) const {
	QmAssertR(IsStartStore(Rec.GetStoreId()), "Start store not supported!");
	return Rec.DoSingleJoin(Base, GetJoinIdV(Rec.GetStoreId()));
}

TFtrExt::TFtrExt(const TWPt<TBase>& _Base, const TJoinSeqV& JoinSeqV): Base(_Base) {
	QmAssertR(!JoinSeqV.Empty(), "At least one join sequence must be supplied!");
	FtrStore = JoinSeqV[0].GetEndStore(Base);
    // get mapping from stores to join sequence
	for (int JoinSeqN = 0; JoinSeqN < JoinSeqV.Len(); JoinSeqN++) {
		const TJoinSeq& JoinSeq = JoinSeqV[JoinSeqN];
        QmAssertR(JoinSeq.GetEndStoreId(Base) == FtrStore->GetStoreId(), "Mismatching feature stores");
		JoinSeqH.AddDat(JoinSeq.GetStartStoreId(), JoinSeq);
	}
}

TFtrExt::TFtrExt(const TWPt<TBase>& _Base, const PJsonVal& ParamVal): Base(_Base) {
    QmAssert(ParamVal->IsObjKey("source"));
    PJsonVal SourceVal = ParamVal->GetObjKey("source");
    if (SourceVal->IsStr()) {
        // we have just store name
        TStr StoreNm = SourceVal->GetStr();
        FtrStore = Base->GetStoreByStoreNm(StoreNm);
        JoinSeqH.AddDat(FtrStore->GetStoreId(), TJoinSeq(FtrStore->GetStoreId()));
    } else if (SourceVal->IsObj()) {
        // get store
        TStr StoreNm = SourceVal->GetObjStr("store");
        FtrStore = Base->GetStoreByStoreNm(StoreNm);       
        // get joins if any given
        if (SourceVal->IsObjKey("join")) {
            TJoinSeq JoinSeq = TJoinSeq(Base, FtrStore->GetStoreId(), SourceVal->GetObjKey("join"));
            FtrStore = JoinSeq.GetEndStore(Base);
            JoinSeqH.AddDat(JoinSeq.GetStartStoreId(), JoinSeq);
        } else {
            JoinSeqH.AddDat(FtrStore->GetStoreId(), TJoinSeq(FtrStore->GetStoreId()));            
        }
    } else if (SourceVal->IsArr()) {
        // parse join sequence
        for (int SourceN = 0; SourceN < SourceVal->GetArrVals(); SourceN++) {
            PJsonVal JoinVal = SourceVal->GetArrVal(SourceN);
            // get store
            TStr StoreNm = JoinVal->GetObjStr("store");
            TWPt<TStore> Store = Base->GetStoreByStoreNm(StoreNm);       
            // get joins if any give
            if (JoinVal->IsObjKey("join")) {                
                TJoinSeq JoinSeq = TJoinSeq(Base, Store->GetStoreId(), JoinVal->GetObjKey("join"));
                Store = JoinSeq.GetEndStore(Base);
        		JoinSeqH.AddDat(JoinSeq.GetStartStoreId(), JoinSeq);
            } else {
                // no join, just remember the store
                JoinSeqH.AddDat(Store->GetStoreId(), TJoinSeq(Store->GetStoreId()));
            }
            // update feature store or assert still consistent
            if (SourceN == 0) { FtrStore = Store; } 
            QmAssertR(FtrStore->GetStoreId() == Store->GetStoreId(), "Mismatch of feature stores");
        }
    }
}

TFtrExt::TFtrExt(const TWPt<TBase>& _Base, TSIn& SIn): Base(_Base) {
    JoinSeqH.Load(SIn);
    FtrStore = TStore::LoadById(Base, SIn);
}

PFtrExt TFtrExt::New(const TWPt<TBase>& Base, const TStr& TypeNm, const PJsonVal& ParamVal) {
    return NewRouter.Fun(TypeNm)(Base, ParamVal);
}

PFtrExt TFtrExt::Load(const TWPt<TBase>& Base, TSIn& SIn) {
	TStr TypeNm(SIn);
    return LoadRouter.Fun(TypeNm)(Base, SIn);
}

void TFtrExt::Save(TSOut& SOut) const { 
    JoinSeqH.Save(SOut); 
    FtrStore->SaveId(SOut);
}

void TFtrExt::AddFullV(const TRec& Rec, TFltV& FullV, int& Offset) const {
    // get sparse vector
    TIntFltKdV SpV; AddSpV(Rec, SpV, Offset);
    // add to full vector
    for (int SpN = 0; SpN < SpV.Len(); SpN++) {
        FullV[SpV[SpN].Key] = SpV[SpN].Dat; }
}

void TFtrExt::ExtractStrV(const TRec& FtrRec, TStrV& StrV) const { 
	throw TQmExcept::New("ExtractStrV not implemented!"); 
}

void TFtrExt::ExtractFltV(const TRec& FtrRec, TFltV& FltV) const {
	throw TQmExcept::New("ExtractFltV not implemented!"); 
}

void TFtrExt::ExtractTmV(const TRec& FtrRec, TTmV& TmV) const {
	throw TQmExcept::New("ExtractTmV not implemented!"); 
}

///////////////////////////////////////////////
// QMiner-Feature-Space
void TFtrSpace::Init() {
    // we start with empty space
	DimV.Gen(FtrExtV.Len(), 0); Dim = 0;
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
        const PFtrExt& FtrExt = FtrExtV[FtrExtN];
        // get dimensionality
		const int FtrExtDim = FtrExt->GetDim();
        // update counts
		DimV.Add(FtrExtDim); Dim += FtrExtDim;
	}   
}
	
TFtrSpace::TFtrSpace(const TWPt<TBase>& _Base, const PFtrExt& FtrExt): 
    Base(_Base), FtrExtV(TFtrExtV::GetV(FtrExt)) { Init(); }

TFtrSpace::TFtrSpace(const TWPt<TBase>& _Base, const TFtrExtV& _FtrExtV): 
    Base(_Base), FtrExtV(_FtrExtV) { Init(); }

TFtrSpace::TFtrSpace(const TWPt<TBase>& _Base, TSIn& SIn): 
        Base(_Base), Dim(SIn), DimV(SIn), VarDimFtrExtNV(SIn) {

    const int FtrExts = TInt(SIn).Val;
    FtrExtV.Gen(FtrExts, 0);
    for (int FtrExtN = 0; FtrExtN < FtrExts; FtrExtN++) {
        PFtrExt FtrExt = TFtrExt::Load(Base, SIn);
        FtrExtV.Add(FtrExt);
    }
}

TPt<TFtrSpace> TFtrSpace::New(const TWPt<TBase>& Base, const PFtrExt& FtrExt) { 
    return new TFtrSpace(Base, FtrExt); 
}

TPt<TFtrSpace> TFtrSpace::New(const TWPt<TBase>& Base, const TFtrExtV& FtrExtV) { 
    return new TFtrSpace(Base, FtrExtV); 
}

TPt<TFtrSpace> TFtrSpace::Load(const TWPt<TBase>& Base, TSIn& SIn) { 
    return new TFtrSpace(Base, SIn); 
}

void TFtrSpace::Save(TSOut& SOut) const {
    Dim.Save(SOut); DimV.Save(SOut); VarDimFtrExtNV.Save(SOut);
    TInt(FtrExtV.Len()).Save(SOut);
    for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
        FtrExtV[FtrExtN]->Save(SOut);
    }
}

void TFtrSpace::AddFtrExt(const PFtrExt& FtrExt) {
	const int FtrExtDim = FtrExt->GetDim();
	DimV.Add(FtrExtDim);
	Dim += FtrExtDim;
	FtrExtV.Add(FtrExt);
}

TStr TFtrSpace::GetNm() const {
	TChA NmChA = "Space:";
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
		NmChA += "\n - ";
        NmChA += FtrExtV[FtrExtN]->GetNm();
	}
	return NmChA;
}

void TFtrSpace::Clr() {
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
		FtrExtV[FtrExtN]->Clr();
	}
}

bool TFtrSpace::Update(const TRec& Rec) {
    bool UpdateDimP = false;
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
        const PFtrExt& FtrExt = FtrExtV[FtrExtN];       
		const bool FtrExtUpdateDimP = FtrExt->Update(Rec);
        if (FtrExtUpdateDimP) {
            // get new dimensionality
            const int NewDim = FtrExt->GetDim();
            // update the dimensionality sum (existing - old + new)
            Dim = Dim - DimV[FtrExtN] + NewDim;
            // update the feature space dimensionality
            DimV[FtrExtN] = NewDim;
            // remember we did this
            UpdateDimP = true;
        }
	}
    return UpdateDimP;
}

bool TFtrSpace::Update(const PRecSet& RecSet) {
    TEnv::Logger->OnStatusFmt("Updating feature space with %d records", RecSet->GetRecs());
    bool UpdateDimP = false;
	for (int RecN = 0; RecN < RecSet->GetRecs(); RecN++) {
		if (RecN % 10000 == 0) { TEnv::Logger->OnStatusFmt("%d\r", RecN); }
        // update according to the record
        const bool RecUpdateDimP = Update(RecSet->GetRec(RecN));
        // check if we did a dimensionality update
		UpdateDimP = UpdateDimP || RecUpdateDimP;
	}
    return UpdateDimP;
}

void TFtrSpace::GetSpV(const TRec& Rec, TIntFltKdV& SpV) const {
	int Offset = 0;
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
		FtrExtV[FtrExtN]->AddSpV(Rec, SpV, Offset);
	}
}

void TFtrSpace::GetFullV(const TRec& Rec, TFltV& FullV) const {
    // create empty full vector
    FullV.Gen(GetDim()); FullV.PutAll(0.0);
	int Offset = 0;
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
		FtrExtV[FtrExtN]->AddFullV(Rec, FullV, Offset);
	}    
}

void TFtrSpace::InvertFullV(const TFltV& FullV, TFltV& InvertV) const {
	int Offset = 0;
	for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
		const PFtrExt FtrExt = FtrExtV[FtrExtN];

		TFltV InvV;	FtrExt->InvFullV(FullV, Offset, InvV);
		InvertV.AddV(InvV);
	}
}

double TFtrSpace::InvertFtr(const int& FtrExtN, const TFlt& FtrVal) const {
	const PFtrExt FtrExt = FtrExtV[FtrExtN];

	TFltV FtrV, InvV;
	FtrV.Add(FtrVal);

	int Offset = 0;
	FtrExt->InvFullV(FtrV, Offset, InvV);

	return InvV[0];
}

void TFtrSpace::GetSpVV(const PRecSet& RecSet, TVec<TIntFltKdV>& SpVV) const {
    TEnv::Logger->OnStatusFmt("Creating sparse feature vectors from %d records", RecSet->GetRecs());
	for (int RecN = 0; RecN < RecSet->GetRecs(); RecN++) {
		if (RecN % 10000 == 0) { TEnv::Logger->OnStatusFmt("%d\r", RecN); }
		SpVV.Add(TIntFltKdV()); GetSpV(RecSet->GetRec(RecN), SpVV.Last());
	}
}

void TFtrSpace::GetFullVV(const PRecSet& RecSet, TVec<TFltV>& FullVV) const {
    TEnv::Logger->OnStatusFmt("Creating full feature vectors from %d records", RecSet->GetRecs());
	for (int RecN = 0; RecN < RecSet->GetRecs(); RecN++) {
		if (RecN % 10000 == 0) { TEnv::Logger->OnStatusFmt("%d\r", RecN); }
		FullVV.Add(TFltV()); GetFullV(RecSet->GetRec(RecN), FullVV.Last());
	}
}

void TFtrSpace::GetFullVV(const PRecSet& RecSet, TFltVV& FullVV) const {
    TEnv::Logger->OnStatusFmt("Creating full feature vectors from %d records", RecSet->GetRecs());
	FullVV.Gen(GetDim(), RecSet->GetRecs());
	TFltV Temp(GetDim());
	for (int RecN = 0; RecN < RecSet->GetRecs(); RecN++) {
		if (RecN % 10000 == 0) { TEnv::Logger->OnStatusFmt("%d\r", RecN); }
		GetFullV(RecSet->GetRec(RecN), Temp);
		FullVV.SetCol(RecN, Temp);
	}
}
	
void TFtrSpace::GetCentroidSpV(const PRecSet& RecSet, 
		TIntFltKdV& CentroidSpV, const bool& NormalizeP) const {

	for (int RecStrN = 0; RecStrN < RecSet->GetRecs(); RecStrN++) { 
		// get record sparse vector
		TIntFltKdV RecSpV; GetSpV(RecSet->GetRec(RecStrN), RecSpV);
		// add it to the centroid
		TIntFltKdV SumSpV;
		TLinAlg::AddVec(RecSpV, CentroidSpV, SumSpV);
		CentroidSpV = SumSpV;
	}
	if (NormalizeP) { TLinAlg::Normalize(CentroidSpV); }
}

void TFtrSpace::GetCentroidV(const PRecSet& RecSet,
		TFltV& CentroidV, const bool& NormalizeP) const {

	CentroidV.Gen(GetDim()); CentroidV.PutAll(0.0);
	for (int RecStrN = 0; RecStrN < RecSet->GetRecs(); RecStrN++) { 
		// get record sparse vector
		TIntFltKdV RecSpV; GetSpV(RecSet->GetRec(RecStrN), RecSpV);
		// add it to the centroid
		TLinAlg::AddVec(1.0, RecSpV, CentroidV, CentroidV);
	}
	if (NormalizeP) { TLinAlg::Normalize(CentroidV); }
}

int TFtrSpace::GetDim() const {
	return Dim;
}

TStr TFtrSpace::GetFtr(const int& FtrN) const {
	int SumDim = 0;
	for (int DimN = 0; DimN < DimV.Len(); DimN++) {
		SumDim += DimV[DimN];
		if (SumDim > FtrN) { 
			const int LocalFtrN = FtrN - (SumDim - DimV[DimN]);
			return FtrExtV[DimN]->GetFtr(LocalFtrN);
		}
	}
	throw TQmExcept::New("Feature number out of bounds!");
	return TStr();
}

int TFtrSpace::GetFtrExts() const {
    return FtrExtV.Len();
}

PFtrExt TFtrSpace::GetFtrExt(const int& FtrExtN) const {
    QmAssert(0 <= FtrExtN && FtrExtN < FtrExtV.Len());
    return FtrExtV[FtrExtN];    
}

int TFtrSpace::GetFtrExtDim(const int& FtrExtN) const {
    QmAssert(0 <= FtrExtN && FtrExtN < FtrExtV.Len());
    return DimV[FtrExtN];
}

int TFtrSpace::GetMnFtrN(const int& FtrExtN) const {
    QmAssert(0 <= FtrExtN && FtrExtN < FtrExtV.Len());
    int MnFtrN = 0, _FtrExtN = 0;
    while (_FtrExtN < FtrExtN) { 
        MnFtrN += DimV[_FtrExtN];
        _FtrExtN++;
    }
    return MnFtrN;
}

int TFtrSpace::GetMxFtrN(const int& FtrExtN) const {
    QmAssert(0 <= FtrExtN && FtrExtN < FtrExtV.Len());
    int MxFtrN = 0, _FtrExtN = 0;
    while (_FtrExtN <= FtrExtN) { 
        MxFtrN += DimV[_FtrExtN];
        _FtrExtN++;
    }
    return MxFtrN;
}

void TFtrSpace::ExtractStrV(const int& DimN, const PJsonVal& RecVal, TStrV &StrV) const {
    QmAssertR(DimN < DimV.Len(), "Dimension out of bounds!");
    PStore Store = FtrExtV[DimN]->GetFtrStore();
    TRec Rec(Store, RecVal);
    FtrExtV[DimN]->ExtractStrV(Rec, StrV);
}

PBowDocBs TFtrSpace::MakeBowDocBs(const PRecSet& FtrRecSet) {
	// prepare documents
	PBowDocBs BowDocBs = TBowDocBs::New();
    for (int RecN = 0; RecN < FtrRecSet->GetRecs(); RecN++) { 
        TStr DocNm =  FtrRecSet->GetRec(RecN).GetRecNm();
        TStrV WdStrV;
        for (int FtrExtN = 0; FtrExtN < FtrExtV.Len(); FtrExtN++) {
            FtrExtV[FtrExtN]->ExtractStrV(FtrRecSet->GetRec(RecN), WdStrV);
        }
        BowDocBs->AddDoc(DocNm, TStrV(), WdStrV);
    }
    return BowDocBs;
}

namespace TFtrExts {

///////////////////////////////////////////////
// Constant Feature Extractor
TConstant::TConstant(const TWPt<TBase>& Base, const PJsonVal& ParamVal): 
	TFtrExt(Base, ParamVal), Constant(ParamVal->GetObjNum("const", 1.0)) { }

TConstant:: TConstant(const TWPt<TBase>& Base, TSIn& SIn): TFtrExt(Base, SIn), Constant(SIn) { }

PFtrExt TConstant::New(const TWPt<TBase>& Base, const PJsonVal& ParamVal) {
    return new TConstant(Base, ParamVal); 
}

PFtrExt TConstant::Load(const TWPt<TBase>& Base, TSIn& SIn) {
    return new TConstant(Base, SIn);
}

void TConstant::Save(TSOut& SOut) const {
    GetType().Save(SOut);
    TFtrExt::Save(SOut);
    
    Constant.Save(SOut);
}

void TConstant::AddSpV(const TRec& FtrRec, TIntFltKdV& SpV, int& Offset) const {
	SpV.Add(TIntFltKd(Offset, Constant.Val)); Offset++;
}

void TConstant::AddFullV(const TRec& Rec, TFltV& FullV, int& Offset) const {
    FullV[Offset] = Constant.Val; Offset++;
}

void TConstant::InvFullV(const TFltV& FullV, int& Offset, TFltV& InvV) const {
	InvV.Add(Constant.Val);
	Offset++;
}

void TConstant::ExtractFltV(const TRec& FtrRec, TFltV& FltV) const {
	FltV.Add(Constant.Val);
}
    
///////////////////////////////////////////////
// Random Feature Extractor
TRandom::TRandom(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, const int& RndSeed): 
    TFtrExt(Base, JoinSeqV), Rnd(RndSeed) { }

TRandom::TRandom(const TWPt<TBase>& Base, const PJsonVal& ParamVal): 
  TFtrExt(Base, ParamVal), Rnd(ParamVal->GetObjInt("seed", 0)) { }

TRandom:: TRandom(const TWPt<TBase>& Base, TSIn& SIn): TFtrExt(Base, SIn), Rnd(SIn) { }
    
PFtrExt TRandom::New(const TWPt<TBase>& Base, const TWPt<TStore>& Store, const int& RndSeed) {
	return new TRandom(Base, TJoinSeqV::GetV(TJoinSeq(Store->GetStoreId())), RndSeed); 
}

PFtrExt TRandom::New(const TWPt<TBase>& Base, const TJoinSeq& JoinSeq, const int& RndSeed) { 
	return new TRandom(Base, TJoinSeqV::GetV(JoinSeq), RndSeed); 
}

PFtrExt TRandom::New(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, const int& RndSeed) { 
	return new TRandom(Base, JoinSeqV, RndSeed);
}

PFtrExt TRandom::New(const TWPt<TBase>& Base, const PJsonVal& ParamVal) {
    return new TRandom(Base, ParamVal); 
}

PFtrExt TRandom::Load(const TWPt<TBase>& Base, TSIn& SIn) {
    return new TRandom(Base, SIn);
}

void TRandom::Save(TSOut& SOut) const {
    GetType().Save(SOut);
    TFtrExt::Save(SOut);
    
    Rnd.Save(SOut);
}

void TRandom::AddSpV(const TRec& FtrRec, TIntFltKdV& SpV, int& Offset) const {
	SpV.Add(TIntFltKd(Offset, Rnd.GetUniDev())); Offset++;
}

void TRandom::AddFullV(const TRec& Rec, TFltV& FullV, int& Offset) const {
    FullV[Offset] = Rnd.GetUniDev(); Offset++;
}

void TRandom::InvFullV(const TFltV& FullV, int& Offset, TFltV& InvV) const {
	throw TExcept::New("Not implemented yet!!", "TRandom::InvFullV");
}

void TRandom::ExtractFltV(const TRec& FtrRec, TFltV& FltV) const {
	FltV.Add(Rnd.GetUniDev());
}

///////////////////////////////////////////////
// Numeric Feature Extractor
double TNumeric::_GetVal(const TRec& FtrRec) const {
    // assert store
    Assert(FtrRec.GetStoreId() == GetFtrStore()->GetStoreId());
    // extract feature value
    if (FtrRec.IsFieldNull(FieldId)) {
        return 0.0;
    } else if (FieldDesc.IsInt()) {
        return (double)FtrRec.GetFieldInt(FieldId);
    } else if (FieldDesc.IsFlt()) {
        return FtrRec.GetFieldFlt(FieldId);
    } else if (FieldDesc.IsUInt64()) {
        return (double)FtrRec.GetFieldUInt64(FieldId);
    } else if (FieldDesc.IsBool()) {
        return FtrRec.GetFieldBool(FieldId) ? 1.0 : 0.0;
    }
    throw TQmExcept::New("Field type " + FieldDesc.GetFieldTypeStr() + 
        " not supported by Numeric Feature Extractor!");
}

double TNumeric::GetVal(const TRec& FtrRec) const {
	Assert(IsStartStore(FtrRec.GetStoreId()));
	if (IsJoin(FtrRec.GetStoreId())) {
        // do the join
        TRec JoinRec = FtrRec.DoSingleJoin(GetBase(), GetJoinIdV(FtrRec.GetStoreId()));
        // get feature value
		return _GetVal(JoinRec);
    } else {
        // get feature value
        return _GetVal(FtrRec);
	}
}

TNumeric::TNumeric(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, 
    const int& _FieldId, const bool& NormalizeP): TFtrExt(Base, JoinSeqV), 
        FtrGen(NormalizeP), FieldId(_FieldId), 
        FieldDesc(GetFtrStore()->GetFieldDesc(FieldId)) { Clr(); }

TNumeric::TNumeric(const TWPt<TBase>& Base, const PJsonVal& ParamVal): TFtrExt(Base, ParamVal) {       
    // parse out parameters and initialize feature generator
    if (ParamVal->IsObjKey("min") && ParamVal->IsObjKey("max")) {
        const double MnVal = ParamVal->GetObjNum("min");
        const double MxVal = ParamVal->GetObjNum("max");
        FtrGen = TFtrGen::TNumeric(MnVal, MxVal);
    } else {
        const bool NormalizeP = ParamVal->GetObjBool("normalize", false);
        FtrGen = TFtrGen::TNumeric(NormalizeP);
    }
    // parse out input parameters  
    TStr FieldNm = ParamVal->GetObjStr("field");
    QmAssertR(GetFtrStore()->IsFieldNm(FieldNm), "Unknown field '" + 
        FieldNm + "' in store '" + GetFtrStore()->GetStoreNm() + "'");
    FieldId = GetFtrStore()->GetFieldId(FieldNm);
    FieldDesc = GetFtrStore()->GetFieldDesc(FieldId);
}

TNumeric::TNumeric(const TWPt<TBase>& Base, TSIn& SIn): 
    TFtrExt(Base, SIn), FtrGen(SIn), FieldId(SIn), FieldDesc(SIn) { }

PFtrExt TNumeric::New(const TWPt<TBase>& Base, const TWPt<TStore>& Store, 
        const int& FieldId, const bool& NormalizeP) { 

    return new TNumeric(Base, TJoinSeqV::GetV(TJoinSeq(Store)), FieldId, NormalizeP); 
}

PFtrExt TNumeric::New(const TWPt<TBase>& Base, const TJoinSeq& JoinSeq, 
        const int& FieldId, const bool& NormalizeP) { 
    
    return new TNumeric(Base, TJoinSeqV::GetV(JoinSeq), FieldId, NormalizeP); 
}

PFtrExt TNumeric::New(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, 
        const int& FieldId, const bool& NormalizeP) { 
    
    return new TNumeric(Base, JoinSeqV, FieldId, NormalizeP); 
}

PFtrExt TNumeric::New(const TWPt<TBase>& Base, const PJsonVal& ParamVal) { 
    return new TNumeric(Base, ParamVal);
}

PFtrExt TNumeric::Load(const TWPt<TBase>& Base, TSIn& SIn) {
    return new TNumeric(Base, SIn);
}

void TNumeric::Save(TSOut& SOut) const {
    GetType().Save(SOut);
    TFtrExt::Save(SOut);
    
    FtrGen.Save(SOut);
    FieldId.Save(SOut);
    FieldDesc.Save(SOut);
}

bool TNumeric::Update(const TRec& Rec) {
    FtrGen.Update(GetVal(Rec));
    return false;
}

void TNumeric::AddSpV(const TRec& Rec, TIntFltKdV& SpV, int& Offset) const {
    FtrGen.AddFtr(GetVal(Rec), SpV, Offset);
}

void TNumeric::AddFullV(const TRec& Rec, TFltV& FullV, int& Offset) const {
    FtrGen.AddFtr(GetVal(Rec), FullV, Offset);
}

void TNumeric::InvFullV(const TFltV& FullV, int& Offset, TFltV& InvV) const {
	InvV.Add(FtrGen.InvFtr(FullV, Offset));
}

void TNumeric::ExtractFltV(const TRec& Rec, TFltV& FltV) const {
	FltV.Add(FtrGen.GetFtr(GetVal(Rec)));   
}

///////////////////////////////////////////////
// Sparse Vector Feature Extractor
void TNumSpV::_GetVal(const TRec& FtrRec, TIntFltKdV& NumSpV) const {
    // assert store
    Assert(FtrRec.GetStoreId() == GetFtrStore()->GetStoreId());
    // extract feature value
    if (FtrRec.IsFieldNull(FieldId)) {
        NumSpV.Clr();
    } else if (FieldDesc.IsNumSpV()) {
        FtrRec.GetFieldNumSpV(FieldId, NumSpV);
    }
    throw TQmExcept::New("Field type " + FieldDesc.GetFieldTypeStr() + 
        " not supported by Numeric Feature Extractor!");
}

void TNumSpV::GetVal(const TRec& FtrRec, TIntFltKdV& NumSpV) const {
	Assert(IsStartStore(FtrRec.GetStoreId()));
	if (IsJoin(FtrRec.GetStoreId())) {
        // do the join
        TRec JoinRec = FtrRec.DoSingleJoin(GetBase(), GetJoinIdV(FtrRec.GetStoreId()));
        // get feature value
		return _GetVal(JoinRec, NumSpV);
    } else {
        // get feature value
        return _GetVal(FtrRec, NumSpV);
	}
}

TNumSpV::TNumSpV(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, const int& _FieldId,
    const int& _Dim, const bool& _NormalizeP): TFtrExt(Base, JoinSeqV),
        Dim(_Dim), NormalizeP(_NormalizeP), FieldId(_FieldId),
        FieldDesc(GetFtrStore()->GetFieldDesc(FieldId)) { }

TNumSpV::TNumSpV(const TWPt<TBase>& Base, const PJsonVal& ParamVal): TFtrExt(Base, ParamVal) {       
    // parse out parameters and initialize feature generator
    Dim = ParamVal->GetObjInt("dimension", 0);
    NormalizeP = ParamVal->GetObjBool("normalize", false);
    // parse out input parameters
    TStr FieldNm = ParamVal->GetObjStr("field");
    QmAssertR(GetFtrStore()->IsFieldNm(FieldNm), "Unknown field '" + 
        FieldNm + "' in store '" + GetFtrStore()->GetStoreNm() + "'");
    FieldId = GetFtrStore()->GetFieldId(FieldNm);
    FieldDesc = GetFtrStore()->GetFieldDesc(FieldId);
}

TNumSpV::TNumSpV(const TWPt<TBase>& Base, TSIn& SIn):
    TFtrExt(Base, SIn), Dim(SIn), NormalizeP(SIn), FieldId(SIn), FieldDesc(SIn) { }

PFtrExt TNumSpV::New(const TWPt<TBase>& Base, const TWPt<TStore>& Store, 
        const int& FieldId, const int& Dim, const bool& NormalizeP) {

    return new TNumSpV(Base, TJoinSeqV::GetV(TJoinSeq(Store)), FieldId, Dim, NormalizeP);
}

PFtrExt TNumSpV::New(const TWPt<TBase>& Base, const TJoinSeq& JoinSeq, 
        const int& FieldId, const int& Dim, const bool& NormalizeP) {
    
    return new TNumSpV(Base, TJoinSeqV::GetV(JoinSeq), FieldId, Dim, NormalizeP);
}

PFtrExt TNumSpV::New(const TWPt<TBase>& Base, const TJoinSeqV& JoinSeqV, 
        const int& FieldId, const int& Dim, const bool& NormalizeP) {
    
    return new TNumSpV(Base, JoinSeqV, FieldId, Dim, NormalizeP);
}

PFtrExt TNumSpV::New(const TWPt<TBase>& Base, const PJsonVal& ParamVal) { 
    return new TNumSpV(Base, ParamVal);
}

PFtrExt TNumSpV::Load(const TWPt<TBase>& Base, TSIn& SIn) {
    return new TNumSpV(Base, SIn);
}

void TNumSpV::Save(TSOut& SOut) const {
    GetType().Save(SOut);
    TFtrExt::Save(SOut);
    
    Dim.Save(SOut);
    NormalizeP.Save(SOut);
    FieldId.Save(SOut);
    FieldDesc.Save(SOut);
}

TStr TNumSpV::GetNm() const {
    return "SpVec[" + GetFtrStore()->GetFieldNm(FieldId) + "]";
};

TStr TNumSpV::GetFtr(const int& FtrN) const {
    return TStr::Fmt("SpVec[%s:%d/%d]", GetFtrStore()->GetFieldNm(FieldId).CStr(), FtrN, Dim.Val);
}
    
bool TNumSpV::Update(const TRec& Rec) {
    // we only need to update dimensionality, if new record makes it out-of-bounds
    TIntFltKdV NumSpV; GetVal(Rec, NumSpV);
    const int NewDim = TLAMisc::GetMaxDimIdx(NumSpV);
    if (NewDim > Dim) { Dim = NewDim; return true; }
    return false;
}

void TNumSpV::AddSpV(const TRec& Rec, TIntFltKdV& SpV, int& Offset) const {
    TIntFltKdV NumSpV; GetVal(Rec, NumSpV);
    if (NormalizeP) { TLinAlg::Normalize(NumSpV); }
    for (int NumSpN = 0; NumSpN < NumSpV.Len(); NumSpN++) {
        const TIntFltKd& NumSp = NumSpV[NumSpN];
        SpV.Add(TIntFltKd(Offset + NumSp.Key, NumSp.Dat));
    }
    Offset += GetDim();
}

void TNumSpV::AddFullV(const TRec& Rec, TFltV& FullV, int& Offset) const {
    TIntFltKdV NumSpV; GetVal(Rec, NumSpV);
    if (NormalizeP) { TLinAlg::Normalize(NumSpV); }
    for (int NumSpN = 0; NumSpN < NumSpV.Len(); NumSpN++) {
        const TIntFltKd& NumSp = NumSpV[NumSpN];
        FullV[Offset + NumSp.Key] = NumSp.Dat;
    }
    Offset += GetDim();
}

void TNumSpV::InvFullV(const TFltV& FullV, int& Offset, TFltV& InvV) const {
	throw TExcept::New("Not implemented yet!!", "TCategorical::InvFullV");
}

///////////////////////////////////////////////
// Categorical Feature Extractor
TStr TCategorical::_GetVal(const TRec& FtrRec) const {
	// assert store
	Assert(FtrRec.GetStoreId() == GetFtrStore()->GetStoreId());
{'fraction_non_alphanumeric': 0.16017817549887142, 'fraction_numerical': 0.0035754948764556657, 'mean_word_length': 3.6467421570447374, 'pattern_counts': {'":': 0, '<': 193, '<?xml version=': 0, '>': 329, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6507807', 'n_tokens_mistral': 19600, 'n_tokens_neox': 19118, 'n_words': 3959}