<commit_before># Copyright (C) 2016 <NAME> <<EMAIL>>
# All rights reserved.

# This file is part of the Python Automatic Forecasting (PyAF) library and is made available under
# the terms of the 3 Clause BSD license



import pandas as pd
import numpy as np


from . import Utils as tsutil

class cExogenousInfo:

    def __init__(self):
        self.mExogenousVariables = None;
        self.mEncodedExogenous = None;
        self.mExogenousVariableCategories = None;
        self.mExogenousDataFrame = None;
        self.mExogenousData = None;        
        self.mDateVariable = None;
        self.mContExogenousStats = None;
        
    def info(self):
        lStr2 = "ExogenousVariables = '" + self.mExogD +"'";
        return lStr2;


    def to_json(self):
        dict1 = {};
        return dict1;

    def fit(self):
        self.mExogenousDataFrame = self.mExogenousData[0];
        self.mExogenousVariables = self.mExogenousData[1];
        self.mDateVariable = self.mTimeInfo.mTime;
        # print("preProcessExogenousVariables , columns", self.mExogenousVariables);
        self.updateExogenousVariableInfo();
        self.createEncodedExogenous();
        # print("preProcessExogenousVariables , dummy columns", self.mEncodedExogenous);

    def addVars(self, df):
        lExogDate = "exog_date_" + self.mDateVariable;
        # print("EXOG_COLUMNS", self.mEncodedExogenousDataFrame.columns);
        df = df.merge(self.mEncodedExogenousDataFrame,
                      how='left',
                      left_on=self.mDateVariable,
                      right_on=lExogDate);
        df = df.drop([lExogDate] , axis = 1);
        return df;

    def transformDataset(self, df):
        # print("BEFORE_EXOG_TRANSFORM_DATASET" , df.shape, df.columns);
        df = self.addVars(df);
        # print("AFTER_EXOG_TRANSFORM_DATASET" , df.shape, df.columns);
        return df;
        
    def createEncodedExogenous(self):
        self.mExogDummiesDataFrame = pd.DataFrame();
        self.mEncodedExogenous = [];
        self.mEncodedExogenousDataFrame = pd.DataFrame();
        self.mEncodedExogenousDataFrame["exog_date_" + self.mDateVariable] = self.mExogenousDataFrame[self.mDateVariable];
        for exog in self.mExogenousVariables:
            lList = self.mExogenousVariableCategories[exog];
            if(lList is not None):
                for lCat in lList:
                    lDummyName = "exog_dummy_" + exog + "=" + str(lCat);
                    self.mEncodedExogenousDataFrame[lDummyName] = np.where(self.mExogenousDataFrame[exog] == lCat , 1, 0);
                    self.mEncodedExogenous = self.mEncodedExogenous + [lDummyName];
            else:
                lExogStats = self.mContExogenousStats[exog];
                self.mEncodedExogenousDataFrame[exog] = (self.mExogenousDataFrame[exog] - lExogStats[0])/ lExogStats[1];
                self.mEncodedExogenousDataFrame[exog].fillna(0.0, inplace=True);
                self.mEncodedExogenous = self.mEncodedExogenous + [exog];


    def updateExogenousVariableInfo(self):
        self.mExogenousVariableCategories = {};
        self.mContExogenousStats = {};
        lEstimFrame = self.mTimeInfo.getEstimPart(self.mExogenousDataFrame)
        for exog in self.mExogenousVariables:
            if(self.mExogenousDataFrame[exog].dtype == np.object):
                # use nan as a category
                lVC = lEstimFrame[exog].value_counts(dropna = False);
                NCat = self.mOptions.mMaxExogenousCategories;
                lList = lVC.index[0:NCat].tolist();
                print("most_frequent_categories_for" , exog, lList);
                self.mExogenousVariableCategories[exog] = lList;
            else:
                self.mExogenousVariableCategories[exog] = None;
                self.mContExogenousStats[exog] = (lEstimFrame[exog].mean(), lEstimFrame[exog].std());<commit_msg>1. renamed encoded exogenous avriables. 2. Correcetd some encoding for future exogenous data.<commit_after># Copyright (C) 2016 <NAME> <<EMAIL>>
# All rights reserved.

# This file is part of the Python Automatic Forecasting (PyAF) library and is made available under
# the terms of the 3 Clause BSD license



import pandas as pd
import numpy as np


from . import Utils as tsutil

class cExogenousInfo:

    def __init__(self):
        self.mExogenousVariables = None;
        self.mEncodedExogenous = None;
        self.mExogenousVariableCategories = None;
        self.mExogenousDataFrame = None;
        self.mExogenousData = None;        
        self.mDateVariable = None;
        self.mContExogenousStats = None;
        
    def info(self):
        lStr2 = "ExogenousVariables = '" + self.mExogD +"'";
        return lStr2;


    def to_json(self):
        dict1 = {};
        return dict1;

    def fit(self):
        self.mExogenousDataFrame = self.mExogenousData[0];
        self.mExogenousVariables = self.mExogenousData[1];
        self.mDateVariable = self.mTimeInfo.mTime;
        # print("preProcessExogenousVariables , columns", self.mExogenousVariables);
        self.updateExogenousVariableInfo();
        self.createEncodedExogenous();
        # print("preProcessExogenousVariables , dummy columns", self.mEncodedExogenous);

    def addVars(self, df):
        lExogDate = self.mDateVariable;
        # print(np.dtype(df[self.mDateVariable]) ,
        #      np.dtype(self.mEncodedExogenousDataFrame[self.mDateVariable]));
        # print(df.info());
        # print(self.mEncodedExogenousDataFrame.info());
        # print(df.head());
        # print(self.mEncodedExogenousDataFrame.head());

        df1 = df[[self.mDateVariable]];
        lCompleteEncoded = df1.merge(self.mEncodedExogenousDataFrame,
                                     how='left',
                                     left_on=self.mDateVariable,
                                     right_on=lExogDate);
        lCompleteEncoded.fillna(0.0, inplace=True);
        
        # print("EXOG_COLUMNS", self.mEncodedExogenousDataFrame.columns);
        df2 = df.merge(lCompleteEncoded,
                       how='left',
                       left_on=self.mDateVariable,
                       right_on=lExogDate);
        # df1 = df1.drop([lExogDate] , axis = 1);
        # print(df.head());
        return df2;

    def transformDataset(self, df):
        # print("BEFORE_EXOG_TRANSFORM_DATASET" , df.shape, df.columns);
        df1 = self.addVars(df);
        # print("AFTER_EXOG_TRANSFORM_DATASET" , df.shape, df.columns);
        return df1;
        
    def createEncodedExogenous(self):
        self.mExogDummiesDataFrame = pd.DataFrame();
        self.mEncodedExogenous = [];
        self.mEncodedExogenousDataFrame = pd.DataFrame();
        self.mEncodedExogenousDataFrame[self.mDateVariable] = self.mExogenousDataFrame[self.mDateVariable];
        for exog in self.mExogenousVariables:
            lList = self.mExogenousVariableCategories[exog];
            if(lList is not None):
                for lCat in lList:
                    lDummyName = exog + "=" + str(lCat);
                    self.mEncodedExogenousDataFrame[lDummyName] = np.where(self.mExogenousDataFrame[exog] == lCat , 1, 0);
                    self.mEncodedExogenous = self.mEncodedExogenous + [lDummyName];
            else:
                lExogStats = self.mContExogenousStats[exog];
                self.mEncodedExogenousDataFrame[exog] = (self.mExogenousDataFrame[exog] - lExogStats[0])/ lExogStats[1];
                self.mEncodedExogenousDataFrame[exog].fillna(0.0, inplace=True);
                self.mEncodedExogenous = self.mEncodedExogenous + [exog];


    def updateExogenousVariableInfo(self):
        self.mExogenousVariableCategories = {};
        self.mContExogenousStats = {};
        lEstimFrame = self.mTimeInfo.getEstimPart(self.mExogenousDataFrame)
        for exog in self.mExogenousVariables:
            if(self.mExogenousDataFrame[exog].dtype == np.object):
                # use nan as a category
                lVC = lEstimFrame[exog].value_counts(dropna = False);
                NCat = self.mOptions.mMaxExogenousCategories;
                lList = lVC.index[0:NCat].tolist();
                # print("most_frequent_categories_for" , exog, lList);
                self.mExogenousVariableCategories[exog] = lList;
            else:
                self.mExogenousVariableCategories[exog] = None;
                self.mContExogenousStats[exog] = (lEstimFrame[exog].mean(), lEstimFrame[exog].std());
{'fraction_non_alphanumeric': 0.10233160621761658, 'fraction_numerical': 0.005887894488930758, 'mean_word_length': 2.406738868832732, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '296495', 'n_tokens_mistral': 2731, 'n_tokens_neox': 2528, 'n_words': 440}