<commit_before># Copyright (C) 2016 <NAME> <<EMAIL>>
# All rights reserved.

# This file is part of the Python Automatic Forecasting (PyAF) library and is made available under
# the terms of the 3 Clause BSD license

import pandas as pd
import numpy as np
import datetime
import sys
import traceback

import dill
dill.settings['recurse'] = False
# import dill
# import multiprocessing as mp
from pathos.multiprocessing import ProcessingPool as Pool

# for timing
import time

import threading

from . import Time as tsti
from . import Exogenous as tsexog
from . import MissingData as tsmiss
from . import Signal_Transformation as tstransf
from . import Perf as tsperf
from . import SignalDecomposition_Trend as tstr
from . import SignalDecomposition_Cycle as tscy
from . import SignalDecomposition_AR as tsar
from . import Options as tsopts
from . import TimeSeriesModel as tsmodel
from . import Utils as tsutil



class cPerf_Arg:
    def __init__(self , name):
        self.mName = name;
        self.mModel = None;
        self.mResult = None;

def compute_perf_func(arg):
    # print("RUNNING " , arg.mName)
    try:
        arg.mModel.updatePerfs();
        return arg;
    except Exception as e:
        logger = tsutil.get_pyaf_logger();
        # print("FAILURE_WITH_EXCEPTION : " + str(e)[:200]);
        logger.error("FAILURE_WITH_EXCEPTION : " + str(e)[:200]);
        logger.error("BENCHMARKING_FAILURE '" + arg.mName + "'");
        traceback.print_exc()
        raise;
    except:
        # print("BENCHMARK_FAILURE '" + arg.mName + "'");
        logger.error("BENCHMARK_FAILURE '" + arg.mName + "'");
        raise

class cSignalDecompositionOneTransform:
        
    def __init__(self):
        self.mSignalFrame = pd.DataFrame()
        self.mTime = None
        self.mSignal = None
        self.mTimeInfo = tsti.cTimeInfo();
        self.mForecastFrame = pd.DataFrame()
        self.mTransformation = tstransf.cSignalTransform_None();
        

    def serialize(self):
        from sklearn.externals import joblib
        joblib.dump(self, self.mTimeInfo.mTime + "_" + self.mSignal + "_TS.pkl")        

    def setParams(self , iInputDS, iTime, iSignal, iHorizon, iTransformation, iExogenousData = None):
        assert(iInputDS.shape[0] > 0)
        assert(iInputDS.shape[1] > 0)
        assert(iTime in iInputDS.columns)
        assert(iSignal in iInputDS.columns)

        # print("setParams , head", iInputDS.head());
        # print("setParams , tail", iInputDS.tail());
        # print("setParams , columns", iInputDS.columns);
        
        self.mTime = iTime
        self.mOriginalSignal = iSignal;
        
        self.mTransformation = iTransformation;
        self.mTransformation.mOriginalSignal = iSignal; 

        lMissingImputer = tsmiss.cMissingDataImputer()
        lMissingImputer.mOptions = self.mOptions
        lSignal = lMissingImputer.interpolate_signal_if_needed(iInputDS, iSignal)
        lTime = lMissingImputer.interpolate_time_if_needed(iInputDS, iTime)

        self.mTransformation.fit(lSignal);

        self.mSignal = iTransformation.get_name(iSignal)
        self.mHorizon = iHorizon;
        self.mSignalFrame = pd.DataFrame()
        self.mSignalFrame[self.mOriginalSignal] = lSignal;
        self.mSignalFrame[self.mSignal] = self.mTransformation.apply(lSignal);
        self.mSignalFrame[self.mTime] = lTime;
        self.mSignalFrame['row_number'] = np.arange(0, iInputDS.shape[0]);
        # self.mSignalFrame.dropna(inplace = True);
        assert(self.mSignalFrame.shape[0] > 0);

        # print("SIGNAL_INFO " , self.mSignalFrame.info());
        
        self.mTimeInfo = tsti.cTimeInfo();
        self.mTimeInfo.mTime = self.mTime;
        self.mTimeInfo.mSignal = self.mSignal;
        self.mTimeInfo.mOriginalSignal = self.mOriginalSignal;
        self.mTimeInfo.mHorizon = self.mHorizon;
        self.mTimeInfo.mSignalFrame = self.mSignalFrame;
        self.mTimeInfo.mOptions = self.mOptions;

        self.mExogenousInfo = None;
        if(iExogenousData is not None):
            self.mExogenousInfo = tsexog.cExogenousInfo();
            self.mExogenousInfo.mExogenousData = iExogenousData;
            self.mExogenousInfo.mTimeInfo = self.mTimeInfo;
            self.mExogenousInfo.mOptions = self.mOptions;
        

    def computePerfsInParallel(self, args):
        lModels = {};
        # print([arg.mName for arg in args]);
        # print([arg.mModel.mOutName for arg in args]);
        pool = Pool(self.mOptions.mNbCores)
        # results = [compute_perf_func(arg) for arg in args];
        for res in pool.imap(compute_perf_func, args):<commit_msg>Use the date column as first column (keep the  index).<commit_after># Copyright (C) 2016 <NAME> <<EMAIL>>
# All rights reserved.

# This file is part of the Python Automatic Forecasting (PyAF) library and is made available under
# the terms of the 3 Clause BSD license

import pandas as pd
import numpy as np
import datetime
import sys
import traceback

import dill
dill.settings['recurse'] = False
# import dill
# import multiprocessing as mp
from pathos.multiprocessing import ProcessingPool as Pool

# for timing
import time

import threading

from . import Time as tsti
from . import Exogenous as tsexog
from . import MissingData as tsmiss
from . import Signal_Transformation as tstransf
from . import Perf as tsperf
from . import SignalDecomposition_Trend as tstr
from . import SignalDecomposition_Cycle as tscy
from . import SignalDecomposition_AR as tsar
from . import Options as tsopts
from . import TimeSeriesModel as tsmodel
from . import Utils as tsutil



class cPerf_Arg:
    def __init__(self , name):
        self.mName = name;
        self.mModel = None;
        self.mResult = None;

def compute_perf_func(arg):
    # print("RUNNING " , arg.mName)
    try:
        arg.mModel.updatePerfs();
        return arg;
    except Exception as e:
        logger = tsutil.get_pyaf_logger();
        # print("FAILURE_WITH_EXCEPTION : " + str(e)[:200]);
        logger.error("FAILURE_WITH_EXCEPTION : " + str(e)[:200]);
        logger.error("BENCHMARKING_FAILURE '" + arg.mName + "'");
        traceback.print_exc()
        raise;
    except:
        # print("BENCHMARK_FAILURE '" + arg.mName + "'");
        logger.error("BENCHMARK_FAILURE '" + arg.mName + "'");
        raise

class cSignalDecompositionOneTransform:
        
    def __init__(self):
        self.mSignalFrame = pd.DataFrame()
        self.mTime = None
        self.mSignal = None
        self.mTimeInfo = tsti.cTimeInfo();
        self.mForecastFrame = pd.DataFrame()
        self.mTransformation = tstransf.cSignalTransform_None();
        

    def serialize(self):
        from sklearn.externals import joblib
        joblib.dump(self, self.mTimeInfo.mTime + "_" + self.mSignal + "_TS.pkl")        

    def setParams(self , iInputDS, iTime, iSignal, iHorizon, iTransformation, iExogenousData = None):
        assert(iInputDS.shape[0] > 0)
        assert(iInputDS.shape[1] > 0)
        assert(iTime in iInputDS.columns)
        assert(iSignal in iInputDS.columns)

        # print("setParams , head", iInputDS.head());
        # print("setParams , tail", iInputDS.tail());
        # print("setParams , columns", iInputDS.columns);
        
        self.mTime = iTime
        self.mOriginalSignal = iSignal;
        
        self.mTransformation = iTransformation;
        self.mTransformation.mOriginalSignal = iSignal; 

        lMissingImputer = tsmiss.cMissingDataImputer()
        lMissingImputer.mOptions = self.mOptions
        lSignal = lMissingImputer.interpolate_signal_if_needed(iInputDS, iSignal)
        lTime = lMissingImputer.interpolate_time_if_needed(iInputDS, iTime)

        self.mTransformation.fit(lSignal);

        self.mSignal = iTransformation.get_name(iSignal)
        self.mHorizon = iHorizon;
        self.mSignalFrame = pd.DataFrame()
        self.mSignalFrame[self.mTime] = lTime;
        self.mSignalFrame[self.mOriginalSignal] = lSignal;
        self.mSignalFrame[self.mSignal] = self.mTransformation.apply(lSignal);
        self.mSignalFrame['row_number'] = np.arange(0, iInputDS.shape[0]);
        # self.mSignalFrame.dropna(inplace = True);
        assert(self.mSignalFrame.shape[0] > 0);

        # print("SIGNAL_INFO " , self.mSignalFrame.info());
        # print(self.mSignalFrame.head())
        
        self.mTimeInfo = tsti.cTimeInfo();
        self.mTimeInfo.mTime = self.mTime;
        self.mTimeInfo.mSignal = self.mSignal;
        self.mTimeInfo.mOriginalSignal = self.mOriginalSignal;
        self.mTimeInfo.mHorizon = self.mHorizon;
        self.mTimeInfo.mSignalFrame = self.mSignalFrame;
        self.mTimeInfo.mOptions = self.mOptions;

        self.mExogenousInfo = None;
        if(iExogenousData is not None):
            self.mExogenousInfo = tsexog.cExogenousInfo();
            self.mExogenousInfo.mExogenousData = iExogenousData;
            self.mExogenousInfo.mTimeInfo = self.mTimeInfo;
            self.mExogenousInfo.mOptions = self.mOptions;
        

    def computePerfsInParallel(self, args):
        lModels = {};
        # print([arg.mName for arg in args]);
        # print([arg.mModel.mOutName for arg in args]);
        pool = Pool(self.mOptions.mNbCores)
        # results = [compute_perf_func(arg) for arg in args];
        for res in pool.imap(compute_perf_func, args):
{'fraction_non_alphanumeric': 0.1004595490007481, 'fraction_numerical': 0.004061130704285562, 'mean_word_length': 2.8289689034369885, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6056940', 'n_tokens_mistral': 3024, 'n_tokens_neox': 2917, 'n_words': 633}