<commit_before>from __future__ import print_function, division
import os
import copy
from datetime import datetime
import numpy as np
#import mat_handler as mhand
#import pst_handler as phand
from pyemu.mat.mat_handler import Matrix, Jco, Cov
from pyemu.pst.pst_handler import Pst


class logger(object):
    """ a basic class for logging events during the linear analysis calculations
        if filename is passed, then an file handle is opened
    Parameters:
    ----------
        filename (bool or string): if string, it is the log file to write
            if a bool, then log is written to the screen
        echo (bool): a flag to force screen output
    Attributes:
    ----------
        items (dict) : tracks when something is started.  If a log entry is
            not in items, then it is treated as a new entry with the string
            being the key and the datetime as the value.  If a log entry is
            in items, then the end time and delta time are written and
            the item is popped from the keys

    """
    def __init__(self,filename, echo=False):
        self.items = {}
        self.echo = bool(echo)
        if filename == True:
            self.echo = True
            self.filename = None
        elif filename:
            self.filename = filename
            self.echo = True
            self.f = open(filename, 'w', 0) #unbuffered
            self.t = datetime.now()
            self.log("opening " + str(filename) + " for logging")
        else:
            self.filename = None


    def statement(self,phrase):
        t = datetime.now()
        s = str(t) + str(phrase)
        if self.echo:
            print(s,)
        if self.filename:
            self.f.write(s)


    def log(self,phrase):
        """log something that happened
        Parameters:
        ----------
            phrase (str) : the thing that happened
        Returns:
        -------
            None
        """
        pass
        t = datetime.now()
        if phrase in self.items.keys():
            s = str(t) + ' finished: ' + str(phrase) + " took: " + \
                str(t - self.items[phrase]) + '\n'
            if self.echo:
                print(s,)
            if self.filename:
                self.f.write(s)
            self.items.pop(phrase)
        else:
            s = str(t) + ' starting: ' + str(phrase) + '\n'
            if self.echo:
                print(s,)
            if self.filename:
                self.f.write(s)
            self.items[phrase] = copy.deepcopy(t)

    def warn(self,message):
        """write a warning to the log file
        Parameters:
        ----------
            message (str) : the warning text
        Returns:
        -------
            None
        """
        s = str(datetime.now()) + " WARNING: " + message + '\n'
        if self.echo:
            print(s,)
        if self.filename:
            self.f.write(s)


class LinearAnalysis(object):
    """ the super class for linear analysis.  Can be used for prior analyses
        only.  The derived types (schur and errvar) are for posterior analyses
        this class tries hard to not load items until they are needed
        all arguments are optional

    Parameters:
    ----------
        jco ([enumerable of] [string,ndarray,matrix objects]) : jacobian
        pst (pst object) : the pest control file object
        parcov ([enumerable of] [string,ndarray,matrix objects]) :
            parameter covariance matrix
        obscov ([enumerable of] [string,ndarray,matrix objects]):
            observation noise covariance matrix
        predictions ([enumerable of] [string,ndarray,matrix objects]) :
            prediction sensitivity vectors
        ref_var (float) : reference variance
        verbose (either bool or string) : controls log file / screen output
    Notes:
        the class makes heavy use of property decorator to encapsulate
        private attributes
    """
    def __init__(self, jco=None, pst=None, parcov=None, obscov=None,
                 predictions=None, ref_var=1.0, verbose=False,
                 resfile=False, forecasts=None,**kwargs):
        self.logger = logger(verbose)
        self.log = self.logger.log
        self.jco_arg = jco
        #if jco is None:
        self.__jco = jco
        if pst is None:
            if isinstance(jco, str):
                pst_case = jco.replace(".jco", ".pst").replace(".jcb",".pst")
                if os.path.exists(pst_case):
                    pst = pst_case
        self.pst_arg = pst
        if parcov is None and pst is not None:
            parcov = pst
        self.parcov_arg = parcov
        if obscov is None and pst is not None:
            obscov = pst
        self.obscov_arg = obscov
        self.ref_var = ref_var
        if forecasts is not None and predictions is not None:
            raise Exception("can't pass both forecasts and predictions")


        #private attributes - access is through @decorated functions
        self.__pst = None
        self.__parcov = None
        self.__obscov = None
        self.__predictions = None
        self.__qhalf = None
        self.__qhalfx = None
        self.__xtqx = None
        self.__fehalf = None
        self.__prior_prediction = None

        self.log("pre-loading base components")
        if jco is not None:
            self.__load_jco()
        if pst is not None:
            self.__load_pst()
        if parcov is not None:
            self.__load_parcov()
        if obscov is not None:
            self.__load_obscov()

        self.prediction_arg = None
        if predictions is not None:
            self.prediction_arg = predictions
        elif forecasts is not None:
            self.prediction_arg = forecasts
        elif self.pst is not None:
            if "forecasts" in self.pst.pestpp_options:
                self.prediction_arg = self.pst.pestpp_options["forecasts"].\
                    lower().split(',')
                self.__load_predictions()
            elif "predictions" in self.pst.pestpp_options:
                self.prediction_arg = self.pst.pestpp_options["predictions"].\
                    lower().split(',')
        if self.prediction_arg:
            self.__load_predictions()


        self.log("pre-loading base components")<commit_msg>bug fix in pestpp forecast parsing<commit_after>from __future__ import print_function, division
import os
import copy
from datetime import datetime
import numpy as np
#import mat_handler as mhand
#import pst_handler as phand
from pyemu.mat.mat_handler import Matrix, Jco, Cov
from pyemu.pst.pst_handler import Pst


class logger(object):
    """ a basic class for logging events during the linear analysis calculations
        if filename is passed, then an file handle is opened
    Parameters:
    ----------
        filename (bool or string): if string, it is the log file to write
            if a bool, then log is written to the screen
        echo (bool): a flag to force screen output
    Attributes:
    ----------
        items (dict) : tracks when something is started.  If a log entry is
            not in items, then it is treated as a new entry with the string
            being the key and the datetime as the value.  If a log entry is
            in items, then the end time and delta time are written and
            the item is popped from the keys

    """
    def __init__(self,filename, echo=False):
        self.items = {}
        self.echo = bool(echo)
        if filename == True:
            self.echo = True
            self.filename = None
        elif filename:
            self.filename = filename
            self.echo = True
            self.f = open(filename, 'w', 0) #unbuffered
            self.t = datetime.now()
            self.log("opening " + str(filename) + " for logging")
        else:
            self.filename = None


    def statement(self,phrase):
        t = datetime.now()
        s = str(t) + str(phrase)
        if self.echo:
            print(s,)
        if self.filename:
            self.f.write(s)


    def log(self,phrase):
        """log something that happened
        Parameters:
        ----------
            phrase (str) : the thing that happened
        Returns:
        -------
            None
        """
        pass
        t = datetime.now()
        if phrase in self.items.keys():
            s = str(t) + ' finished: ' + str(phrase) + " took: " + \
                str(t - self.items[phrase]) + '\n'
            if self.echo:
                print(s,)
            if self.filename:
                self.f.write(s)
            self.items.pop(phrase)
        else:
            s = str(t) + ' starting: ' + str(phrase) + '\n'
            if self.echo:
                print(s,)
            if self.filename:
                self.f.write(s)
            self.items[phrase] = copy.deepcopy(t)

    def warn(self,message):
        """write a warning to the log file
        Parameters:
        ----------
            message (str) : the warning text
        Returns:
        -------
            None
        """
        s = str(datetime.now()) + " WARNING: " + message + '\n'
        if self.echo:
            print(s,)
        if self.filename:
            self.f.write(s)


class LinearAnalysis(object):
    """ the super class for linear analysis.  Can be used for prior analyses
        only.  The derived types (schur and errvar) are for posterior analyses
        this class tries hard to not load items until they are needed
        all arguments are optional

    Parameters:
    ----------
        jco ([enumerable of] [string,ndarray,matrix objects]) : jacobian
        pst (pst object) : the pest control file object
        parcov ([enumerable of] [string,ndarray,matrix objects]) :
            parameter covariance matrix
        obscov ([enumerable of] [string,ndarray,matrix objects]):
            observation noise covariance matrix
        predictions ([enumerable of] [string,ndarray,matrix objects]) :
            prediction sensitivity vectors
        ref_var (float) : reference variance
        verbose (either bool or string) : controls log file / screen output
    Notes:
        the class makes heavy use of property decorator to encapsulate
        private attributes
    """
    def __init__(self, jco=None, pst=None, parcov=None, obscov=None,
                 predictions=None, ref_var=1.0, verbose=False,
                 resfile=False, forecasts=None,**kwargs):
        self.logger = logger(verbose)
        self.log = self.logger.log
        self.jco_arg = jco
        #if jco is None:
        self.__jco = jco
        if pst is None:
            if isinstance(jco, str):
                pst_case = jco.replace(".jco", ".pst").replace(".jcb",".pst")
                if os.path.exists(pst_case):
                    pst = pst_case
        self.pst_arg = pst
        if parcov is None and pst is not None:
            parcov = pst
        self.parcov_arg = parcov
        if obscov is None and pst is not None:
            obscov = pst
        self.obscov_arg = obscov
        self.ref_var = ref_var
        if forecasts is not None and predictions is not None:
            raise Exception("can't pass both forecasts and predictions")


        #private attributes - access is through @decorated functions
        self.__pst = None
        self.__parcov = None
        self.__obscov = None
        self.__predictions = None
        self.__qhalf = None
        self.__qhalfx = None
        self.__xtqx = None
        self.__fehalf = None
        self.__prior_prediction = None

        self.log("pre-loading base components")
        if jco is not None:
            self.__load_jco()
        if pst is not None:
            self.__load_pst()
        if parcov is not None:
            self.__load_parcov()
        if obscov is not None:
            self.__load_obscov()

        self.prediction_arg = None
        if predictions is not None:
            self.prediction_arg = predictions
        elif forecasts is not None:
            self.prediction_arg = forecasts
        elif self.pst is not None:
            if "forecasts" in self.pst.pestpp_options:
                self.prediction_arg = [i.strip() for i in self.pst.pestpp_options["forecasts"].\
                    lower().split(',')]
            elif "predictions" in self.pst.pestpp_options:
                self.prediction_arg = [i.strip() for i in self.pst.pestpp_options["predictions"].\
                    lower().split(',')]
        if self.prediction_arg:
            self.__load_predictions()


        self.log("pre-loading base components")
{'fraction_non_alphanumeric': 0.09332062584385672, 'fraction_numerical': 0.0004765308553728854, 'mean_word_length': 1.700986700986701, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5057793', 'n_tokens_mistral': 3481, 'n_tokens_neox': 3357, 'n_words': 1133}