<commit_before>"""
Statistical tools for time series analysis
"""

import numpy as np
from scipy import stats, signal
from scikits.statsmodels.regression.linear_model import OLS, yule_walker
from scikits.statsmodels.tools.tools import add_constant
from tsatools import lagmat, lagmat2ds, add_trend
#from scikits.statsmodels.sandbox.tsa import var
from adfvalues import *
#from scikits.statsmodels.sandbox.rls import RLS

#NOTE: now in two places to avoid circular import
#TODO: I like the bunch pattern for this too.
class ResultsStore(object):
    def __str__(self):
        return self._str

def _autolag(mod, endog, exog, startlag, maxlag, method, modargs=(),
        fitargs=()):
    """
    Returns the results for the lag length that maximimizes the info criterion.

    Parameters
    ----------
    mod : Model class
        Model estimator class.
    modargs : tuple
        args to pass to model.  See notes.
    fitargs : tuple
        args to pass to fit.  See notes.
    lagstart : int
        The first zero-indexed column to hold a lag.  See Notes.
    maxlag : int
        The highest lag order for lag length selection.
    method : str {"aic","bic","t-stat"}
        aic - Akaike Information Criterion
        bic - Bayes Information Criterion
        t-stat - Based on last lag

    Returns
    -------
    icbest : float
        Best information criteria.
    bestlag : int
        The lag length that maximizes the information criterion.


    Notes
    -----
    Does estimation like mod(endog, exog[:,:i], *modargs).fit(*fitargs)
    where i goes from lagstart to lagstart+maxlag+1.  Therefore, lags are
    assumed to be in contiguous columns from low to high lag length with
    the highest lag in the last column.
    """
#TODO: can tcol be replaced by maxlag + 2?
#TODO: This could be changed to laggedRHS and exog keyword arguments if this
#    will be more general.

    results = {}
    method = method.lower()
    for lag in range(startlag,maxlag+1):
        mod_instance = mod(endog, exog[:,:lag], *modargs)
        results[lag] = mod_instance.fit()

    if method == "aic":
        icbest, bestlag = max((v.aic,k) for k,v in results.iteritems())
    elif method == "bic":
        icbest, bestlag = max((v.bic,k) for k,v in results.iteritems())
    elif method == "t-stat":
        lags = sorted(results.keys())[::-1]
#        stop = stats.norm.ppf(.95)
        stop = 1.6448536269514722
        for lag in range(maxlag,startlag-1,-1):
            print lag
            print results[lag].values
            icbest = np.abs(results[lag].tvalues[-1])
            print icbest
            if np.abs(icbest) >= stop:
                bestlag = lag
                icbest = icbest
                break
    else:
        raise ValueError("Information Criterion %s not understood.") % method
    return icbest, bestlag

#this needs to be converted to a class like HetGoldfeldQuandt, 3 different returns are a mess
# See:
#Ng and Perron(2001), Lag length selection and the construction of unit root
#tests with good size and power, Econometrica, Vol 69 (6) pp 1519-1554
#TODO: include drift keyword, only valid with regression == "c"<commit_msg>BUG: Remove print statement in tsa.stattools._autolag<commit_after>"""
Statistical tools for time series analysis
"""

import numpy as np
from scipy import stats, signal
from scikits.statsmodels.regression.linear_model import OLS, yule_walker
from scikits.statsmodels.tools.tools import add_constant
from tsatools import lagmat, lagmat2ds, add_trend
#from scikits.statsmodels.sandbox.tsa import var
from adfvalues import *
#from scikits.statsmodels.sandbox.rls import RLS

#NOTE: now in two places to avoid circular import
#TODO: I like the bunch pattern for this too.
class ResultsStore(object):
    def __str__(self):
        return self._str

def _autolag(mod, endog, exog, startlag, maxlag, method, modargs=(),
        fitargs=()):
    """
    Returns the results for the lag length that maximimizes the info criterion.

    Parameters
    ----------
    mod : Model class
        Model estimator class.
    modargs : tuple
        args to pass to model.  See notes.
    fitargs : tuple
        args to pass to fit.  See notes.
    lagstart : int
        The first zero-indexed column to hold a lag.  See Notes.
    maxlag : int
        The highest lag order for lag length selection.
    method : str {"aic","bic","t-stat"}
        aic - Akaike Information Criterion
        bic - Bayes Information Criterion
        t-stat - Based on last lag

    Returns
    -------
    icbest : float
        Best information criteria.
    bestlag : int
        The lag length that maximizes the information criterion.


    Notes
    -----
    Does estimation like mod(endog, exog[:,:i], *modargs).fit(*fitargs)
    where i goes from lagstart to lagstart+maxlag+1.  Therefore, lags are
    assumed to be in contiguous columns from low to high lag length with
    the highest lag in the last column.
    """
#TODO: can tcol be replaced by maxlag + 2?
#TODO: This could be changed to laggedRHS and exog keyword arguments if this
#    will be more general.

    results = {}
    method = method.lower()
    for lag in range(startlag,maxlag+1):
        mod_instance = mod(endog, exog[:,:lag], *modargs)
        results[lag] = mod_instance.fit()

    if method == "aic":
        icbest, bestlag = max((v.aic,k) for k,v in results.iteritems())
    elif method == "bic":
        icbest, bestlag = max((v.bic,k) for k,v in results.iteritems())
    elif method == "t-stat":
        lags = sorted(results.keys())[::-1]
#        stop = stats.norm.ppf(.95)
        stop = 1.6448536269514722
        for lag in range(maxlag,startlag-1,-1):
            icbest = np.abs(results[lag].tvalues[-1])
            if np.abs(icbest) >= stop:
                bestlag = lag
                icbest = icbest
                break
    else:
        raise ValueError("Information Criterion %s not understood.") % method
    return icbest, bestlag

#this needs to be converted to a class like HetGoldfeldQuandt, 3 different returns are a mess
# See:
#Ng and Perron(2001), Lag length selection and the construction of unit root
#tests with good size and power, Econometrica, Vol 69 (6) pp 1519-1554
#TODO: include drift keyword, only valid with regression == "c"
{'fraction_non_alphanumeric': 0.09308975377283558, 'fraction_numerical': 0.013661636219221604, 'mean_word_length': 2.641411220358589, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6421858', 'n_tokens_mistral': 1942, 'n_tokens_neox': 1839, 'n_words': 700}