<commit_before>'''
(c) 2011, 2012 Georgia Tech Research Corporation
This source code is released under the New BSD license.  Please see
http://wiki.quantsoftware.org/index.php?title=QSTK_License
for license details.

Created on Nov 7, 2011

@author: <NAME>
@contact: <EMAIL>
@summary: Contains utility functions to interact with feature functions in features.py
'''

''' Python imports '''
import math
import pickle
import datetime as dt
from dateutil.relativedelta import relativedelta

''' 3rd Party Imports '''
import numpy as np
import matplotlib.pyplot as plt

''' Our Imports '''
import qstklearn.kdtknn as kdt
from qstkutil import DataAccess as da
from qstkutil import qsdateutil as du
from qstkutil import tsutil as tsu

from qstkfeat.features import *
from qstkfeat.classes import class_fut_ret



def getMarketRel( dData, sRel='$SPX' ):
    '''
    @summary: Calculates market relative data.
    @param dData - Dictionary containing data to be used, requires specific naming: open/high/low/close/volume
    @param sRel - Stock ticker to make the data relative to, $SPX is default.
    @return: Dictionary of market relative values
    '''
    
    if sRel not in dData['close'].columns:
        raise KeyError( 'Market relative stock %s not found in getMR()'%sRel )
    
    
    dRet = {}
       
    ''' Make all data market relative, except for volume '''
    for sKey in dData.keys():
        
        ''' Don't calculate market relative volume, but still copy it over '''
        if sKey == 'volume':
            dRet['volume'] = dData['volume']
            continue
        
        dfMarkRel = dData[sKey].copy()
        
        #Get returns
        tsu.returnize0(  dfMarkRel.values )
        
        # Subtract market returns and make them 1-based, stocks start at 100
        dfMarkRel = (dfMarkRel - dfMarkRel[sRel]) + 1.
        dfMarkRel.ix[0, :] = 100.
        
        dfMarkRel = dfMarkRel.cumprod(axis=0)

        ''' Do not change market stock '''
        dfMarkRel[sRel] = dData[sKey][sRel]
        
        ''' Add dataFrame to dictionary to return, move to next key '''
        dRet[sKey] = dfMarkRel
        
        
    return dRet



def applyFeatures( dData, lfcFeatures, ldArgs, sMarketRel=None, sLog=None, bMin=False ):
    '''
    @summary: Calculates the feature values using a list of feature functions and arguments.
    @param dData - Dictionary containing data to be used, requires specific naming: open/high/low/close/volume
    @param lfcFeatures: List of feature functions, most likely coming from features.py
    @param ldArgs: List of dictionaries containing arguments, passed as **kwargs
                   There is a special argument 'MR', if it exists, the data will be made market relative
    @param sMarketRel: If not none, the data will all be made relative to the symbol provided
    @param sLog: If not None, will be filename to log all of the features to 
    @param bMin: If true, only run for the last day
    @return: list of dataframes containing values
    '''



        
    ldfRet = []
    
    ''' Calculate market relative data '''
    if sMarketRel != None:
        dDataRelative = getMarketRel( dData, sRel=sMarketRel )
    
    
    ''' Loop though feature functions, pass each data dictionary and arguments '''
    for i, fcFeature in enumerate(lfcFeatures):
        #dt_start = dt.datetime.now()
        #print fcFeature, ldArgs[i], ' in:',
        ''' Check for special arguments '''
        if 'MR' in ldArgs[i]:
            
            if ldArgs[i]['MR'] == False:
                print 'Warning, setting MR to false will still be Market Relative',\
                      'simply do not include MR key in args'
        
            if sMarketRel == None:
                raise AssertionError('Functions require market relative stock but sMarketRel=None')
            del ldArgs[i]['MR']
            if bMin:
                # bMin means only calculate the LAST row of the stock
                dTmp = {}
                for sKey in dDataRelative:
                    dTmp[sKey] = dDataRelative[sKey].ix[ -ldArgs[i]['lLookback']:]
                ldfRet.append( fcFeature( dTmp, **ldArgs[i] ).ix[-1:] )
            else:
                ldfRet.append( fcFeature( dDataRelative, **ldArgs[i] ) )
        else:
            if bMin:
                # bMin means only calculate the LAST row of the stock
                dTmp = {}
                for sKey in dData:
                    dTmp[sKey] = dData[sKey].ix[ -ldArgs[i]['lLookback']:]
                ldfRet.append( fcFeature( dTmp, **ldArgs[i] ).ix[-1:] )
            else:
                ldfRet.append( fcFeature( dData, **ldArgs[i] ) )
        #print  dt.datetime.now() - dt_start

    <commit_msg>For bMin support (calculate ONE value of a feature).  For an lLookback of 5, we usually need 5 days of data.  However, when we use returns, i.e. with stdev, beta, etc, we need 6. 6 days of price data produces 5 days of returns.<commit_after>'''
(c) 2011, 2012 Georgia Tech Research Corporation
This source code is released under the New BSD license.  Please see
http://wiki.quantsoftware.org/index.php?title=QSTK_License
for license details.

Created on Nov 7, 2011

@author: <NAME>
@contact: <EMAIL>
@summary: Contains utility functions to interact with feature functions in features.py
'''

''' Python imports '''
import math
import pickle
import datetime as dt
from dateutil.relativedelta import relativedelta

''' 3rd Party Imports '''
import numpy as np
import matplotlib.pyplot as plt

''' Our Imports '''
import qstklearn.kdtknn as kdt
from qstkutil import DataAccess as da
from qstkutil import qsdateutil as du
from qstkutil import tsutil as tsu

from qstkfeat.features import *
from qstkfeat.classes import class_fut_ret



def getMarketRel( dData, sRel='$SPX' ):
    '''
    @summary: Calculates market relative data.
    @param dData - Dictionary containing data to be used, requires specific naming: open/high/low/close/volume
    @param sRel - Stock ticker to make the data relative to, $SPX is default.
    @return: Dictionary of market relative values
    '''
    
    if sRel not in dData['close'].columns:
        raise KeyError( 'Market relative stock %s not found in getMR()'%sRel )
    
    
    dRet = {}
       
    ''' Make all data market relative, except for volume '''
    for sKey in dData.keys():
        
        ''' Don't calculate market relative volume, but still copy it over '''
        if sKey == 'volume':
            dRet['volume'] = dData['volume']
            continue
        
        dfMarkRel = dData[sKey].copy()
        
        #Get returns
        tsu.returnize0(  dfMarkRel.values )
        
        # Subtract market returns and make them 1-based, stocks start at 100
        dfMarkRel = (dfMarkRel - dfMarkRel[sRel]) + 1.
        dfMarkRel.ix[0, :] = 100.
        
        dfMarkRel = dfMarkRel.cumprod(axis=0)

        ''' Do not change market stock '''
        dfMarkRel[sRel] = dData[sKey][sRel]
        
        ''' Add dataFrame to dictionary to return, move to next key '''
        dRet[sKey] = dfMarkRel
        
        
    return dRet



def applyFeatures( dData, lfcFeatures, ldArgs, sMarketRel=None, sLog=None, bMin=False ):
    '''
    @summary: Calculates the feature values using a list of feature functions and arguments.
    @param dData - Dictionary containing data to be used, requires specific naming: open/high/low/close/volume
    @param lfcFeatures: List of feature functions, most likely coming from features.py
    @param ldArgs: List of dictionaries containing arguments, passed as **kwargs
                   There is a special argument 'MR', if it exists, the data will be made market relative
    @param sMarketRel: If not none, the data will all be made relative to the symbol provided
    @param sLog: If not None, will be filename to log all of the features to 
    @param bMin: If true, only run for the last day
    @return: list of dataframes containing values
    '''



        
    ldfRet = []
    
    ''' Calculate market relative data '''
    if sMarketRel != None:
        dDataRelative = getMarketRel( dData, sRel=sMarketRel )
    
    
    ''' Loop though feature functions, pass each data dictionary and arguments '''
    for i, fcFeature in enumerate(lfcFeatures):
        #dt_start = dt.datetime.now()
        #print fcFeature, ldArgs[i], ' in:',
        ''' Check for special arguments '''
        if 'MR' in ldArgs[i]:
            
            if ldArgs[i]['MR'] == False:
                print 'Warning, setting MR to false will still be Market Relative',\
                      'simply do not include MR key in args'
        
            if sMarketRel == None:
                raise AssertionError('Functions require market relative stock but sMarketRel=None')
            del ldArgs[i]['MR']
            if bMin:
                # bMin means only calculate the LAST row of the stock
                dTmp = {}
                for sKey in dDataRelative:
                    dTmp[sKey] = dDataRelative[sKey].ix[ -(ldArgs[i]['lLookback'] + 1):]
                ldfRet.append( fcFeature( dTmp, **ldArgs[i] ).ix[-1:] )
            else:
                ldfRet.append( fcFeature( dDataRelative, **ldArgs[i] ) )
        else:
            if bMin:
                # bMin means only calculate the LAST row of the stock
                dTmp = {}
                for sKey in dData:
                    dTmp[sKey] = dData[sKey].ix[ -(ldArgs[i]['lLookback'] + 1):]
                ldfRet.append( fcFeature( dTmp, **ldArgs[i] ).ix[-1:] )
            else:
                ldfRet.append( fcFeature( dData, **ldArgs[i] ) )
        #print  dt.datetime.now() - dt_start

    
{'fraction_non_alphanumeric': 0.09227910817506194, 'fraction_numerical': 0.00629644921552436, 'mean_word_length': 2.3113465481886535, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5284496', 'n_tokens_mistral': 2903, 'n_tokens_neox': 2743, 'n_words': 1019}