<commit_before>'''
Created on Nov 7, 2011

@author: <NAME>
@contact: <EMAIL>
@summary: Contains utility functions to interact with feature functions in features.py
'''

''' Python imports '''
import math
import pickle
import datetime as dt
from dateutil.relativedelta import relativedelta

''' 3rd Party Imports '''
import numpy as np
import matplotlib.pyplot as plt

''' Our Imports '''
import qstklearn.kdtknn as kdt
from qstkutil import DataAccess as da
from qstkutil import dateutil as du
from qstkutil import tsutil as tsu

from qstkfeat.features import *
from qstkfeat.classes import classFutRet


def makeMarketRel( dfPrice, sRel ):
    pass

def applyFeatures( dData, lfcFeatures, ldArgs, sMarketRel=None, sLog=None ):
    '''
    @summary: Calculates the feature values using a list of feature functions and arguments.
    @param dData - Dictionary containing data to be used, requires specific naming: open/high/low/close/volume
    @param lfcFeatures: List of feature functions, most likely coming from features.py
    @param ldArgs: List of dictionaries containing arguments, passed as **kwargs
    @param sMarketRel:   
    @param sLog: If not None, will be filename to log all of the features to 
    @return: list of dataframes containing values
    '''

    dfPrice = dData['close']

    if not sMarketRel == None:
    
        naRets = dfPrice.values.copy()
        tsu.returnize0( naRets )
        
        print sMarketRel
        naMarkRets = naRets[:, list(dfPrice.columns).index(sMarketRel) ]
        
        for i, sStock in enumerate(dfPrice.columns):
            ''' Don't change the 'market' stock '''
            if sStock == sMarketRel:
                continue
            
            naMarkRel = (naRets[:,i] - naMarkRets) + 1.0
        
#            if sStock == 'COV':
#                print naRets[:,i]
#                print naMarkRets
#                print naMarkRel
#                print dfPrice.values[:,i]
        
            for j in range(0, dfPrice.values.shape[0]):
                if pand.isnull( dfPrice.values[j][i] ):
                    continue
                dfPrice.values[j][i] = 100
                break
                
            for j in range(j+1, dfPrice.values.shape[0]):
                dfPrice.values[j][i] =  dfPrice.values[j-1][i] * naMarkRel[j]

#            if sStock == 'COV':
#                print '\n\n\n'
#                print dfPrice.values[:,i]

    
#    print dfPrice['COV'].values
#    print dfPrice['SPY'].values
#    import matplotlib.pyplot as plt
#    plt.clf()
#    plt.plot(tsBefore.index, tsBefore.values)
#    plt.plot(dfPrice['COV'].index, dfPrice['COV'].values )
#    plt.plot(dfPrice['SPY'].index, dfPrice['SPY'].values )
#    plt.legend(('before', 'after', 'spy'))
#    plt.show()
        
    ldfRet = []
    
    ''' Loop though feature functions, pass each data dictionary and arguments '''
    for i, fcFeature in enumerate(lfcFeatures):
        ldfRet.append( fcFeature( dData, **ldArgs[i] ) )

        
    if not sLog == None:
        with open( sLog, 'wb' ) as fFile:
            pickle.dump( ldfRet, fFile, -1 )
        
    return ldfRet

def loadFeatures( sLog ):
    '''
    @summary: Loads cached features.
    @param sLog: Filename of features.
    @return: Numpy array containing values
    '''
    
    ldfRet = []
    <commit_msg>Quick fix for market relative param<commit_after>'''
Created on Nov 7, 2011

@author: <NAME>
@contact: <EMAIL>
@summary: Contains utility functions to interact with feature functions in features.py
'''

''' Python imports '''
import math
import pickle
import datetime as dt
from dateutil.relativedelta import relativedelta

''' 3rd Party Imports '''
import numpy as np
import matplotlib.pyplot as plt

''' Our Imports '''
import qstklearn.kdtknn as kdt
from qstkutil import DataAccess as da
from qstkutil import dateutil as du
from qstkutil import tsutil as tsu

from qstkfeat.features import *
from qstkfeat.classes import classFutRet


def makeMarketRel( dfPrice, sRel ):
    pass

def applyFeatures( dData, lfcFeatures, ldArgs, sMarketRel=None, sLog=None ):
    '''
    @summary: Calculates the feature values using a list of feature functions and arguments.
    @param dData - Dictionary containing data to be used, requires specific naming: open/high/low/close/volume
    @param lfcFeatures: List of feature functions, most likely coming from features.py
    @param ldArgs: List of dictionaries containing arguments, passed as **kwargs
    @param sMarketRel:   
    @param sLog: If not None, will be filename to log all of the features to 
    @return: list of dataframes containing values
    '''

    dfPrice = dData['close']

    if not sMarketRel == None:
    
        naRets = dfPrice.values.copy()
        tsu.returnize0( naRets )
        
        print sMarketRel
        naMarkRets = naRets[:, list(dfPrice.columns).index(sMarketRel) ]
        
        for i, sStock in enumerate(dfPrice.columns):
            ''' Don't change the 'market' stock '''
            if sStock == sMarketRel:
                continue
            
            naMarkRel = (naRets[:,i] - naMarkRets) + 1.0
        
#            if sStock == 'COV':
#                print naRets[:,i]
#                print naMarkRets
#                print naMarkRel
#                print dfPrice.values[:,i]
        
            for j in range(0, dfPrice.values.shape[0]):
                if pand.isnull( dfPrice.values[j][i] ):
                    continue
                dfPrice.values[j][i] = 100
                break
                
            for j in range(j+1, dfPrice.values.shape[0]):
                dfPrice.values[j][i] =  dfPrice.values[j-1][i] * naMarkRel[j]

#            if sStock == 'COV':
#                print '\n\n\n'
#                print dfPrice.values[:,i]

    
#    print dfPrice['COV'].values
#    print dfPrice['SPY'].values
#    import matplotlib.pyplot as plt
#    plt.clf()
#    plt.plot(tsBefore.index, tsBefore.values)
#    plt.plot(dfPrice['COV'].index, dfPrice['COV'].values )
#    plt.plot(dfPrice['SPY'].index, dfPrice['SPY'].values )
#    plt.legend(('before', 'after', 'spy'))
#    plt.show()
        
    ldfRet = []
    
    ''' Loop though feature functions, pass each data dictionary and arguments '''
    for i, fcFeature in enumerate(lfcFeatures):
        
        ''' TODO fix this, delete all MR for now '''
        if 'MR' in ldArgs[i]:
            del ldArgs[i]['MR']
        
        ldfRet.append( fcFeature( dData, **ldArgs[i] ) )

        
    if not sLog == None:
        with open( sLog, 'wb' ) as fFile:
            pickle.dump( ldfRet, fFile, -1 )
        
    return ldfRet

def loadFeatures( sLog ):
    '''
    @summary: Loads cached features.
    @param sLog: Filename of features.
    @return: Numpy array containing values
    '''
    
    ldfRet = []
    
{'fraction_non_alphanumeric': 0.10820295364819418, 'fraction_numerical': 0.005263927474777014, 'mean_word_length': 2.254043767840152, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3278780', 'n_tokens_mistral': 2161, 'n_tokens_neox': 2072, 'n_words': 623}