<commit_before>'''
(c) 2011, 2012 Georgia Tech Research Corporation
This source code is released under the New BSD license.  Please see
http://wiki.quantsoftware.org/index.php?title=QSTK_License
for license details.

Created on April 20, 2012

@author: <NAME>
@contact: <EMAIL>
@summary: Demonstrates the use of the CVXOPT portfolio optimization call.
'''

import qstkutil.qsdateutil as du
import qstkutil.tsutil as tsu
import qstkutil.DataAccess as da
import datetime as dt

import matplotlib.pyplot as plt
from pylab import *
from math import isnan
from copy import copy

''' Function gets a 100 sample point frontier for given returns '''
def getFrontier(naData):
    ''' Special case for fTarget = None, just get average returns '''
    (naAvgRets,naStd, b_error) = tsu.OptPort( naData, None )

    naLower = np.zeros(naData.shape[1])
    naUpper = np.ones(naData.shape[1])
    
    (fMin, fMax) = tsu.getRetRange( naData, naLower, naUpper, naAvgRets, s_type="long")
    
    fStep = (fMax - fMin) / 100.0
    
    lfReturn =  [fMin + x * fStep for x in range(101)]
    lfStd = []
    lnaPortfolios = []
    
    ''' Call the function 100 times for the given range '''
    for fTarget in lfReturn: 
        (naWeights, fStd, b_error) = tsu.OptPort( naData, fTarget, naLower, naUpper, s_type = "long")
        #if b_error == False:
        lfStd.append(fStd)
        lnaPortfolios.append( naWeights )
        #lfReturn.pop(lfReturn.index(fTarget))
    return (lfReturn, lfStd, lnaPortfolios, naAvgRets, naStd)
    


''' ******************************************************* '''
''' ******************** MAIN SCRIPT ********************** '''
''' ******************************************************* '''

''' S&P100 '''
lsSymbols = [ 'AA', 'AAPL', 'ABT', 'AEP', 'ALL', 'AMGN', 'AMZN', 'AVP', 'AXP', 'BA', 'BAC', 'BAX', 'BHI', 'BK', 'BMY', 'BNI', 'C', 'CAT', 'CL', 'CMCSA', 'COF', 'COP', 'COST', 'COV', 'CPB', 'CSCO', 'CVS', 'CVX', 'DD', 'DELL', 'DIS', 'DOW', 'DVN', 'EMC', 'ETR', 'EXC', 'F', 'FDX', 'GD', 'GE', 'GILD', 'GOOG', 'GS', 'HAL', 'HD', 'HNZ', 'HON', 'HPQ', 'IBM', 'INTC', 'JNJ', 'JPM', 'KFT', 'KO', 'LMT', 'LOW', 'MA', 'MCD', 'MDT', 'MMM', 'MO', 'MRK', 'MS', 'MSFT', 'NKE', 'NOV', 'NSC', 'NYX', 'ORCL', 'OXY', 'PEP', 'PFE', 'PG', 'PM', 'QCOM', 'RF', 'RTN', 'S', 'SGP', 'SLB', 'SLE', 'SO', 'T', 'TGT', 'TWX', 'TXN', 'TYC', 'UNH', 'UPS', 'USB', 'UTX', 'VZ', 'WAG', 'WFC', 'WMB', 'WMT', 'WY', 'WYE', 'XOM', 'XRX']

''' Create norgate object and query it for stock data '''
norgateObj = da.DataAccess('Yahoo')

lsAll = norgateObj.get_all_symbols()
intersect = set(lsAll) & set(lsSymbols)

if len(intersect) < len(lsSymbols):
    print "Warning: S&P100 contains symbols that do not exist: ", 
    print set(lsSymbols) - intersect 
    
    lsSymbols = sort(list( intersect )) 

''''Read in historical data'''
lYear = 2009
dtEnd = dt.datetime(lYear+1,1,1) 
dtStart = dtEnd - dt.timedelta(days=365) 
dtTest = dtEnd + dt.timedelta(days=365) 
timeofday=dt.timedelta(hours=16)

ldtTimestamps = du.getNYSEdays( dtStart, dtEnd, timeofday )
ldtTimestampTest = du.getNYSEdays( dtEnd, dtTest, timeofday )

dmClose = norgateObj.get_data(ldtTimestamps, lsSymbols, "close")
dmTest = norgateObj.get_data(ldtTimestampTest, lsSymbols, "close")

naData = dmClose.values.copy()
naDataTest = dmTest.values.copy()<commit_msg>Updated the S&P100 to current list<commit_after>'''
(c) 2011, 2012 Georgia Tech Research Corporation
This source code is released under the New BSD license.  Please see
http://wiki.quantsoftware.org/index.php?title=QSTK_License
for license details.

Created on April 20, 2012

@author: <NAME>
@contact: <EMAIL>
@summary: Demonstrates the use of the CVXOPT portfolio optimization call.
'''

import qstkutil.qsdateutil as du
import qstkutil.tsutil as tsu
import qstkutil.DataAccess as da
import datetime as dt

import matplotlib.pyplot as plt
from pylab import *
from math import isnan
from copy import copy

''' Function gets a 100 sample point frontier for given returns '''
def getFrontier(naData):
    ''' Special case for fTarget = None, just get average returns '''
    (naAvgRets,naStd, b_error) = tsu.OptPort( naData, None )

    naLower = np.zeros(naData.shape[1])
    naUpper = np.ones(naData.shape[1])
    
    (fMin, fMax) = tsu.getRetRange( naData, naLower, naUpper, naAvgRets, s_type="long")
    
    fStep = (fMax - fMin) / 100.0
    
    lfReturn =  [fMin + x * fStep for x in range(101)]
    lfStd = []
    lnaPortfolios = []
    
    ''' Call the function 100 times for the given range '''
    for fTarget in lfReturn: 
        (naWeights, fStd, b_error) = tsu.OptPort( naData, fTarget, naLower, naUpper, s_type = "long")
        #if b_error == False:
        lfStd.append(fStd)
        lnaPortfolios.append( naWeights )
        #lfReturn.pop(lfReturn.index(fTarget))
    return (lfReturn, lfStd, lnaPortfolios, naAvgRets, naStd)
    


''' ******************************************************* '''
''' ******************** MAIN SCRIPT ********************** '''
''' ******************************************************* '''

''' S&P100 '''

lsSymbols = ['AAPL', 'ABT', 'ACN', 'AEP', 'ALL', 'AMGN', 'AMZN', 'APC', 'AXP', 'BA', 'BAC', 'BAX', 'BHI', 'BK', 'BMY', 'BRK.B', 'CAT', 'C', 'CL', 'CMCSA', 'COF', 'COP', 'COST', 'CPB', 'CSCO', 'CVS', 'CVX', 'DD', 'DELL', 'DIS', 'DOW', 'DVN', 'EBAY', 'EMC', 'EXC', 'F', 'FCX', 'FDX', 'GD', 'GE', 'GILD', 'GOOG', 'GS', 'HAL', 'HD', 'HNZ', 'HON', 'HPQ', 'IBM', 'INTC', 'JNJ', 'JPM', 'KFT', 'KO', 'LLY', 'LMT', 'LOW', 'MA', 'MCD', 'MDT', 'MET', 'MMM', 'MO', 'MON', 'MRK', 'MS', 'MSFT', 'NKE', 'NOV', 'NSC', 'NWSA', 'NYX', 'ORCL', 'OXY', 'PEP', 'PFE', 'PG', 'PM', 'QCOM', 'RF', 'RTN', 'SBUX', 'SLB', 'HSH', 'SO', 'SPG', 'T', 'TGT', 'TWX', 'TXN', 'UNH', 'UPS', 'USB', 'UTX', 'VZ', 'WAG', 'WFC', 'WMB', 'WMT', 'XOM']

''' Create norgate object and query it for stock data '''
norgateObj = da.DataAccess('Yahoo')

lsAll = norgateObj.get_all_symbols()
intersect = set(lsAll) & set(lsSymbols)

if len(intersect) < len(lsSymbols):
    print "Warning: S&P100 contains symbols that do not exist: ", 
    print set(lsSymbols) - intersect 
    
    lsSymbols = sort(list( intersect )) 

''''Read in historical data'''
lYear = 2009
dtEnd = dt.datetime(lYear+1,1,1) 
dtStart = dtEnd - dt.timedelta(days=365) 
dtTest = dtEnd + dt.timedelta(days=365) 
timeofday=dt.timedelta(hours=16)

ldtTimestamps = du.getNYSEdays( dtStart, dtEnd, timeofday )
ldtTimestampTest = du.getNYSEdays( dtEnd, dtTest, timeofday )

dmClose = norgateObj.get_data(ldtTimestamps, lsSymbols, "close")
dmTest = norgateObj.get_data(ldtTimestampTest, lsSymbols, "close")

naData = dmClose.values.copy()
naDataTest = dmTest.values.copy()
{'fraction_non_alphanumeric': 0.22502250225022502, 'fraction_numerical': 0.015451545154515451, 'mean_word_length': 4.514474772539288, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7361429', 'n_tokens_mistral': 2683, 'n_tokens_neox': 2533, 'n_words': 678}