<commit_before>import traceback
import json
import imp
import urllib
import urllib2
import webbrowser
import re
import datetime
import time
import inspect
import os
import os.path
import sys
import ssl
from copy import deepcopy

import pandas as pd
import numpy as np

import matplotlib
matplotlib.use("TkAgg")
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib import style
import matplotlib.pyplot as plt

import Tkinter as tk
import ttk


def loadData(marketList=None, dataToLoad=None, refresh=False, beginInSample=None, endInSample=None, dataDir = 'tickerData'):
    ''' prepares and returns market data for specified markets.

        prepares and returns related to the entries in the dataToLoad list. When refresh is true, data is updated from the Quantiacs server. If inSample is left as none, all available data dates will be returned.

        Args:
            marketList (list): list of market data to be supplied
            dataToLoad (list): list of financial data types to load
            refresh (bool): boolean value determining whether or not to update the local data from the Quantiacs server.
            beginInSample (str): a str in the format of YYYYMMDD defining the begining of the time series
            endInSample (str): a str in the format of YYYYMMDD defining the end of the time series

        Returns:
            dataDict (dict): mapping all data types requested by dataToLoad. The data is returned as a numpy array or list and is ordered by marketList along columns and date along the row.

    Copyright Quantiacs LLC - March 2015
    '''
    if marketList is None:
        print "warning: no markets supplied"
        return

    dataToLoad = set(dataToLoad)
    requiredData = set(['DATE', 'OPEN', 'HIGH', 'LOW', 'CLOSE', 'P', 'RINFO', 'p'])
    fieldNames = set()

    dataToLoad.update(requiredData)

    nMarkets = len(marketList)

    # set up data director
    if not os.path.isdir(dataDir):
        os.mkdir(dataDir)

    for j in range(nMarkets):
        path = os.path.join(dataDir, marketList[j]+'.txt')

        # check to see if market data is present. If not (or refresh is true), download data from quantiacs.
        if not os.path.isfile(path) or refresh:
            try:
                if False: #sys.version_info > (2,7,9):
                    gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                    data = urllib.urlopen('https://www.quantiacs.com/data/' +
                                          marketList[j]+'.txt',
                                          context=gcontext).read()
                else:
                    data = urllib.urlopen('https://www.quantiacs.com/data/' +
                                          marketList[j]+'.txt').read()

                with open(path, 'w') as dataFile:
                    dataFile.write(data)
                print 'Downloading ' + marketList[j]

            except:
                print 'Unable to download ' + marketList[j]
                marketList.remove(marketList[j])

    print 'Loading Data...'
    sys.stdout.flush()
    dataDict = {}
    largeDateRange = range(datetime.datetime(1990, 1, 1).toordinal(),
                           datetime.datetime.today().toordinal() + 1)
    DATE_Large = [int(datetime.datetime.fromordinal(j).strftime('%Y%m%d')) for j in largeDateRange]


    # Loading all markets into memory.
    for i, market in enumerate(marketList):
        marketFile = os.path.join('tickerData', market+'.txt')
        data = pd.read_csv(marketFile, engine='c')
        data.columns = map(str.strip, data.columns)
        fieldNames.update(list(data.columns.values))
        data.set_index('DATE', inplace=True)
        data['DATE'] = data.index

        for j, dataType in enumerate(dataToLoad):
            if dataType == 'p':
                data.rename(columns={'p':'P'},inplace = True)
                dataType = 'P'

            if dataType != 'DATE' and dataType not in dataDict and dataType in data:
                dataDict[dataType] = pd.DataFrame(index=DATE_Large, columns=marketList)
                dataDict[dataType][market] = data[dataType]

            elif dataType != 'DATE' and dataType in data:
                dataDict[dataType][market] = data[dataType]

    # get args that are not in requiredData and fieldsNames
    # find out what has not been loaded
    additionDataToLoad = dataToLoad.difference(requiredData.union(fieldNames))
    for i, additionData in enumerate(additionDataToLoad):
        dataFile = os.path.join('tickerData', additionData + '.txt')
        data = pd.read_csv(dataFile, engine='c')
        data.columns = map(str.strip, data.columns)
        data.set_index('DATE', inplace=True)
        data['DATE'] = data.index

        for j, column in enumerate(data.columns):
            if column != 'DATE':
                if additionData not in dataDict:
                    columns = set(data.columns)
                    columns.remove('DATE')
                    dataDict[additionData] = pd.DataFrame(index=DATE_Large, columns=columns)
                dataDict[additionData][column] = data[column]

    # fill the gap for additional data for the whole data range
    for i, additionData in enumerate(additionDataToLoad):
        dataDict[additionData][:] = fillnans(dataDict[additionData].values)

    # drop rows in CLOSE if none of the markets have data on that date
    dataDict['CLOSE'].dropna(how='all', inplace=True)

    # In-sample date management.
    if beginInSample is not None:
        beginInSample = datetime.datetime.strptime(beginInSample, '%Y%m%d')
    else:
        beginInSample = datetime.datetime(1990, 1, 1)
    beginInSampleInt = int(beginInSample.strftime('%Y%m%d'))

    if endInSample is not None:
        endInSample = datetime.datetime.strptime(endInSample, '%Y%m%d')
        endInSampleInt = int(endInSample.strftime('%Y%m%d'))
        dataDict['DATE'] = dataDict['CLOSE'].loc[beginInSampleInt:endInSampleInt, :].index.values
    else:
        dataDict['DATE'] = dataDict['CLOSE'].loc[beginInSampleInt:, :].index.values

    for index, dataType in enumerate(dataToLoad):
        if dataType != 'DATE' and dataType in dataDict:
            dataDict[dataType] = dataDict[dataType].loc[dataDict['DATE'], :]
            dataDict[dataType] = dataDict[dataType].values
<commit_msg>loading data from server for indicators<commit_after>import traceback
import json
import imp
import urllib
import urllib2
import webbrowser
import re
import datetime
import time
import inspect
import os
import os.path
import sys
import ssl
from copy import deepcopy

import pandas as pd
import numpy as np

import matplotlib
matplotlib.use("TkAgg")
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib import style
import matplotlib.pyplot as plt

import Tkinter as tk
import ttk


def loadData(marketList=None, dataToLoad=None, refresh=False, beginInSample=None, endInSample=None, dataDir = 'tickerData'):
    ''' prepares and returns market data for specified markets.

        prepares and returns related to the entries in the dataToLoad list. When refresh is true, data is updated from the Quantiacs server. If inSample is left as none, all available data dates will be returned.

        Args:
            marketList (list): list of market data to be supplied
            dataToLoad (list): list of financial data types to load
            refresh (bool): boolean value determining whether or not to update the local data from the Quantiacs server.
            beginInSample (str): a str in the format of YYYYMMDD defining the begining of the time series
            endInSample (str): a str in the format of YYYYMMDD defining the end of the time series

        Returns:
            dataDict (dict): mapping all data types requested by dataToLoad. The data is returned as a numpy array or list and is ordered by marketList along columns and date along the row.

    Copyright Quantiacs LLC - March 2015
    '''
    if marketList is None:
        print "warning: no markets supplied"
        return

    dataToLoad = set(dataToLoad)
    requiredData = set(['DATE', 'OPEN', 'HIGH', 'LOW', 'CLOSE', 'P', 'RINFO', 'p'])
    fieldNames = set()

    dataToLoad.update(requiredData)

    nMarkets = len(marketList)

    # set up data director
    if not os.path.isdir(dataDir):
        os.mkdir(dataDir)

    for j in range(nMarkets):
        path = os.path.join(dataDir, marketList[j]+'.txt')

        # check to see if market data is present. If not (or refresh is true), download data from quantiacs.
        if not os.path.isfile(path) or refresh:
            try:
                if False: #sys.version_info > (2,7,9):
                    gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                    data = urllib.urlopen('https://www.quantiacs.com/data/' +
                                          marketList[j]+'.txt',
                                          context=gcontext).read()
                else:
                    data = urllib.urlopen('https://www.quantiacs.com/data/' +
                                          marketList[j]+'.txt').read()
                with open(path, 'w') as dataFile:
                    dataFile.write(data)
                print 'Downloading ' + marketList[j]

            except:
                print 'Unable to download ' + marketList[j]
                marketList.remove(marketList[j])
            finally:
                dataFile.close()

    print 'Loading Data...'
    sys.stdout.flush()
    dataDict = {}
    largeDateRange = range(datetime.datetime(1990, 1, 1).toordinal(),
                           datetime.datetime.today().toordinal() + 1)
    DATE_Large = [int(datetime.datetime.fromordinal(j).strftime('%Y%m%d')) for j in largeDateRange]


    # Loading all markets into memory.
    for i, market in enumerate(marketList):
        marketFile = os.path.join('tickerData', market+'.txt')
        data = pd.read_csv(marketFile, engine='c')
        data.columns = map(str.strip, data.columns)
        fieldNames.update(list(data.columns.values))
        data.set_index('DATE', inplace=True)    
        data['DATE'] = data.index

        for j, dataType in enumerate(dataToLoad):
            if dataType == 'p':
                data.rename(columns={'p':'P'},inplace = True)
                dataType = 'P'

            if dataType != 'DATE' and dataType not in dataDict and dataType in data:
                dataDict[dataType] = pd.DataFrame(index=DATE_Large, columns=marketList)
                dataDict[dataType][market] = data[dataType]

            elif dataType != 'DATE' and dataType in data:
                dataDict[dataType][market] = data[dataType]


    # get args that are not in requiredData and fieldsNames
    additionDataToLoad = dataToLoad.difference(requiredData.union(fieldNames))
    additionDataFailed = set([])
    for i, additionData in enumerate(additionDataToLoad):
        filePath = os.path.join('tickerData', additionData + '.txt')
        # check to see if data is present. If not (or refresh is true), download data from quantiacs.
        if not os.path.isfile(filePath) or refresh:
            try:
                if False: #sys.version_info > (2,7,9):
                    gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                    data = urllib.urlopen('https://www.quantiacs.com/data/' +
                                          additionData+'.txt',
                                          context=gcontext).read()
                else:
                    data = urllib.urlopen('https://www.quantiacs.com/data/' +
                                          additionData+'.txt').read()
                with open(filePath, 'w') as dataFile:
                    dataFile.write(data)
                print 'Downloading ' + additionData
            except:
                print 'Unable to download ' + additionData
                additionDataFailed.add(additionData)
                continue
            finally:
                dataFile.close()

        # read data from text file and load to memory
        data = pd.read_csv(filePath, engine='c')
        data.columns = map(str.strip, data.columns)
        data.set_index('DATE', inplace=True)
        data['DATE'] = data.index
        for j, column in enumerate(data.columns):
            if column != 'DATE':
                if additionData not in dataDict:
                    columns = set(data.columns)
                    columns.remove('DATE')
                    dataDict[additionData] = pd.DataFrame(index=DATE_Large, columns=columns)
                dataDict[additionData][column] = data[column]

    additionDataToLoad = additionDataToLoad.difference(additionDataFailed)
    # fill the gap for additional data for the whole data range
    for i, additionData in enumerate(additionDataToLoad):
        dataDict[additionData][:] = fillnans(dataDict[additionData].values)

    # drop rows in CLOSE if none of the markets have data on that date
    dataDict['CLOSE'].dropna(how='all', inplace=True)

    # In-sample date management.
    if beginInSample is not None:
        beginInSample = datetime.datetime.strptime(beginInSample, '%Y%m%d')
    else:
        beginInSample = datetime.datetime(1990, 1, 1)
    beginInSampleInt = int(beginInSample.strftime('%Y%m%d'))

    if endInSample is not None:
        endInSample = datetime.datetime.strptime(endInSample, '%Y%m%d')
        endInSampleInt = int(endInSample.strftime('%Y%m%d'))
        dataDict['DATE'] = dataDict['CLOSE'].loc[beginInSampleInt:endInSampleInt, :].index.values
    else:
        dataDict['DATE'] = dataDict['CLOSE'].loc[beginInSampleInt:, :].index.values

    for index, dataType in enumerate(dataToLoad):
        if dataType != 'DATE' and dataType in dataDict:
            dataDict[dataType] = dataDict[dataType].loc[dataDict['DATE'], :]
            dataDict[dataType] = dataDict[dataType].values

{'fraction_non_alphanumeric': 0.09840195796141664, 'fraction_numerical': 0.0034552260293694214, 'mean_word_length': 2.3794697153977133, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4024355', 'n_tokens_mistral': 3974, 'n_tokens_neox': 3785, 'n_words': 1137}