<commit_before># Luke
from pybrain.datasets import SupervisedDataSet, SequentialDataSet
from pybrain.tools.shortcuts import buildNetwork
from pybrain.supervised.trainers import BackpropTrainer, Trainer
from pybrain.structure.modules import TanhLayer, LinearLayer, MdrnnLayer, LSTMLayer, GaussianLayer, SoftmaxLayer
import matplotlib.pyplot as plt
import time
from normalizer import normalize
from normalizer import denormalize
from price_parsing import *
from util import *


# ticker = "GOOG"
# date = '11/24/15'

def graphNN(ticker, date, runs):

    print 'building dataset...'
    xTrain = []
    yTrain = []
    xTest = []
    yTest = []
    pred = []

    data = getStockPrices(ticker, frequency="daily", update=True)
    trainData, testData = splitByDate(data, date)
    xTrain, yTrain = preprocessStocks(trainData)
    xTest, yTest = preprocessStocks(testData)

    del xTrain[len(xTrain) - 1]
    del yTrain[len(yTrain) - 1]

    xTrain, yTrain, xTest, yTest, priceScaleFactor, timeScaleFactor = normalize(xTrain, yTrain, xTest, yTest)


    # build data set from x training data and y training data
    ds = SupervisedDataSet(1,1)
    for i in range(0, len(xTrain)):
       ds.appendLinked(xTrain[i], yTrain[i])

    # number of runs
    #runs = 5
    print 'complete.'
    print 'buidling netowrk...'
    rnn = buildNetwork(1, 3, 3, 3, 3, 3, 3, 3, 3, 1, bias=True, recurrent=True, hiddenclass=TanhLayer)
    trainer = BackpropTrainer(rnn, ds, learningrate=0.01)

    print 'complete'
    EpochNumber = []
    epochTimes = []
    ErrorValues = []
    percentError = []

    print 'training network...'
    totalTimeStart = time.time()
    for i in range(1,runs):
        print (str((i/(runs*1.0)) *100) + '% complete')
        startEpochTime = time.time()
        EpochNumber.append(i)
        ErrorValues.append(trainer.train())
        endEpochTime = time.time()
        epochTimes.append(endEpochTime - startEpochTime)

    totalTimeEnd = time.time()
    totalTime = (totalTimeEnd - totalTimeStart)

    timePerEpoch = sum(epochTimes)/len(epochTimes)

    print 'training network 100.0% complete.'
    print 'predicting...'
    # make predictions with network
    pred2 = []
    for i in xTrain:
        pred2.append(rnn.activate(i))

    for i in xTest:
        pred.append(rnn.activate(i))



    print 'predictions complete.'
    print 'generating graphs...'
    # denormalize
    xTrain, yTrain, xTest, yTest, pred, pred2 = denormalize(xTrain, yTrain, xTest, yTest, pred, pred2, priceScaleFactor, timeScaleFactor)

    # calculate percent error
    for i in range(0, len(yTest)):
        percentError.append((abs((yTest[i] - pred[i])/yTest[i]) *100))

    sumPercentError = sum(percentError)
    averageError = sumPercentError / len(percentError)

    plt.figure(1)
    #plt.xkcd()
    # plt.subplot(4, 1,1)
    # plt.plot(EpochNumber, ErrorValues, 'bo')
    # plt.xlabel('Epoch Number')
    # plt.ylabel('Error Value')

    plt.subplot(3, 1, 1)
    plt.plot(xTest, yTest, 'w-')
    plt.plot(xTest, pred, 'w--')
    plt.xlabel('xTest')
    plt.ylabel('yTest')
    ax = plt.gca()
    ax.spines['bottom'].set_color('#91A2C4')
    ax.spines['top'].set_color('#91A2C4')
    ax.spines['left'].set_color('#91A2C4')
    ax.spines['right'].set_color('#91A2C4')
    ax.tick_params(axis='both', colors='#91A2C4')
    ax.xaxis.label.set_color('#91A2C4')
    ax.yaxis.label.set_color('#91A2C4')


    plt.subplot(3, 1, 2)
    plt.plot(xTrain, yTrain, 'w-')
    plt.plot(xTrain, pred2, 'w--')
    plt.xlabel('xTrain')
    plt.ylabel('yTrain')
    ax = plt.gca()
    ax.spines['bottom'].set_color('#91A2C4')
    ax.spines['top'].set_color('#91A2C4')
    ax.spines['left'].set_color('#91A2C4')
    ax.spines['right'].set_color('#91A2C4')
    ax.tick_params(axis='both', colors='#91A2C4')
    ax.xaxis.label.set_color('#91A2C4')
    ax.yaxis.label.set_color('#91A2C4')


    plt.subplot(3, 1, 3)
    # plt.text(0.02, 0.85, 'Hello', fontsize=12)
    plt.text(0.02, 0.70, 'Number of Epochs  = ' + str(runs), fontsize=12, color='#91A2C4')
    plt.text(0.02, 0.55, 'Number of Data Points  = ' + str(len(xTrain)), fontsize=12, color='#91A2C4')
    plt.text(0.02, 0.40, 'Time per Epoch = ' + str(timePerEpoch) + 's   Total Time = ' + str(totalTime) + 's', fontsize=12, color='#91A2C4')
    plt.text(0.02, 0.25, 'Average Percent Error Value = ' + str(averageError), fontsize=12, color='#91A2C4')
    plt.text(0.02, 0.10, 'Minimum Percent Error Value = ' + str(min(percentError)), fontsize=12, color='#91A2C4')
    ax = plt.gca()
    ax.spines['bottom'].set_color('#91A2C4')
    ax.spines['top'].set_color('#91A2C4')
    ax.spines['left'].set_color('#91A2C4')
    ax.spines['right'].set_color('#91A2C4')
    ax.tick_params(axis='both', colors='#91A2C4')
    ax.xaxis.label.set_color('#91A2C4')
    ax.yaxis.label.set_color('#91A2C4')


    #plt.show()
    plt.savefig(ticker+'.png', transparent=True)
    print 'graphs complete.'

    return

<commit_msg>xkcd graph #2<commit_after># Luke
from pybrain.datasets import SupervisedDataSet
from pybrain.tools.shortcuts import buildNetwork
from pybrain.supervised.trainers import BackpropTrainer
from pybrain.structure.modules import TanhLayer
import matplotlib.pyplot as plt
from normalizer import normalize
from normalizer import denormalize
from price_parsing import *
from util import *

def graphNN(ticker, date, runs):

    print 'building dataset...'
    xTrain = []
    yTrain = []
    xTest = []
    yTest = []
    pred = []

    data = getStockPrices(ticker, frequency="daily", update=True)
    trainData, testData = splitByDate(data, date)
    xTrain, yTrain = preprocessStocks(trainData)
    xTest, yTest = preprocessStocks(testData)

    del xTrain[len(xTrain) - 1]
    del yTrain[len(yTrain) - 1]

    xTrain, yTrain, xTest, yTest, priceScaleFactor, timeScaleFactor = normalize(xTrain, yTrain, xTest, yTest)


    # build data set from x training data and y training data
    ds = SupervisedDataSet(1,1)
    for i in range(0, len(xTrain)):
       ds.appendLinked(xTrain[i], yTrain[i])

    # number of runs
    #runs = 5
    print 'complete.'
    print 'buidling netowrk...'
    rnn = buildNetwork(1, 3, 3, 3, 3, 3, 3, 3, 3, 1, bias=True, recurrent=True, hiddenclass=TanhLayer)
    trainer = BackpropTrainer(rnn, ds, learningrate=0.01)

    print 'complete'
    EpochNumber = []
    epochTimes = []
    ErrorValues = []
    percentError = []

    print 'training network...'
    totalTimeStart = time.time()
    for i in range(1,runs):
        print (str((i/(runs*1.0)) *100) + '% complete')
        startEpochTime = time.time()
        EpochNumber.append(i)
        ErrorValues.append(trainer.train())
        endEpochTime = time.time()
        epochTimes.append(endEpochTime - startEpochTime)

    totalTimeEnd = time.time()
    totalTime = (totalTimeEnd - totalTimeStart)

    timePerEpoch = sum(epochTimes)/len(epochTimes)

    print 'training network 100.0% complete.'
    print 'predicting...'
    # prime the network
    for i in xTrain:
        rnn.activate(i)

    # make predictions with network
    pred2 = []
    for i in xTrain:
        pred2.append(rnn.activate(i))

    for i in xTest:
        pred.append(rnn.activate(i))



    print 'predictions complete.'
    print 'generating graphs...'
    # denormalize
    xTrain, yTrain, xTest, yTest, pred, pred2 = denormalize(xTrain, yTrain, xTest, yTest, pred, pred2, priceScaleFactor, timeScaleFactor)

    # calculate percent error
    for i in range(0, len(yTest)):
        percentError.append((abs((yTest[i] - pred[i])/yTest[i]) *100))

    sumPercentError = sum(percentError)
    averageError = sumPercentError / len(percentError)

    plt.figure(1)
    plt.xkcd()
    # plt.subplot(4, 1,1)
    # plt.plot(EpochNumber, ErrorValues, 'bo')
    # plt.xlabel('Epoch Number')
    # plt.ylabel('Error Value')

    bx = plt.subplot(3, 1, 1)
    plt.tight_layout()
    l1, = plt.plot(xTest, yTest, 'w-', label='line1')
    l2, = plt.plot(xTest, pred, 'w--', label='line2')
    plt.xlabel('Time (days)')
    plt.ylabel('Price (USD)')
    ax = plt.gca()
    box = ax.get_position()
    ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
    leg = plt.legend([l1, l2], ['Actual Values', 'Predictions'], framealpha=0, loc='center left', bbox_to_anchor=(1, 0.5), borderaxespad=0.)
    for text in leg.get_texts():
        text.set_color('#91A2C4')

    ax.spines['bottom'].set_color('#91A2C4')
    ax.spines['top'].set_color('#91A2C4')
    ax.spines['left'].set_color('#91A2C4')
    ax.spines['right'].set_color('#91A2C4')
    ax.tick_params(axis='both', colors='#91A2C4')
    ax.xaxis.label.set_color('#91A2C4')
    ax.yaxis.label.set_color('#91A2C4')


    plt.subplot(3, 1, 2)
    plt.tight_layout()
    plt.plot(xTrain, yTrain, 'w-')
    plt.plot(xTrain, pred2, 'w--')
    plt.xlabel('Time (days)')
    plt.ylabel('Price (USD)')
    ax = plt.gca()
    ax.spines['bottom'].set_color('#91A2C4')
    ax.spines['top'].set_color('#91A2C4')
    ax.spines['left'].set_color('#91A2C4')
    ax.spines['right'].set_color('#91A2C4')
    ax.tick_params(axis='both', colors='#91A2C4')
    ax.xaxis.label.set_color('#91A2C4')
    ax.yaxis.label.set_color('#91A2C4')



    plt.subplot(3, 1, 3)
    plt.tight_layout()
    # plt.text(0.02, 0.85, 'Hello', fontsize=12)
    plt.text(0.02, 0.70, 'Number of Epochs  = ' + str(runs), fontsize=12, color='#91A2C4')
    plt.text(0.02, 0.55, 'Number of Data Points  = ' + str(len(xTrain)), fontsize=12, color='#91A2C4')
    plt.text(0.02, 0.40, 'Time per Epoch = ' + str(timePerEpoch) + 's   Total Time = ' + str(totalTime) + 's', fontsize=12, color='#91A2C4')
    plt.text(0.02, 0.25, 'Average Percent Error Value = ' + str(averageError), fontsize=12, color='#91A2C4')
    plt.text(0.02, 0.10, 'Minimum Percent Error Value = ' + str(min(percentError)), fontsize=12, color='#91A2C4')
    ax = plt.gca()
    ax.spines['bottom'].set_color('#91A2C4')
    ax.spines['top'].set_color('#91A2C4')
    ax.spines['left'].set_color('#91A2C4')
    ax.spines['right'].set_color('#91A2C4')
    ax.tick_params(axis='both', colors='#91A2C4')
    ax.xaxis.label.set_color('#91A2C4')
    ax.yaxis.label.set_color('#91A2C4')


    #plt.show()
    plt.savefig(ticker + 'NN.png', transparent=True, bbox_extra_artists=(leg,), bbox_inches='tight')
    print 'graphs complete.'

    return


{'fraction_non_alphanumeric': 0.1604459524963645, 'fraction_numerical': 0.041880756180319924, 'mean_word_length': 3.4891209747606613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4207001', 'n_tokens_mistral': 4123, 'n_tokens_neox': 3838, 'n_words': 750}