<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BrightWire.ExecutionGraph;
using BrightWire.Models;
using ProtoBuf;

namespace BrightWire.SampleCode
{
	public partial class Program
	{
		/// <summary>
		/// Uses a recurrent LSTM neural network to predict stock price movements
		/// Data can be downloaded from https://raw.githubusercontent.com/plotly/datasets/master/stockdata.csv
		/// </summary>
		static void StockData(string dataFilePath)
		{
			// load and normalise the data
			var dataSet = new StreamReader(dataFilePath).ParseCSV(',', true);
			var normalised = dataSet.Normalise(NormalisationType.FeatureScale);
			var rows = normalised.GetNumericRows(dataSet.Columns.Where(c => c.Name != "Date").Select(c => c.Index));

			// build the data table with a window of input data and the prediction as the following value
			var builder = BrightWireProvider.CreateDataTableBuilder();
			builder.AddColumn(ColumnType.Matrix, "Past");
			builder.AddColumn(ColumnType.Vector, "Future");
			const int LAST_X_DAYS = 14;
			for (var i = 0; i < rows.Count - LAST_X_DAYS - 1; i++) {
				var inputVector = new List<FloatVector>();
				for (var j = 0; j < LAST_X_DAYS; j++)
					inputVector.Add(FloatVector.Create(rows[i + j]));
				var input = FloatMatrix.Create(inputVector.ToArray());
				var target = FloatVector.Create(rows[i+1]);
				builder.Add(input, target);
			}
			var data = builder.Build().Split(trainingPercentage: 0.2);

			using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) {
				var graph = new GraphFactory(lap);
				var errorMetric = graph.ErrorMetric.Quadratic;

				// create the property set
				graph.CurrentPropertySet
					.Use(graph.GradientDescent.Adam)
					.Use(graph.WeightInitialisation.Xavier);

				// create the engine
				var trainingData = graph.CreateDataSource(data.Training);
				var testData = trainingData.CloneWith(data.Test);
				var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.03f, batchSize: 128);
<commit_msg>fixed array indexing typo and changed to cpu based training<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BrightWire.ExecutionGraph;
using BrightWire.Models;
using ProtoBuf;

namespace BrightWire.SampleCode
{
	public partial class Program
	{
		/// <summary>
		/// Uses a recurrent LSTM neural network to predict stock price movements
		/// Data can be downloaded from https://raw.githubusercontent.com/plotly/datasets/master/stockdata.csv
		/// </summary>
		static void StockData(string dataFilePath)
		{
			// load and normalise the data
			var dataSet = new StreamReader(dataFilePath).ParseCSV(',', true);
			var normalised = dataSet.Normalise(NormalisationType.FeatureScale);
			var rows = normalised.GetNumericRows(dataSet.Columns.Where(c => c.Name != "Date").Select(c => c.Index));

			// build the data table with a window of input data and the prediction as the following value
			var builder = BrightWireProvider.CreateDataTableBuilder();
			builder.AddColumn(ColumnType.Matrix, "Past");
			builder.AddColumn(ColumnType.Vector, "Future");
			const int LAST_X_DAYS = 14;
			for (var i = 0; i < rows.Count - LAST_X_DAYS - 1; i++) {
				var inputVector = new List<FloatVector>();
				for (var j = 0; j < LAST_X_DAYS; j++)
					inputVector.Add(FloatVector.Create(rows[i + j]));
				var input = FloatMatrix.Create(inputVector.ToArray());
				var target = FloatVector.Create(rows[i + LAST_X_DAYS + 1]);
				builder.Add(input, target);
			}
			var data = builder.Build().Split(trainingPercentage: 0.2);

			using (var lap = BrightWireProvider.CreateLinearAlgebra()) {
				var graph = new GraphFactory(lap);
				var errorMetric = graph.ErrorMetric.Quadratic;

				// create the property set
				graph.CurrentPropertySet
					.Use(graph.GradientDescent.Adam)
					.Use(graph.WeightInitialisation.Xavier);

				// create the engine
				var trainingData = graph.CreateDataSource(data.Training);
				var testData = trainingData.CloneWith(data.Test);
				var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.03f, batchSize: 128);

{'fraction_non_alphanumeric': 0.114413776834159, 'fraction_numerical': 0.006605331446095777, 'mean_word_length': 4.108433734939759, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2095455', 'n_tokens_mistral': 1471, 'n_tokens_neox': 1254, 'n_words': 335}