<commit_before>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml;

namespace BrightWire.Connectionist.Training.Helper
{
    internal class RecurrentContext : IRecurrentTrainingContext
    {
        readonly ILinearAlgebraProvider _lap;
        readonly ITrainingContext _trainingContext;
        readonly List<INeuralNetworkRecurrentTrainerFilter> _filter = new List<INeuralNetworkRecurrentTrainerFilter>();

        public RecurrentContext(ILinearAlgebraProvider lap, ITrainingContext trainingContext)
        {
            _lap = lap;
            _trainingContext = trainingContext;
        }

        public ITrainingContext TrainingContext { get { return _trainingContext; } }

        public void AddFilter(INeuralNetworkRecurrentTrainerFilter filter)
        {
            _filter.Add(filter);
        }

        IMatrix _CreateMemory(float[] data, int columns)
        {
            return _lap.Create(columns, data.Length, (x, y) => data[y]);
        }

        public void ExecuteForward(ISequentialMiniBatch miniBatch, float[] memory, Action<int, List<IMatrix>> onT)
        {
            var sequenceLength = miniBatch.SequenceLength;
            var context = new List<IMatrix>();

            // initialise context with initial memory and space for the input at position 0
            context.Add(null);
            context.Add(_CreateMemory(memory, miniBatch.BatchSize));

            for (var k = 0; k < sequenceLength; k++) {
                // push the sequence into context
                context[0] = miniBatch.Input[k];

                // apply any filters
                foreach (var filter in _filter)
                    filter.BeforeFeedForward(miniBatch, k, context);

                onT(k, context);
            }
        }

        //public void ExecuteForwardSingle(Tuple<float[], float[]>[] data, float[] memory, int dataIndex, Action<int, List<IMatrix>> onT)
        //{
        //    var context = new List<IMatrix>();
        //    context.Add(null);
        //    context.Add(_lap.Create(memory).ToRowMatrix());

        //    var dataIndexArray = new[] { dataIndex };
        //    for (var k = 0; k < data.Length; k++) {
        //        // push the sequence into context
        //        context[0] = _lap.Create(data[k].Item1).ToRowMatrix();

        //        // apply any filters
        //        foreach (var filter in _filter)
        //            filter.BeforeFeedForward(dataIndexArray, k, data.Length, context);

        //        onT(k, context);
        //    }
        //}

        void _Add(Dictionary<Tuple<int, bool>, List<INeuralNetworkRecurrentBackpropagation>> table, Tuple<int, bool> key, INeuralNetworkRecurrentBackpropagation bp)
        {
            List<INeuralNetworkRecurrentBackpropagation> temp;
            if (!table.TryGetValue(key, out temp))
                table.Add(key, temp = new List<INeuralNetworkRecurrentBackpropagation>());
            temp.Add(bp);
        }

        public void ExecuteBidirectional(
            ISequentialMiniBatch miniBatch, 
            IReadOnlyList<INeuralNetworkBidirectionalLayer> layers, 
            float[] memoryForward,
            float[] memoryBackward,
            int padding,
            Stack<Tuple<Stack<Tuple<INeuralNetworkRecurrentBackpropagation, INeuralNetworkRecurrentBackpropagation>>, IMatrix, IMatrix, ISequentialMiniBatch, int>> updateStack,
            Action<List<IIndexableVector[]>, List<IMatrix>> onFinished
        ){
            IMatrix temp;
            var layerStackTable = new Dictionary<Tuple<int, bool>, List<INeuralNetworkRecurrentBackpropagation>>();
            var memoryOutput = new Dictionary<Tuple<int, bool>, IMatrix>();
            var garbage = new List<IMatrix>();

            // init
            var sequenceLength = miniBatch.SequenceLength;
            var batchSize = miniBatch.BatchSize;
            var shouldBackpropagate = updateStack != null;
            var lastRecurrentLayer = layers.Where(l => l.Forward.IsRecurrent).Last();

            // create the forward context
            var forwardContext = new List<IMatrix>();
            forwardContext.Add(null);
            forwardContext.Add(_CreateMemory(memoryForward, batchSize));

            // create the backward context
            var backwardContext = new List<IMatrix>();
            backwardContext.Add(null);
            backwardContext.Add(_CreateMemory(memoryBackward, batchSize));

            // load the first inputs
            var input = new Dictionary<Tuple<int, bool>, IMatrix>();
            for (var k = 0; k < sequenceLength; k++) {<commit_msg>CreateMemory is now public on RecurrentContext<commit_after>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml;

namespace BrightWire.Connectionist.Training.Helper
{
    internal class RecurrentContext : IRecurrentTrainingContext
    {
        readonly ILinearAlgebraProvider _lap;
        readonly ITrainingContext _trainingContext;
        readonly List<INeuralNetworkRecurrentTrainerFilter> _filter = new List<INeuralNetworkRecurrentTrainerFilter>();

        public RecurrentContext(ILinearAlgebraProvider lap, ITrainingContext trainingContext)
        {
            _lap = lap;
            _trainingContext = trainingContext;
        }

        public ITrainingContext TrainingContext { get { return _trainingContext; } }

        public void AddFilter(INeuralNetworkRecurrentTrainerFilter filter)
        {
            _filter.Add(filter);
        }

        public IMatrix CreateMemory(float[] data, int columns)
        {
            return _lap.Create(columns, data.Length, (x, y) => data[y]);
        }

        public void ExecuteForward(ISequentialMiniBatch miniBatch, float[] memory, Action<int, List<IMatrix>> onT)
        {
            var sequenceLength = miniBatch.SequenceLength;
            var context = new List<IMatrix>();

            // initialise context with initial memory and space for the input at position 0
            context.Add(null);
            context.Add(CreateMemory(memory, miniBatch.BatchSize));

            for (var k = 0; k < sequenceLength; k++) {
                // push the sequence into context
                context[0] = miniBatch.Input[k];

                // apply any filters
                foreach (var filter in _filter)
                    filter.BeforeFeedForward(miniBatch, k, context);

                onT(k, context);
            }
        }

        //public void ExecuteForwardSingle(Tuple<float[], float[]>[] data, float[] memory, int dataIndex, Action<int, List<IMatrix>> onT)
        //{
        //    var context = new List<IMatrix>();
        //    context.Add(null);
        //    context.Add(_lap.Create(memory).ToRowMatrix());

        //    var dataIndexArray = new[] { dataIndex };
        //    for (var k = 0; k < data.Length; k++) {
        //        // push the sequence into context
        //        context[0] = _lap.Create(data[k].Item1).ToRowMatrix();

        //        // apply any filters
        //        foreach (var filter in _filter)
        //            filter.BeforeFeedForward(dataIndexArray, k, data.Length, context);

        //        onT(k, context);
        //    }
        //}

        void _Add(Dictionary<Tuple<int, bool>, List<INeuralNetworkRecurrentBackpropagation>> table, Tuple<int, bool> key, INeuralNetworkRecurrentBackpropagation bp)
        {
            List<INeuralNetworkRecurrentBackpropagation> temp;
            if (!table.TryGetValue(key, out temp))
                table.Add(key, temp = new List<INeuralNetworkRecurrentBackpropagation>());
            temp.Add(bp);
        }

        public void ExecuteBidirectional(
            ISequentialMiniBatch miniBatch, 
            IReadOnlyList<INeuralNetworkBidirectionalLayer> layers, 
            float[] memoryForward,
            float[] memoryBackward,
            int padding,
            Stack<Tuple<Stack<Tuple<INeuralNetworkRecurrentBackpropagation, INeuralNetworkRecurrentBackpropagation>>, IMatrix, IMatrix, ISequentialMiniBatch, int>> updateStack,
            Action<List<IIndexableVector[]>, List<IMatrix>> onFinished
        ){
            IMatrix temp;
            var layerStackTable = new Dictionary<Tuple<int, bool>, List<INeuralNetworkRecurrentBackpropagation>>();
            var memoryOutput = new Dictionary<Tuple<int, bool>, IMatrix>();
            var garbage = new List<IMatrix>();

            // init
            var sequenceLength = miniBatch.SequenceLength;
            var batchSize = miniBatch.BatchSize;
            var shouldBackpropagate = updateStack != null;
            var lastRecurrentLayer = layers.Where(l => l.Forward.IsRecurrent).Last();

            // create the forward context
            var forwardContext = new List<IMatrix>();
            forwardContext.Add(null);
            forwardContext.Add(CreateMemory(memoryForward, batchSize));

            // create the backward context
            var backwardContext = new List<IMatrix>();
            backwardContext.Add(null);
            backwardContext.Add(CreateMemory(memoryBackward, batchSize));

            // load the first inputs
            var input = new Dictionary<Tuple<int, bool>, IMatrix>();
            for (var k = 0; k < sequenceLength; k++) {
{'fraction_non_alphanumeric': 0.09707503179313268, 'fraction_numerical': 0.001483679525222552, 'mean_word_length': 2.2597582037996546, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7143458', 'n_tokens_mistral': 2506, 'n_tokens_neox': 2384, 'n_words': 580}