<commit_before>﻿using System;
using System.Diagnostics;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Utils.DataStructures;
using Utils.Other;

namespace UtilsTests.SplayTree
{
    internal class MyCommandGenerator
    {
        private enum CommandState
        {
            Init,
            Starts,
            Inserts,
            Finds,
        }


        #region Constants

        private const int TimeOut = 6000000; // 6000 seconds = 100 minutes timeout
        private const int BuilderInitSize = 8000;
        private const int BuilderSizeIncrement = 3000;
        private const int Seed = 82;

        private const string ToolsPath = @"Tools";
        private const string GeneratorName = "generator.exe";

        private string FolderPath
        {
            get { return AppDomain.CurrentDomain.BaseDirectory; }
        }

        #endregion

        #region Fields

        private readonly AsyncBuffer<Stack<int>> _buffer;
        private readonly CancellationTokenSource _cancellationTokenSource;

        private CommandState _state = CommandState.Init;
        private Stack<int> _currentCommands = new Stack<int>(BuilderInitSize);

        #endregion

        #region Genesis

        public MyCommandGenerator(AsyncBuffer<Stack<int>> buffer, CancellationTokenSource cts)
        {
            _buffer = buffer;
            _cancellationTokenSource = cts;
        }

        #endregion

        #region Generating

        private void DataReceivedHandler(object sender, DataReceivedEventArgs e)
        {
            if (e.Data == null)
                return;

            try
            {
                switch (e.Data[0])
                {
                    case '#':
                        Debug.Assert(_state == CommandState.Init || _state == CommandState.Finds); // A race condition -- another DataReceivedHandler changed the state

                        // Replace the builder by a new instance
                        Stack<int> commands = _currentCommands;

                        if (_state != CommandState.Init)
                        {
                            _buffer.Add(commands);

                            _currentCommands = new Stack<int>(commands.Count + BuilderSizeIncrement);
                        }

                        // Start gathering a new batch
                        _state = CommandState.Starts;
                        break;

                    case 'I':
                        Debug.Assert(_state == CommandState.Inserts || _state == CommandState.Starts); // A race condition -- another DataReceivedHandler changed the state

                        if (_state == CommandState.Starts)
                            _state = CommandState.Inserts;
                        break;

                    case 'F':
                        Debug.Assert(_state == CommandState.Finds || _state == CommandState.Inserts); // A race condition -- another DataReceivedHandler changed the state

                        if (_state == CommandState.Inserts)
                            _state = CommandState.Finds;
                        break;
                }

                _currentCommands.Push(int.Parse(e.Data.Substring(2)));
            }
            catch (Exception ex)
            {
                _cancellationTokenSource.Cancel();
                Console.WriteLine("Error in parsing generator data:\n" + ex.Message);
            }
        }

        public Task RunGeneratorSubset(int t)
        {
            Debug.Assert(t > 0);

            var pars = string.Format("-s {0} -t {1}", Seed, t);
            return RunGenerator(pars);
        }

        public Task RunGeneratorSequential()
        {
            var pars = string.Format("-s {0} -b", Seed);
            return RunGenerator(pars);
        }

        private async Task RunGenerator(string pars)
        {
            using (var generatorProcess = new ProcessHelper(
                Console.WriteLine,
                DataReceivedHandler,
                (sender, args) => Thread.CurrentThread.Abort(),
                null))
            {
                generatorProcess.StartProcess(
                    Path.Combine(ToolsPath, GeneratorName),
                    pars,
                    FolderPath,
                    _cancellationTokenSource.Token);

                var result = await generatorProcess.Wait(TimeOut).ConfigureAwait(false);

                if (result != WaitResult.Ok)
                    _cancellationTokenSource.Cancel();

                Assert.IsTrue(result.HasFlag(WaitResult.Ok), "Generator process error: " + result);
                Console.WriteLine("Generator finished. Running time: " + generatorProcess.GetElapsedTime());
            }
        }

        #endregion
    }
}<commit_msg>@ Fixed adding of the last generated command list<commit_after>﻿using System;
using System.Diagnostics;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Utils.DataStructures;
using Utils.Other;

namespace UtilsTests.SplayTree
{
    internal class MyCommandGenerator
    {
        private enum CommandState
        {
            Init,
            Starts,
            Inserts,
            Finds,
        }


        #region Constants

        private const int TimeOut = 6000000; // 6000 seconds = 100 minutes timeout
        private const int BuilderInitSize = 8000;
        private const int BuilderSizeIncrement = 3000;
        private const int Seed = 82;

        private const string ToolsPath = @"Tools";
        private const string GeneratorName = "generator.exe";

        private string FolderPath
        {
            get { return AppDomain.CurrentDomain.BaseDirectory; }
        }

        #endregion

        #region Fields

        private readonly AsyncBuffer<Stack<int>> _buffer;
        private readonly CancellationTokenSource _cancellationTokenSource;

        private CommandState _state = CommandState.Init;
        private Stack<int> _currentCommands = new Stack<int>(BuilderInitSize);

        #endregion

        #region Genesis

        public MyCommandGenerator(AsyncBuffer<Stack<int>> buffer, CancellationTokenSource cts)
        {
            _buffer = buffer;
            _cancellationTokenSource = cts;
        }

        #endregion

        #region Generating

        private void DataReceivedHandler(object sender, DataReceivedEventArgs e)
        {
            if (e.Data == null)
                return;

            try
            {
                switch (e.Data[0])
                {
                    case '#':
                        Debug.Assert(_state == CommandState.Init || _state == CommandState.Finds); // A race condition -- another DataReceivedHandler changed the state

                        // Replace the builder by a new instance
                        Stack<int> commands = _currentCommands;

                        if (_state != CommandState.Init)
                        {
                            _buffer.Add(commands);

                            _currentCommands = new Stack<int>(commands.Count + BuilderSizeIncrement);
                        }

                        // Start gathering a new batch
                        _state = CommandState.Starts;
                        break;

                    case 'I':
                        Debug.Assert(_state == CommandState.Inserts || _state == CommandState.Starts); // A race condition -- another DataReceivedHandler changed the state

                        if (_state == CommandState.Starts)
                            _state = CommandState.Inserts;
                        break;

                    case 'F':
                        Debug.Assert(_state == CommandState.Finds || _state == CommandState.Inserts); // A race condition -- another DataReceivedHandler changed the state

                        if (_state == CommandState.Inserts)
                            _state = CommandState.Finds;
                        break;
                }

                _currentCommands.Push(int.Parse(e.Data.Substring(2)));
            }
            catch (Exception ex)
            {
                _cancellationTokenSource.Cancel();
                Console.WriteLine("Error in parsing generator data:\n" + ex.Message);
            }
        }

        public Task RunGeneratorSubset(int t)
        {
            Debug.Assert(t > 0);

            var pars = string.Format("-s {0} -t {1}", Seed, t);
            return RunGenerator(pars);
        }

        public Task RunGeneratorSequential()
        {
            var pars = string.Format("-s {0} -b", Seed);
            return RunGenerator(pars);
        }

        private async Task RunGenerator(string pars)
        {
            using (var generatorProcess = new ProcessHelper(
                Console.WriteLine,
                DataReceivedHandler,
                (sender, args) => Thread.CurrentThread.Abort(),
                null))
            {
                generatorProcess.StartProcess(
                    Path.Combine(ToolsPath, GeneratorName),
                    pars,
                    FolderPath,
                    _cancellationTokenSource.Token);

                var result = await generatorProcess.Wait(TimeOut).ConfigureAwait(false);

                if (result != WaitResult.Ok)
                    _cancellationTokenSource.Cancel();

                if (_currentCommands != null)
                {
                    _buffer.Add(_currentCommands);
                    _currentCommands = null;
                }

                Assert.IsTrue(result.HasFlag(WaitResult.Ok), "Generator process error: " + result);
                Console.WriteLine("Generator finished. Running time: " + generatorProcess.GetElapsedTime());
            }
        }

        #endregion
    }
}
{'fraction_non_alphanumeric': 0.07776096822995461, 'fraction_numerical': 0.006051437216338881, 'mean_word_length': 1.4722014460234356, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7308760', 'n_tokens_mistral': 2386, 'n_tokens_neox': 2215, 'n_words': 568}