<commit_before>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using RunTests.Cache;
using Newtonsoft.Json.Linq;
using RestSharp;
using System.Collections.Immutable;
using Newtonsoft.Json;
using System.Reflection;
using System.Diagnostics;

namespace RunTests
{
    internal sealed partial class Program
    {
        internal const int ExitSuccess = 0;
        internal const int ExitFailure = 1;

        internal static int Main(string[] args)
        {
            Logger.Log("RunTest command line");
            Logger.Log(string.Join(" ", args));

            var options = Options.Parse(args);
            if (options == null)
            {
                Options.PrintUsage();
                return ExitFailure;
            }

            // Setup cancellation for ctrl-c key presses
            var cts = new CancellationTokenSource();
            Console.CancelKeyPress += delegate
            {
                cts.Cancel();
            };

            return Run(options, cts.Token).GetAwaiter().GetResult();
        }

        private static async Task<int> Run(Options options, CancellationToken cancellationToken)
        {
            if (options.Timeout == null)
            {
                return await RunCore(options, cancellationToken);
            }

            var timeoutTask = Task.Delay(options.Timeout.Value);
            var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var runTask = RunCore(options, cts.Token);

            if (cancellationToken.IsCancellationRequested)
            {
                return ExitFailure;
            }

            var finishedTask = await Task.WhenAny(timeoutTask, runTask);
            if (finishedTask == timeoutTask)
            {
                HandleTimeout(options);
                cts.Cancel();

                try
                {
                    // Need to await here to ensure that all of the child processes are properly 
                    // killed before we exit.
                    await runTask;
                }
                catch
                {
                    // Cancellation exceptions expected here. 
                }

                return ExitFailure;
            }

            return await runTask;
        }

        private static async Task<int> RunCore(Options options, CancellationToken cancellationToken)
        {
            if (!CheckAssemblyList(options))
            {
                return ExitFailure;
            }

            var testExecutor = CreateTestExecutor(options);
            var testRunner = new TestRunner(options, testExecutor);
            var start = DateTime.Now;
            var assemblyInfoList = GetAssemblyList(options);

            Console.WriteLine($"Data Storage: {testExecutor.DataStorage.Name}");
            Console.WriteLine($"Running {options.Assemblies.Count()} test assemblies in {assemblyInfoList.Count} partitions");

            var result = await testRunner.RunAllAsync(assemblyInfoList, cancellationToken).ConfigureAwait(true);
            var elapsed = DateTime.Now - start;

            Console.WriteLine($"Test execution time: {elapsed}");

            WriteLogFile(options);
            DisplayResults(options.Display, result.TestResults);

            if (CanUseWebStorage() && options.UseCachedResults)
            {
                await SendRunStats(options, testExecutor.DataStorage, elapsed, result, assemblyInfoList.Count, cancellationToken).ConfigureAwait(true);
            }

            if (!result.Succeeded)
            {
                ConsoleUtil.WriteLine(ConsoleColor.Red, $"Test failures encountered");
                return ExitFailure;
            }

            Console.WriteLine($"All tests passed");
            return ExitSuccess;
        }

        private static void WriteLogFile(Options options)
        {
            var logFilePath = options.LogFilePath;
            if (string.IsNullOrEmpty(logFilePath))
            {
                return;
            }

            try
            {
                using (var writer = new StreamWriter(logFilePath, append: false))
                {
                    Logger.WriteTo(writer);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Error writing log file {logFilePath}");
                Console.WriteLine(ex);
            }

            Logger.Clear();
        }

        /// <summary>
        /// Invoked when a timeout occurs and we need to dump all of the test processes and shut down 
        /// the runnner.
        /// </summary>
        private static void HandleTimeout(Options options)
        {
            Console.WriteLine("Roslyn Error: test timeout exceeded, dumping remaining processes");
            var dumpDir = options.LogFilePath != null
                ? Path.GetDirectoryName(options.LogFilePath)
                : Directory.GetCurrentDirectory();

            var counter = 0;
            foreach (var proc in ProcessUtil.GetProcessTree(Process.GetCurrentProcess()).OrderBy(x => x.ProcessName))
            {
                var dumpFilePath = Path.Combine(dumpDir, $"{proc.ProcessName}-{counter}.dmp");
                counter++;
                Console.WriteLine($"Dumping {proc.ProcessName} {proc.Id} to {dumpFilePath}");
                DumpUtil.WriteDump(proc, dumpFilePath);
            }

            WriteLogFile(options);
        }<commit_msg>Don't let a single dump failure, stop others<commit_after>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using RunTests.Cache;
using Newtonsoft.Json.Linq;
using RestSharp;
using System.Collections.Immutable;
using Newtonsoft.Json;
using System.Reflection;
using System.Diagnostics;

namespace RunTests
{
    internal sealed partial class Program
    {
        internal const int ExitSuccess = 0;
        internal const int ExitFailure = 1;

        internal static int Main(string[] args)
        {
            Logger.Log("RunTest command line");
            Logger.Log(string.Join(" ", args));

            var options = Options.Parse(args);
            if (options == null)
            {
                Options.PrintUsage();
                return ExitFailure;
            }

            // Setup cancellation for ctrl-c key presses
            var cts = new CancellationTokenSource();
            Console.CancelKeyPress += delegate
            {
                cts.Cancel();
            };

            return Run(options, cts.Token).GetAwaiter().GetResult();
        }

        private static async Task<int> Run(Options options, CancellationToken cancellationToken)
        {
            if (options.Timeout == null)
            {
                return await RunCore(options, cancellationToken);
            }

            var timeoutTask = Task.Delay(options.Timeout.Value);
            var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var runTask = RunCore(options, cts.Token);

            if (cancellationToken.IsCancellationRequested)
            {
                return ExitFailure;
            }

            var finishedTask = await Task.WhenAny(timeoutTask, runTask);
            if (finishedTask == timeoutTask)
            {
                HandleTimeout(options);
                cts.Cancel();

                try
                {
                    // Need to await here to ensure that all of the child processes are properly 
                    // killed before we exit.
                    await runTask;
                }
                catch
                {
                    // Cancellation exceptions expected here. 
                }

                return ExitFailure;
            }

            return await runTask;
        }

        private static async Task<int> RunCore(Options options, CancellationToken cancellationToken)
        {
            if (!CheckAssemblyList(options))
            {
                return ExitFailure;
            }

            var testExecutor = CreateTestExecutor(options);
            var testRunner = new TestRunner(options, testExecutor);
            var start = DateTime.Now;
            var assemblyInfoList = GetAssemblyList(options);

            Console.WriteLine($"Data Storage: {testExecutor.DataStorage.Name}");
            Console.WriteLine($"Running {options.Assemblies.Count()} test assemblies in {assemblyInfoList.Count} partitions");

            var result = await testRunner.RunAllAsync(assemblyInfoList, cancellationToken).ConfigureAwait(true);
            var elapsed = DateTime.Now - start;

            Console.WriteLine($"Test execution time: {elapsed}");

            WriteLogFile(options);
            DisplayResults(options.Display, result.TestResults);

            if (CanUseWebStorage() && options.UseCachedResults)
            {
                await SendRunStats(options, testExecutor.DataStorage, elapsed, result, assemblyInfoList.Count, cancellationToken).ConfigureAwait(true);
            }

            if (!result.Succeeded)
            {
                ConsoleUtil.WriteLine(ConsoleColor.Red, $"Test failures encountered");
                return ExitFailure;
            }

            Console.WriteLine($"All tests passed");
            return ExitSuccess;
        }

        private static void WriteLogFile(Options options)
        {
            var logFilePath = options.LogFilePath;
            if (string.IsNullOrEmpty(logFilePath))
            {
                return;
            }

            try
            {
                using (var writer = new StreamWriter(logFilePath, append: false))
                {
                    Logger.WriteTo(writer);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Error writing log file {logFilePath}");
                Console.WriteLine(ex);
            }

            Logger.Clear();
        }

        /// <summary>
        /// Invoked when a timeout occurs and we need to dump all of the test processes and shut down 
        /// the runnner.
        /// </summary>
        private static void HandleTimeout(Options options)
        {
            Console.WriteLine("Roslyn Error: test timeout exceeded, dumping remaining processes");
            var dumpDir = options.LogFilePath != null
                ? Path.GetDirectoryName(options.LogFilePath)
                : Directory.GetCurrentDirectory();

            var counter = 0;
            foreach (var proc in ProcessUtil.GetProcessTree(Process.GetCurrentProcess()).OrderBy(x => x.ProcessName))
            {
                var dumpFilePath = Path.Combine(dumpDir, $"{proc.ProcessName}-{counter}.dmp");
                counter++;
                Console.Write($"Dumping {proc.ProcessName} {proc.Id} to {dumpFilePath} ... ");
                try
                {
                    DumpUtil.WriteDump(proc, dumpFilePath);
                    Console.WriteLine("succeeded");
                }
                catch (Exception ex)
                {
                    Console.WriteLine("FAILED");
                    Console.WriteLine(ex.Message);
                }
            }

            WriteLogFile(options);
        }
{'fraction_non_alphanumeric': 0.08496234238808496, 'fraction_numerical': 0.0008462384700008462, 'mean_word_length': 1.6689250225835592, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4875241', 'n_tokens_mistral': 2830, 'n_tokens_neox': 2681, 'n_words': 700}