<commit_before>﻿using Byond.TopicSender;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Compiler;
using Tgstation.Server.Host.Components.Interop;
using Tgstation.Server.Host.Core;

namespace Tgstation.Server.Host.Components.Watchdog
{
	/// <inheritdoc />
	sealed class Watchdog : IWatchdog, ICustomCommandHandler
	{
		/// <summary>
		/// The time in seconds to wait from starting <see cref="alphaServer"/> to start <see cref="bravoServer"/>. Does not take responsiveness into account
		/// </summary>
		const int AlphaBravoStartupSeperationInterval = 3;

		/// <inheritdoc />
		public bool Running { get; private set; }

		/// <inheritdoc />
		public bool AlphaIsActive { get; private set; }

		/// <inheritdoc />
		public Models.CompileJob ActiveCompileJob => (AlphaIsActive ? alphaServer : bravoServer)?.Dmb.CompileJob;

		/// <inheritdoc />
		public LaunchResult LastLaunchResult { get; private set; }

		/// <inheritdoc />
		public DreamDaemonLaunchParameters ActiveLaunchParameters { get; private set; }

		/// <inheritdoc />
		public DreamDaemonLaunchParameters LastLaunchParameters { get; private set; }

		/// <inheritdoc />
		public RebootState? RebootState => Running ? (AlphaIsActive ? alphaServer?.RebootState : bravoServer?.RebootState) : null;

		/// <summary>
		/// The <see cref="IChat"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IChat chat;

		/// <summary>
		/// The <see cref="ISessionControllerFactory"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly ISessionControllerFactory sessionControllerFactory;

		/// <summary>
		/// The <see cref="IDmbFactory"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IDmbFactory dmbFactory;

		/// <summary>
		/// The <see cref="ILogger{TCategoryName}"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly ILogger<Watchdog> logger;

		/// <summary>
		/// The <see cref="IReattachInfoHandler"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IReattachInfoHandler reattachInfoHandler;

		/// <summary>
		/// The <see cref="IDatabaseContextFactory"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IDatabaseContextFactory databaseContextFactory;

		/// <summary>
		/// The <see cref="IByondTopicSender"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IByondTopicSender byondTopicSender;

		/// <summary>
		/// The <see cref="IEventConsumer"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IEventConsumer eventConsumer;

		/// <summary>
		/// The <see cref="SemaphoreSlim"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly SemaphoreSlim semaphore;

		/// <summary>
		/// The <see cref="Api.Models.Instance"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly Api.Models.Instance instance;

		/// <summary>
		/// If the <see cref="Watchdog"/> should <see cref="LaunchNoLock(bool, bool, bool, CancellationToken)"/> in <see cref="StartAsync(CancellationToken)"/>
		/// </summary>
		readonly bool autoStart;

		/// <summary>
		/// The <see cref="CancellationTokenSource"/> for the monitor loop
		/// </summary>
		CancellationTokenSource monitorCts;

		/// <summary>
		/// The <see cref="Task"/> running the monitor loop
		/// </summary>
		Task monitorTask;

		/// <summary>
		/// <see cref="TaskCompletionSource{TResult}"/> that completes when <see cref="ActiveLaunchParameters"/> are changed and we are <see cref="Running"/>
		/// </summary>
		TaskCompletionSource<object> activeParametersUpdated;

		/// <summary>
		/// Server designation alpha
		/// </summary>
		ISessionController alphaServer;
		/// <summary>
		/// Server designation bravo
		/// </summary>
		ISessionController bravoServer;

		/// <summary>
		/// If the servers should be released instead of shutdown
		/// </summary>
		bool releaseServers;

		/// <summary>
		/// Construct a <see cref="IWatchdog"/>
		/// </summary>
		/// <param name="chat">The value of <see cref="chat"/></param>
		/// <param name="sessionControllerFactory">The value of <see cref="sessionControllerFactory"/></param>
		/// <param name="dmbFactory">The value of <see cref="dmbFactory"/></param>
		/// <param name="serverUpdater">The <see cref="IServerControl"/> for the <see cref="Watchdog"/></param>
		/// <param name="logger">The value of <see cref="logger"/></param>
		/// <param name="reattachInfoHandler">The value of <see cref="reattachInfoHandler"/></param>
		/// <param name="databaseContextFactory">The value of <see cref="databaseContextFactory"/></param>
		/// <param name="byondTopicSender">The value of <see cref="byondTopicSender"/></param>
		/// <param name="initialLaunchParameters">The initial value of <see cref="ActiveLaunchParameters"/></param>
		/// <param name="instance">The value of <see cref="instance"/></param>
		/// <param name="autoStart">The value of <see cref="autoStart"/></param>
		/// <param name="eventConsumer">The value of <see cref="eventConsumer"/></param>
		public Watchdog(IChat chat, ISessionControllerFactory sessionControllerFactory, IDmbFactory dmbFactory, IServerControl serverUpdater, ILogger<Watchdog> logger, IReattachInfoHandler reattachInfoHandler, IDatabaseContextFactory databaseContextFactory, IByondTopicSender byondTopicSender, IEventConsumer eventConsumer, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, bool autoStart)
		{
			this.chat = chat ?? throw new ArgumentNullException(nameof(chat));
			this.sessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory));
			this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory));
			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
			this.reattachInfoHandler = reattachInfoHandler ?? throw new ArgumentNullException(nameof(reattachInfoHandler));
			this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
			this.byondTopicSender = byondTopicSender ?? throw new ArgumentNullException(nameof(byondTopicSender));
			this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
			this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
			this.autoStart = autoStart;

			if (serverUpdater == null)
				throw new ArgumentNullException(nameof(serverUpdater));

			serverUpdater.RegisterForRestart(() => releaseServers = true);

			chat.RegisterCommandHandler(this);

			AlphaIsActive = true;
			ActiveLaunchParameters = initialLaunchParameters;
			releaseServers = false;
			semaphore = new SemaphoreSlim(1);
			activeParametersUpdated = new TaskCompletionSource<object>();
		}

		/// <inheritdoc />
		public void Dispose()
		{
			DisposeAndNullControllers();
			semaphore.Dispose();
		}

		/// <summary>
		/// Call <see cref="IDisposable.Dispose"/> on <see cref="alphaServer"/> and <see cref="bravoServer"/> and set them to <see langword="null"/>
		/// </summary>
		void DisposeAndNullControllers()
		{
			alphaServer?.Dispose();
			alphaServer = null;
			bravoServer?.Dispose();
			bravoServer = null;
			Running = false;
		}

		/// <summary>
		/// Implementation of <see cref="Restart(bool, CancellationToken)"/>. Does not lock <see cref="semaphore"/>
		/// </summary>
		async Task<WatchdogLaunchResult> RestartNoLock(bool graceful, CancellationToken cancellationToken)
		{
			var running = Running;
			if (!graceful || !running)
			{
				Task chatTask;
				if (running)
				{
					chatTask = chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken);
					await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false);
				}
				else
					chatTask = Task.CompletedTask;
				var result = await LaunchNoLock(true, !running, false, cancellationToken).ConfigureAwait(false);
				await chatTask.ConfigureAwait(false);
				return result;
			}
			var toReboot = AlphaIsActive ? alphaServer : bravoServer;
			var other = AlphaIsActive ? bravoServer : alphaServer;
			if (toReboot != null)
				//todo, log the result
				await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false);
			return null;
		}		

		/// <summary>
		/// Implementation of <see cref="Terminate(bool, CancellationToken)"/>. Does not lock <see cref="semaphore"/>
		/// </summary>
		/// <param name="graceful">If <see langword="true"/> the termination will be delayed until a reboot is detected in the active server's DMAPI and this function will return immediately</param>
		/// <param name="announce">If <see langword="true"/> the termination will be announced using <see cref="chat"/></param>
		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
		async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken)
		{
			if (!Running)
				return;
			if (!graceful)
			{
				var chatTask = announce ? chat.SendWatchdogMessage("Terminating...", cancellationToken) : Task.CompletedTask;
				await StopMonitor().ConfigureAwait(false);
				DisposeAndNullControllers();
				await chatTask.ConfigureAwait(false);
				return;
			}
			var toKill = AlphaIsActive ? alphaServer : bravoServer;
			var other = AlphaIsActive ? bravoServer : alphaServer;
			if (toKill != null)
				await toKill.SetRebootState(Components.Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false);
		}

		/// <summary>
		/// Handles the actions to take when the monitor has to "wake up"
		/// </summary>
		/// <param name="activationReason">The <see cref="MonitorActivationReason"/> that caused the invocation</param>
		/// <param name="monitorState">The current <see cref="MonitorState"/>. Will be modified upon retrn</param>
		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
		/// <returns>A <see cref="Task"/> representing the running operation</returns>
		async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken)
		{
			logger.LogInformation("Monitor activation. Reason: {0}", activationReason);

			//returns true if the inactive server can't be used immediately
			bool FullRestartDeadInactive()
			{
				if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail)
				{
					logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting");
					monitorState.NextAction = MonitorAction.Restart;    //will dispose server
					return true;
				}
				return false;
			};

			//trys to set inactive server's port to the private port
			async Task<bool> MakeInactiveActive()
			{
				logger.LogInformation("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value);
				var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false);

				if (!result)
				{
					logger.LogWarning("Failed to activate inactive server! Restarting monitor...");
					monitorState.NextAction = MonitorAction.Restart;    //will dispose server
					return false;
				}

				// should always be set for InactiveServer
				monitorState.InactiveServer.ClosePortOnReboot = false;
				monitorState.ActiveServer.ClosePortOnReboot = true;

				//inactive server should always be using active launch parameters
				LastLaunchParameters = ActiveLaunchParameters;

				var tmp = monitorState.ActiveServer;
				monitorState.ActiveServer = monitorState.InactiveServer;
				monitorState.InactiveServer = tmp;
				AlphaIsActive = !AlphaIsActive;
				return true;
			}

			// Tries to load inactive server with latest dmb, falling back to current dmb on failure. Requires a lock on <see cref="semaphore"/>
			async Task<bool> RestartInactiveServer()
			{
				logger.LogInformation("Rebooting inactive server...");
				var newDmb = dmbFactory.LockNextDmb(1);
				bool usedMostRecentDmb;
				try
				{
					monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, newDmb, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false);
					usedMostRecentDmb = true;
				}
				catch (OperationCanceledException)
				{
					throw;
				}
				catch (Exception e)
				{
					logger.LogError("Exception occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString());
					//ahh jeez, what do we do here?
					//this is our fault, so it should never happen but
					//idk maybe a database error while handling the newest dmb?
					//either way try to start it using the active server's dmb as a backup
					try
					{
						var dmbBackup = await dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false);

						if (dmbBackup == null)	//NANI!?
							//just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has
							throw new JobException("Creating backup DMB provider failed!");

						monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false);
						usedMostRecentDmb = false;
						await chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false);
					}
					catch (OperationCanceledException)
					{
						throw;
					}
					catch (Exception e2)
					{
						//fuuuuucckkk
						logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString());
						monitorState.InactiveServerCritFail = true;
						await chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false);
						return true;    //we didn't use the old dmb
					}
				}

				logger.LogInformation("Successfully relaunched inactive server!");
				monitorState.RebootingInactiveServer = true;
				// should always be set for InactiveServer
				monitorState.InactiveServer.ClosePortOnReboot = false;
				return usedMostRecentDmb;
			}

			async Task UpdateAndRestartInactiveServer(bool breakAfter)
			{
				//replace the notification tcs here so that the next loop will read a fresh one
				activeParametersUpdated = new TaskCompletionSource<object>();
				monitorState.InactiveServer.Dispose();  //kill or recycle it
				monitorState.NextAction = breakAfter ? MonitorAction.Break : MonitorAction.Continue;

				var usedLatestDmb = await RestartInactiveServer().ConfigureAwait(false);

				if (monitorState.NextAction == (breakAfter ? MonitorAction.Break : MonitorAction.Continue))
				{
					monitorState.ActiveServer.ClosePortOnReboot = false;
					if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb)
						monitorState.InactiveServerHasStagedDmb = false;    //don't try to load it again though
				}
			};

			//reason handling
			switch (activationReason)
			{
				case MonitorActivationReason.ActiveServerCrashed:
					if(monitorState.ActiveServer.RebootState == Components.Watchdog.RebootState.Shutdown)
					{
						await chat.SendWatchdogMessage("Active server crashed or exited! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
						monitorState.NextAction = MonitorAction.Exit;
						break;
					}

					if (FullRestartDeadInactive())
					{
						await chat.SendWatchdogMessage("Active server crashed or exited! Inactive server unable to online!", cancellationToken).ConfigureAwait(false);
						break;
					}

					await chat.SendWatchdogMessage("Active server crashed or exited! Onlining inactive server...", cancellationToken).ConfigureAwait(false);
					if (!await MakeInactiveActive().ConfigureAwait(false))
						break;
					
					monitorState.ActiveServer.ClosePortOnReboot = false;
					await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);
					break;
				case MonitorActivationReason.InactiveServerCrashed:
					await chat.SendWatchdogMessage("Inactive server crashed or exited! Rebooting...", cancellationToken).ConfigureAwait(false);
					await UpdateAndRestartInactiveServer(false).ConfigureAwait(false);
					break;
				case MonitorActivationReason.ActiveServerRebooted:
					if (FullRestartDeadInactive())
						break;

					//what matters here is the RebootState
					bool restartOnceSwapped = false;
					var rebootState = monitorState.ActiveServer.RebootState;
					monitorState.ActiveServer.ResetRebootState();	//the DMAPI has already done this internally

					switch (rebootState)
					{
						case Components.Watchdog.RebootState.Normal:
							break;
						case Components.Watchdog.RebootState.Restart:
							restartOnceSwapped = true;
							break;
						case Components.Watchdog.RebootState.Shutdown:
							await chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
							DisposeAndNullControllers();
							monitorState.NextAction = MonitorAction.Exit;
							return;
					}

					var sameCompileJob = monitorState.InactiveServer.Dmb.CompileJob.Id == monitorState.ActiveServer.Dmb.CompileJob.Id;
					if (sameCompileJob && monitorState.InactiveServerHasStagedDmb)
						//both servers up to date
						monitorState.InactiveServerHasStagedDmb = false;
					if (!sameCompileJob || ActiveLaunchParameters != LastLaunchParameters)
						//need a new launch in ActiveServer
						restartOnceSwapped = true;

					if (!await MakeInactiveActive().ConfigureAwait(false))
						break;

					if(!restartOnceSwapped)
						//try to reopen inactive server on the private port so it's not pinging all the time
						//failing that, just reboot it
						restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false);

					if (restartOnceSwapped) //for one reason or another, 
						await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);	//break because worse case, active server is still booting
					else
						monitorState.NextAction = MonitorAction.Break;
					break;
				case MonitorActivationReason.InactiveServerRebooted:
					monitorState.RebootingInactiveServer = true;
					monitorState.InactiveServer.ResetRebootState();   //the DMAPI has already done this internally
					monitorState.ActiveServer.ClosePortOnReboot = false;
					monitorState.NextAction = MonitorAction.Continue;
					break;
				case MonitorActivationReason.InactiveServerStartupComplete:
					//eziest case of my life
					monitorState.RebootingInactiveServer = false;
					monitorState.ActiveServer.ClosePortOnReboot = true;
					monitorState.NextAction = MonitorAction.Continue;
					break;
				case MonitorActivationReason.NewDmbAvailable:
					monitorState.InactiveServerHasStagedDmb = true;
					await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);	//next case does same thing
					break;
				case MonitorActivationReason.ActiveLaunchParametersUpdated:
					await UpdateAndRestartInactiveServer(false).ConfigureAwait(false);
					break;
			}
		}

		/// <summary>
		/// The loop that watches the watchdog
		/// </summary>
		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
		/// <returns>A <see cref="Task"/> representing the running operation</returns>
		async Task MonitorLifetimes(CancellationToken cancellationToken)
		{
			logger.LogDebug("Entered MonitorLifetimes");
			var iteration = 1;
			for(var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration)
			{
				logger.LogDebug("New iteration of monitor loop");
				try
				{
					if(AlphaIsActive)
						logger.LogDebug("Alpha is the active server");
					else
						logger.LogDebug("Bravo is the active server");

					if(monitorState.InactiveServerHasStagedDmb)
						logger.LogDebug("Inactive server has staged .dmb");
					if (monitorState.RebootingInactiveServer)
						logger.LogDebug("Inactive server is rebooting");

					monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer;
					monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer;

					var activeServerLifetime = monitorState.ActiveServer.Lifetime;
					var inactiveServerLifetime = monitorState.InactiveServer.Lifetime;
					var activeServerReboot = monitorState.ActiveServer.OnReboot;
					var inactiveServerReboot = monitorState.InactiveServer.OnReboot;
					var inactiveServerStartup = monitorState.RebootingInactiveServer ? monitorState.InactiveServer.LaunchResult : null;
					var activeLaunchParametersChanged = activeParametersUpdated.Task;
					var newDmbAvailable = dmbFactory.OnNewerDmb;
<commit_msg>Log monitor iteration number<commit_after>﻿using Byond.TopicSender;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Compiler;
using Tgstation.Server.Host.Components.Interop;
using Tgstation.Server.Host.Core;

namespace Tgstation.Server.Host.Components.Watchdog
{
	/// <inheritdoc />
	sealed class Watchdog : IWatchdog, ICustomCommandHandler
	{
		/// <summary>
		/// The time in seconds to wait from starting <see cref="alphaServer"/> to start <see cref="bravoServer"/>. Does not take responsiveness into account
		/// </summary>
		const int AlphaBravoStartupSeperationInterval = 3;

		/// <inheritdoc />
		public bool Running { get; private set; }

		/// <inheritdoc />
		public bool AlphaIsActive { get; private set; }

		/// <inheritdoc />
		public Models.CompileJob ActiveCompileJob => (AlphaIsActive ? alphaServer : bravoServer)?.Dmb.CompileJob;

		/// <inheritdoc />
		public LaunchResult LastLaunchResult { get; private set; }

		/// <inheritdoc />
		public DreamDaemonLaunchParameters ActiveLaunchParameters { get; private set; }

		/// <inheritdoc />
		public DreamDaemonLaunchParameters LastLaunchParameters { get; private set; }

		/// <inheritdoc />
		public RebootState? RebootState => Running ? (AlphaIsActive ? alphaServer?.RebootState : bravoServer?.RebootState) : null;

		/// <summary>
		/// The <see cref="IChat"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IChat chat;

		/// <summary>
		/// The <see cref="ISessionControllerFactory"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly ISessionControllerFactory sessionControllerFactory;

		/// <summary>
		/// The <see cref="IDmbFactory"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IDmbFactory dmbFactory;

		/// <summary>
		/// The <see cref="ILogger{TCategoryName}"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly ILogger<Watchdog> logger;

		/// <summary>
		/// The <see cref="IReattachInfoHandler"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IReattachInfoHandler reattachInfoHandler;

		/// <summary>
		/// The <see cref="IDatabaseContextFactory"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IDatabaseContextFactory databaseContextFactory;

		/// <summary>
		/// The <see cref="IByondTopicSender"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IByondTopicSender byondTopicSender;

		/// <summary>
		/// The <see cref="IEventConsumer"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly IEventConsumer eventConsumer;

		/// <summary>
		/// The <see cref="SemaphoreSlim"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly SemaphoreSlim semaphore;

		/// <summary>
		/// The <see cref="Api.Models.Instance"/> for the <see cref="Watchdog"/>
		/// </summary>
		readonly Api.Models.Instance instance;

		/// <summary>
		/// If the <see cref="Watchdog"/> should <see cref="LaunchNoLock(bool, bool, bool, CancellationToken)"/> in <see cref="StartAsync(CancellationToken)"/>
		/// </summary>
		readonly bool autoStart;

		/// <summary>
		/// The <see cref="CancellationTokenSource"/> for the monitor loop
		/// </summary>
		CancellationTokenSource monitorCts;

		/// <summary>
		/// The <see cref="Task"/> running the monitor loop
		/// </summary>
		Task monitorTask;

		/// <summary>
		/// <see cref="TaskCompletionSource{TResult}"/> that completes when <see cref="ActiveLaunchParameters"/> are changed and we are <see cref="Running"/>
		/// </summary>
		TaskCompletionSource<object> activeParametersUpdated;

		/// <summary>
		/// Server designation alpha
		/// </summary>
		ISessionController alphaServer;
		/// <summary>
		/// Server designation bravo
		/// </summary>
		ISessionController bravoServer;

		/// <summary>
		/// If the servers should be released instead of shutdown
		/// </summary>
		bool releaseServers;

		/// <summary>
		/// Construct a <see cref="IWatchdog"/>
		/// </summary>
		/// <param name="chat">The value of <see cref="chat"/></param>
		/// <param name="sessionControllerFactory">The value of <see cref="sessionControllerFactory"/></param>
		/// <param name="dmbFactory">The value of <see cref="dmbFactory"/></param>
		/// <param name="serverUpdater">The <see cref="IServerControl"/> for the <see cref="Watchdog"/></param>
		/// <param name="logger">The value of <see cref="logger"/></param>
		/// <param name="reattachInfoHandler">The value of <see cref="reattachInfoHandler"/></param>
		/// <param name="databaseContextFactory">The value of <see cref="databaseContextFactory"/></param>
		/// <param name="byondTopicSender">The value of <see cref="byondTopicSender"/></param>
		/// <param name="initialLaunchParameters">The initial value of <see cref="ActiveLaunchParameters"/></param>
		/// <param name="instance">The value of <see cref="instance"/></param>
		/// <param name="autoStart">The value of <see cref="autoStart"/></param>
		/// <param name="eventConsumer">The value of <see cref="eventConsumer"/></param>
		public Watchdog(IChat chat, ISessionControllerFactory sessionControllerFactory, IDmbFactory dmbFactory, IServerControl serverUpdater, ILogger<Watchdog> logger, IReattachInfoHandler reattachInfoHandler, IDatabaseContextFactory databaseContextFactory, IByondTopicSender byondTopicSender, IEventConsumer eventConsumer, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, bool autoStart)
		{
			this.chat = chat ?? throw new ArgumentNullException(nameof(chat));
			this.sessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory));
			this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory));
			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
			this.reattachInfoHandler = reattachInfoHandler ?? throw new ArgumentNullException(nameof(reattachInfoHandler));
			this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
			this.byondTopicSender = byondTopicSender ?? throw new ArgumentNullException(nameof(byondTopicSender));
			this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
			this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
			this.autoStart = autoStart;

			if (serverUpdater == null)
				throw new ArgumentNullException(nameof(serverUpdater));

			serverUpdater.RegisterForRestart(() => releaseServers = true);

			chat.RegisterCommandHandler(this);

			AlphaIsActive = true;
			ActiveLaunchParameters = initialLaunchParameters;
			releaseServers = false;
			semaphore = new SemaphoreSlim(1);
			activeParametersUpdated = new TaskCompletionSource<object>();
		}

		/// <inheritdoc />
		public void Dispose()
		{
			DisposeAndNullControllers();
			semaphore.Dispose();
		}

		/// <summary>
		/// Call <see cref="IDisposable.Dispose"/> on <see cref="alphaServer"/> and <see cref="bravoServer"/> and set them to <see langword="null"/>
		/// </summary>
		void DisposeAndNullControllers()
		{
			alphaServer?.Dispose();
			alphaServer = null;
			bravoServer?.Dispose();
			bravoServer = null;
			Running = false;
		}

		/// <summary>
		/// Implementation of <see cref="Restart(bool, CancellationToken)"/>. Does not lock <see cref="semaphore"/>
		/// </summary>
		async Task<WatchdogLaunchResult> RestartNoLock(bool graceful, CancellationToken cancellationToken)
		{
			var running = Running;
			if (!graceful || !running)
			{
				Task chatTask;
				if (running)
				{
					chatTask = chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken);
					await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false);
				}
				else
					chatTask = Task.CompletedTask;
				var result = await LaunchNoLock(true, !running, false, cancellationToken).ConfigureAwait(false);
				await chatTask.ConfigureAwait(false);
				return result;
			}
			var toReboot = AlphaIsActive ? alphaServer : bravoServer;
			var other = AlphaIsActive ? bravoServer : alphaServer;
			if (toReboot != null)
				//todo, log the result
				await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false);
			return null;
		}		

		/// <summary>
		/// Implementation of <see cref="Terminate(bool, CancellationToken)"/>. Does not lock <see cref="semaphore"/>
		/// </summary>
		/// <param name="graceful">If <see langword="true"/> the termination will be delayed until a reboot is detected in the active server's DMAPI and this function will return immediately</param>
		/// <param name="announce">If <see langword="true"/> the termination will be announced using <see cref="chat"/></param>
		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
		async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken)
		{
			if (!Running)
				return;
			if (!graceful)
			{
				var chatTask = announce ? chat.SendWatchdogMessage("Terminating...", cancellationToken) : Task.CompletedTask;
				await StopMonitor().ConfigureAwait(false);
				DisposeAndNullControllers();
				await chatTask.ConfigureAwait(false);
				return;
			}
			var toKill = AlphaIsActive ? alphaServer : bravoServer;
			var other = AlphaIsActive ? bravoServer : alphaServer;
			if (toKill != null)
				await toKill.SetRebootState(Components.Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false);
		}

		/// <summary>
		/// Handles the actions to take when the monitor has to "wake up"
		/// </summary>
		/// <param name="activationReason">The <see cref="MonitorActivationReason"/> that caused the invocation</param>
		/// <param name="monitorState">The current <see cref="MonitorState"/>. Will be modified upon retrn</param>
		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
		/// <returns>A <see cref="Task"/> representing the running operation</returns>
		async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken)
		{
			logger.LogInformation("Monitor activation. Reason: {0}", activationReason);

			//returns true if the inactive server can't be used immediately
			bool FullRestartDeadInactive()
			{
				if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail)
				{
					logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting");
					monitorState.NextAction = MonitorAction.Restart;    //will dispose server
					return true;
				}
				return false;
			};

			//trys to set inactive server's port to the private port
			async Task<bool> MakeInactiveActive()
			{
				logger.LogInformation("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value);
				var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false);

				if (!result)
				{
					logger.LogWarning("Failed to activate inactive server! Restarting monitor...");
					monitorState.NextAction = MonitorAction.Restart;    //will dispose server
					return false;
				}

				// should always be set for InactiveServer
				monitorState.InactiveServer.ClosePortOnReboot = false;
				monitorState.ActiveServer.ClosePortOnReboot = true;

				//inactive server should always be using active launch parameters
				LastLaunchParameters = ActiveLaunchParameters;

				var tmp = monitorState.ActiveServer;
				monitorState.ActiveServer = monitorState.InactiveServer;
				monitorState.InactiveServer = tmp;
				AlphaIsActive = !AlphaIsActive;
				return true;
			}

			// Tries to load inactive server with latest dmb, falling back to current dmb on failure. Requires a lock on <see cref="semaphore"/>
			async Task<bool> RestartInactiveServer()
			{
				logger.LogInformation("Rebooting inactive server...");
				var newDmb = dmbFactory.LockNextDmb(1);
				bool usedMostRecentDmb;
				try
				{
					monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, newDmb, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false);
					usedMostRecentDmb = true;
				}
				catch (OperationCanceledException)
				{
					throw;
				}
				catch (Exception e)
				{
					logger.LogError("Exception occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString());
					//ahh jeez, what do we do here?
					//this is our fault, so it should never happen but
					//idk maybe a database error while handling the newest dmb?
					//either way try to start it using the active server's dmb as a backup
					try
					{
						var dmbBackup = await dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false);

						if (dmbBackup == null)	//NANI!?
							//just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has
							throw new JobException("Creating backup DMB provider failed!");

						monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false);
						usedMostRecentDmb = false;
						await chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false);
					}
					catch (OperationCanceledException)
					{
						throw;
					}
					catch (Exception e2)
					{
						//fuuuuucckkk
						logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString());
						monitorState.InactiveServerCritFail = true;
						await chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false);
						return true;    //we didn't use the old dmb
					}
				}

				logger.LogInformation("Successfully relaunched inactive server!");
				monitorState.RebootingInactiveServer = true;
				// should always be set for InactiveServer
				monitorState.InactiveServer.ClosePortOnReboot = false;
				return usedMostRecentDmb;
			}

			async Task UpdateAndRestartInactiveServer(bool breakAfter)
			{
				//replace the notification tcs here so that the next loop will read a fresh one
				activeParametersUpdated = new TaskCompletionSource<object>();
				monitorState.InactiveServer.Dispose();  //kill or recycle it
				monitorState.NextAction = breakAfter ? MonitorAction.Break : MonitorAction.Continue;

				var usedLatestDmb = await RestartInactiveServer().ConfigureAwait(false);

				if (monitorState.NextAction == (breakAfter ? MonitorAction.Break : MonitorAction.Continue))
				{
					monitorState.ActiveServer.ClosePortOnReboot = false;
					if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb)
						monitorState.InactiveServerHasStagedDmb = false;    //don't try to load it again though
				}
			};

			//reason handling
			switch (activationReason)
			{
				case MonitorActivationReason.ActiveServerCrashed:
					if(monitorState.ActiveServer.RebootState == Components.Watchdog.RebootState.Shutdown)
					{
						await chat.SendWatchdogMessage("Active server crashed or exited! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
						monitorState.NextAction = MonitorAction.Exit;
						break;
					}

					if (FullRestartDeadInactive())
					{
						await chat.SendWatchdogMessage("Active server crashed or exited! Inactive server unable to online!", cancellationToken).ConfigureAwait(false);
						break;
					}

					await chat.SendWatchdogMessage("Active server crashed or exited! Onlining inactive server...", cancellationToken).ConfigureAwait(false);
					if (!await MakeInactiveActive().ConfigureAwait(false))
						break;
					
					monitorState.ActiveServer.ClosePortOnReboot = false;
					await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);
					break;
				case MonitorActivationReason.InactiveServerCrashed:
					await chat.SendWatchdogMessage("Inactive server crashed or exited! Rebooting...", cancellationToken).ConfigureAwait(false);
					await UpdateAndRestartInactiveServer(false).ConfigureAwait(false);
					break;
				case MonitorActivationReason.ActiveServerRebooted:
					if (FullRestartDeadInactive())
						break;

					//what matters here is the RebootState
					bool restartOnceSwapped = false;
					var rebootState = monitorState.ActiveServer.RebootState;
					monitorState.ActiveServer.ResetRebootState();	//the DMAPI has already done this internally

					switch (rebootState)
					{
						case Components.Watchdog.RebootState.Normal:
							break;
						case Components.Watchdog.RebootState.Restart:
							restartOnceSwapped = true;
							break;
						case Components.Watchdog.RebootState.Shutdown:
							await chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
							DisposeAndNullControllers();
							monitorState.NextAction = MonitorAction.Exit;
							return;
					}

					var sameCompileJob = monitorState.InactiveServer.Dmb.CompileJob.Id == monitorState.ActiveServer.Dmb.CompileJob.Id;
					if (sameCompileJob && monitorState.InactiveServerHasStagedDmb)
						//both servers up to date
						monitorState.InactiveServerHasStagedDmb = false;
					if (!sameCompileJob || ActiveLaunchParameters != LastLaunchParameters)
						//need a new launch in ActiveServer
						restartOnceSwapped = true;

					if (!await MakeInactiveActive().ConfigureAwait(false))
						break;

					if(!restartOnceSwapped)
						//try to reopen inactive server on the private port so it's not pinging all the time
						//failing that, just reboot it
						restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false);

					if (restartOnceSwapped) //for one reason or another, 
						await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);	//break because worse case, active server is still booting
					else
						monitorState.NextAction = MonitorAction.Break;
					break;
				case MonitorActivationReason.InactiveServerRebooted:
					monitorState.RebootingInactiveServer = true;
					monitorState.InactiveServer.ResetRebootState();   //the DMAPI has already done this internally
					monitorState.ActiveServer.ClosePortOnReboot = false;
					monitorState.NextAction = MonitorAction.Continue;
					break;
				case MonitorActivationReason.InactiveServerStartupComplete:
					//eziest case of my life
					monitorState.RebootingInactiveServer = false;
					monitorState.ActiveServer.ClosePortOnReboot = true;
					monitorState.NextAction = MonitorAction.Continue;
					break;
				case MonitorActivationReason.NewDmbAvailable:
					monitorState.InactiveServerHasStagedDmb = true;
					await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);	//next case does same thing
					break;
				case MonitorActivationReason.ActiveLaunchParametersUpdated:
					await UpdateAndRestartInactiveServer(false).ConfigureAwait(false);
					break;
			}
		}

		/// <summary>
		/// The loop that watches the watchdog
		/// </summary>
		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
		/// <returns>A <see cref="Task"/> representing the running operation</returns>
		async Task MonitorLifetimes(CancellationToken cancellationToken)
		{
			logger.LogDebug("Entered MonitorLifetimes");
			var iteration = 1;
			for(var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration)
			{
				logger.LogDebug("Iteration {0} of monitor loop", iteration);
				try
				{
					if(AlphaIsActive)
						logger.LogDebug("Alpha is the active server");
					else
						logger.LogDebug("Bravo is the active server");

					if(monitorState.InactiveServerHasStagedDmb)
						logger.LogDebug("Inactive server has staged .dmb");
					if (monitorState.RebootingInactiveServer)
						logger.LogDebug("Inactive server is rebooting");

					monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer;
					monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer;

					var activeServerLifetime = monitorState.ActiveServer.Lifetime;
					var inactiveServerLifetime = monitorState.InactiveServer.Lifetime;
					var activeServerReboot = monitorState.ActiveServer.OnReboot;
					var inactiveServerReboot = monitorState.InactiveServer.OnReboot;
					var inactiveServerStartup = monitorState.RebootingInactiveServer ? monitorState.InactiveServer.LaunchResult : null;
					var activeLaunchParametersChanged = activeParametersUpdated.Task;
					var newDmbAvailable = dmbFactory.OnNewerDmb;

{'fraction_non_alphanumeric': 0.10927199095874929, 'fraction_numerical': 0.0005886230928611792, 'mean_word_length': 4.092075290732526, 'pattern_counts': {'":': 0, '<': 343, '<?xml version=': 0, '>': 349, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2272871', 'n_tokens_mistral': 14001, 'n_tokens_neox': 11656, 'n_words': 3008}