WatchdogBase.cs

Go to the documentation of this file.

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
 
using Microsoft.Extensions.Logging;
 
using Prometheus;
 
using Serilog.Context;
 
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Api.Rights;
using Tgstation.Server.Common.Extensions;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Deployment.Remote;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Components.Interop;
using Tgstation.Server.Host.Components.Interop.Topic;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.Core;
using Tgstation.Server.Host.Extensions;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.Jobs;
using Tgstation.Server.Host.Utils;
 
namespace Tgstation.Server.Host.Components.Watchdog
{
#pragma warning disable CA1506 // TODO: Decomplexify
    abstract class WatchdogBase : IWatchdog, ICustomCommandHandler, IRestartHandler
    {
        public long? SessionId => GetActiveController()?.ReattachInformation.Id;
 
        public long? WorldIteration => GetActiveController()?.StartupBridgeRequestsReceived;
 
        public uint? ClientCount { get; private set; }
 
        public DateTimeOffset? LaunchTime => GetActiveController()?.LaunchTime;
 
        public WatchdogStatus Status
        {
            get => status;
            protected set
            {
                var oldStatus = status;
                status = value;
                Logger.LogTrace("Status set from {oldStatus} to {status}", oldStatus, status);
            }
        }
 
        public long? MemoryUsage => GetActiveController()?.MemoryUsage;
 
        public abstract bool AlphaIsActive { get; }
 
        public DreamDaemonLaunchParameters ActiveLaunchParameters { get; protected set; }
 
        public DreamDaemonLaunchParameters? LastLaunchParameters { get; protected set; }
 
        public Models.CompileJob? ActiveCompileJob => GetActiveController()?.CompileJob;
 
        public abstract RebootState? RebootState { get; }
 
        protected ISessionPersistor SessionPersistor { get; }
 
        protected ILogger<WatchdogBase> Logger { get; }
 
        protected IChatManager Chat { get; }
 
        protected ISessionControllerFactory SessionControllerFactory { get; }
 
        protected IDmbFactory DmbFactory { get; }
 
        protected IAsyncDelayer AsyncDelayer { get; }
 
        protected IIOManager GameIOManager { get; }
 
        readonly Api.Models.Instance metadata;
 
        readonly SemaphoreSlim synchronizationSemaphore;
 
        readonly SemaphoreSlim controllerDisposeSemaphore;
 
        readonly Gauge watchdogStatusMetric;
 
        readonly Gauge cpuUsageMetric;
 
        readonly Gauge ramUsageMetric;
 
        readonly IEventConsumer eventConsumer;
 
        readonly IJobManager jobManager;
 
        readonly IRestartRegistration restartRegistration;
 
        readonly IIOManager diagnosticsIOManager;
 
        readonly IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory;
 
        readonly bool autoStart;
 
        volatile TaskCompletionSource activeParametersUpdated;
 
        CancellationTokenSource? monitorCts;
 
        Task? monitorTask;
 
        WatchdogStatus status;
 
        int healthChecksMissed;
 
        bool releaseServers;
 
        bool disposed;
 
        protected WatchdogBase(
            IChatManager chat,
            ISessionControllerFactory sessionControllerFactory,
            IDmbFactory dmbFactory,
            ISessionPersistor sessionPersistor,
            IJobManager jobManager,
            IServerControl serverControl,
            IAsyncDelayer asyncDelayer,
            IIOManager diagnosticsIOManager,
            IEventConsumer eventConsumer,
            IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
            IMetricFactory metricFactory,
            IIOManager gameIOManager,
            ILogger<WatchdogBase> logger,
            DreamDaemonLaunchParameters initialLaunchParameters,
            Api.Models.Instance metadata,
            bool autoStart)
        {
            Chat = chat ?? throw new ArgumentNullException(nameof(chat));
            SessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory));
            DmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory));
            SessionPersistor = sessionPersistor ?? throw new ArgumentNullException(nameof(sessionPersistor));
            this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager));
            AsyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
            this.diagnosticsIOManager = diagnosticsIOManager ?? throw new ArgumentNullException(nameof(diagnosticsIOManager));
            this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
            this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
            ArgumentNullException.ThrowIfNull(metricFactory);
            GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager));
            Logger = logger ?? throw new ArgumentNullException(nameof(logger));
            ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters));
            this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
            this.autoStart = autoStart;
 
            ArgumentNullException.ThrowIfNull(serverControl);
 
            watchdogStatusMetric = metricFactory.CreateGauge(
                "tgs_watchdog_status",
                $"TGS Watchdog status: {(int)WatchdogStatus.Offline} = Offline, {(int)WatchdogStatus.Online} = Online, {(int)WatchdogStatus.Restoring} = Restoring, {(int)WatchdogStatus.DelayedRestart} = Delayed Restart");
            cpuUsageMetric = metricFactory.CreateGauge("tgs_game_cpu_usage", "Estimated total CPU usage time for the game process from 0-1");
            ramUsageMetric = metricFactory.CreateGauge("tgs_game_ram_usage", "Total used bytes of private memory for the game process");
 
            chat.RegisterCommandHandler(this);
 
            ActiveLaunchParameters = initialLaunchParameters;
            releaseServers = false;
            activeParametersUpdated = new TaskCompletionSource();
 
            restartRegistration = serverControl.RegisterForRestart(this);
            try
            {
                synchronizationSemaphore = new SemaphoreSlim(1);
                controllerDisposeSemaphore = new SemaphoreSlim(1);
            }
            catch
            {
                restartRegistration.Dispose();
                synchronizationSemaphore?.Dispose();
                throw;
            }
 
            Logger.LogTrace("Created watchdog");
        }
 
        public async ValueTask DisposeAsync()
        {
            Logger.LogTrace("Disposing...");
            synchronizationSemaphore.Dispose();
            restartRegistration.Dispose();
 
            await DisposeAndNullControllersImpl();
            controllerDisposeSemaphore.Dispose();
            monitorCts?.Dispose();
 
            disposed = true;
        }
 
        public async ValueTask<bool> ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken))
            {
                var currentLaunchParameters = ActiveLaunchParameters;
                ActiveLaunchParameters = launchParameters;
                var currentEngine = GetActiveController()?.EngineVersion.Engine;
                if (!currentEngine.HasValue)
                    return false;
 
                bool match = launchParameters.CanApplyWithoutReboot(currentLaunchParameters, currentEngine.Value);
                if (match || Status == WatchdogStatus.Offline || Status == WatchdogStatus.DelayedRestart)
                    return false;
 
                var oldTcs = Interlocked.Exchange(ref activeParametersUpdated, new TaskCompletionSource());
                oldTcs.SetResult();
            }
 
            return true;
        }
 
        public async ValueTask<MessageContent> HandleChatCommand(string commandName, string arguments, ChatUser sender, CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken))
            {
                var commandObject = new ChatCommand(sender, commandName, arguments);
                var command = new TopicParameters(commandObject);
                var activeServer = GetActiveController();
                if (Status != WatchdogStatus.Online || activeServer == null)
                    return new MessageContent
                    {
                        Text = "TGS: Server offline!",
                    };
 
                var commandResult = await activeServer.SendCommand(command, cancellationToken);
 
                if (commandResult == null)
                    return new MessageContent
                    {
                        Text = "TGS: Bad topic exchange!",
                    };
 
                if (commandResult == null)
                    return new MessageContent
                    {
                        Text = "TGS: Bad topic response!",
                    };
 
                var commandResponse = new MessageContent
                {
                    Text = commandResult.CommandResponse?.Text ?? commandResult.CommandResponseMessage,
                    Embed = commandResult.CommandResponse?.Embed,
                };
 
                if (commandResponse.Text == null && commandResponse.Embed == null)
                {
                    commandResponse.Text = "TGS: Command processed but no DMAPI response returned!";
                }
 
                HandleChatResponses(commandResult);
 
                return commandResponse;
            }
        }
 
        public async ValueTask Launch(CancellationToken cancellationToken)
        {
            if (Status != WatchdogStatus.Offline)
                throw new JobException(ErrorCode.WatchdogRunning);
            using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken))
                await LaunchNoLock(true, true, true, null, cancellationToken);
        }
 
        public virtual async ValueTask ResetRebootState(CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken))
            {
                if (Status == WatchdogStatus.Offline)
                    return;
                var toClear = GetActiveController();
                if (toClear != null)
                    await toClear.SetRebootState(Session.RebootState.Normal, cancellationToken);
            }
        }
 
        public async ValueTask Restart(bool graceful, CancellationToken cancellationToken)
        {
            if (Status == WatchdogStatus.Offline)
                throw new JobException(ErrorCode.WatchdogNotRunning);
 
            Logger.LogTrace("Begin Restart. Graceful: {gracefulFlag}", graceful);
            using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken))
            {
                if (!graceful)
                {
                    Chat.QueueWatchdogMessage("Manual restart triggered...");
                    await TerminateNoLock(false, false, cancellationToken);
                    await LaunchNoLock(true, false, true, null, cancellationToken);
                    return;
                }
 
                var toReboot = GetActiveController();
                if (toReboot != null
                    && !await toReboot.SetRebootState(Session.RebootState.Restart, cancellationToken))
                    Logger.LogWarning("Unable to send reboot state change event!");
            }
        }
 
        public async Task StartAsync(CancellationToken cancellationToken)
        {
            var reattachInfo = await SessionPersistor.Load(cancellationToken);
            var reattaching = reattachInfo != null;
            if (!autoStart && !reattaching)
                return;
 
            var job = Models.Job.Create(
                reattaching
                    ? JobCode.StartupWatchdogReattach
                    : JobCode.StartupWatchdogLaunch,
                null,
                metadata,
                DreamDaemonRights.Shutdown);
            await jobManager.RegisterOperation(
                job,
                async (core, databaseContextFactory, paramJob, progressFunction, ct) =>
                {
                    if (core?.Watchdog != this)
                        throw new InvalidOperationException(Instance.DifferentCoreExceptionMessage);
 
                    using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, ct))
                        await LaunchNoLock(true, true, true, reattachInfo, ct);
 
                    await Chat.UpdateTrackingContexts(ct);
                },
                cancellationToken);
        }
 
        public async Task StopAsync(CancellationToken cancellationToken) =>
            await TerminateNoLock(false, !releaseServers, cancellationToken);
 
        public async ValueTask Terminate(bool graceful, CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken))
                await TerminateNoLock(graceful, !releaseServers, cancellationToken);
        }
 
        public async ValueTask HandleRestart(Version? updateVersion, bool handlerMayDelayShutdownWithExtremelyLongRunningTasks, CancellationToken cancellationToken)
        {
            if (handlerMayDelayShutdownWithExtremelyLongRunningTasks)
            {
                await Terminate(true, cancellationToken);
 
                if (Status != WatchdogStatus.Offline)
                {
                    Logger.LogDebug("Waiting for server to gracefully shut down.");
                    await monitorTask!.WaitAsync(cancellationToken);
                }
                else
                    Logger.LogTrace("Graceful shutdown requested but server is already offline.");
 
                return;
            }
 
            releaseServers = true;
            if (Status == WatchdogStatus.Online)
                Chat.QueueWatchdogMessage("Detaching...");
            else
                Logger.LogTrace("Not sending detach chat message as status is: {status}", Status);
        }
 
        public abstract ValueTask InstanceRenamed(string newInstanceName, CancellationToken cancellationToken);
 
        public async ValueTask CreateDump(CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken))
                await CreateDumpNoLock(cancellationToken);
        }
 
        public async ValueTask<bool> Broadcast(string message, CancellationToken cancellationToken)
        {
            ArgumentNullException.ThrowIfNull(message);
 
            var activeServer = GetActiveController();
            if (activeServer == null)
            {
                Logger.LogInformation("Attempted broadcast failed, no active server!");
                return false;
            }
 
            if (!activeServer.DMApiAvailable)
            {
                Logger.LogInformation("Attempted broadcast failed, no DMAPI!");
                return false;
            }
 
            var minimumRequiredVersion = new Version(5, 7, 0);
            if (activeServer.DMApiVersion < minimumRequiredVersion)
            {
                Logger.LogInformation(
                    "Attempted broadcast failed, insufficient interop version: {interopVersion}. Requires {minimumRequiredVersion}!",
                    activeServer.DMApiVersion,
                    minimumRequiredVersion);
                return false;
            }
 
            Logger.LogInformation("Broadcasting: {message}", message);
 
            var response = await activeServer.SendCommand(
                TopicParameters.CreateBroadcastParameters(message),
                cancellationToken);
 
            return response != null && response.ErrorMessage == null;
        }
 
        public void RunMetricsScrape()
        {
            watchdogStatusMetric.Set((int)Status);
            var controller = GetActiveController();
            ramUsageMetric.Set(controller?.MemoryUsage ?? 0);
            cpuUsageMetric.Set(controller?.MeasureProcessorTimeDelta() ?? 0);
        }
 
        async ValueTask IEventConsumer.HandleEvent(EventType eventType, IEnumerable<string?> parameters, bool deploymentPipeline, CancellationToken cancellationToken)
        {
            ArgumentNullException.ThrowIfNull(parameters);
 
            // Method explicitly implemented to prevent accidental calls when this.eventConsumer should be used.
            var activeServer = GetActiveController();
 
            // Server may have ended
            if (activeServer == null)
                return;
 
            var notification = new EventNotification(eventType, parameters);
            var result = await activeServer.SendCommand(
                new TopicParameters(notification),
                cancellationToken);
 
            HandleChatResponses(result);
        }
 
        ValueTask? IEventConsumer.HandleCustomEvent(string eventName, IEnumerable<string?> parameters, CancellationToken cancellationToken)
            => throw new NotSupportedException("Watchdogs do not support custom events!");
 
        protected abstract ValueTask InitController(ValueTask eventTask, ReattachInformation? reattachInfo, CancellationToken cancellationToken);
 
        protected async ValueTask LaunchNoLock(
            bool startMonitor,
            bool announce,
            bool announceFailure,
            ReattachInformation? reattachInfo,
            CancellationToken cancellationToken)
        {
            Logger.LogTrace("Begin LaunchImplNoLock");
            if (startMonitor && Status != WatchdogStatus.Offline)
                throw new JobException(ErrorCode.WatchdogRunning);
 
            if (reattachInfo == null && !DmbFactory.DmbAvailable)
                throw new JobException(ErrorCode.WatchdogCompileJobCorrupted);
 
            // this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
            var eventTask = ValueTask.CompletedTask;
            if (announce)
            {
                Chat.QueueWatchdogMessage(
                    reattachInfo == null
                        ? "Launching..."
                        : "Reattaching..."); // simple announce
                if (reattachInfo == null)
                    eventTask = HandleEventImpl(EventType.WatchdogLaunch, Enumerable.Empty<string>(), false, cancellationToken);
            }
 
            // since neither server is running, this is safe to do
            LastLaunchParameters = ActiveLaunchParameters;
            healthChecksMissed = 0;
            ClientCount = null;
 
            try
            {
                await InitController(eventTask, reattachInfo, cancellationToken);
            }
            catch (OperationCanceledException ex)
            {
                Logger.LogTrace(ex, "Controller initialization cancelled!");
                throw;
            }
            catch (Exception e)
            {
                Logger.LogWarning(e, "Failed to start watchdog!");
                var originalChatTask = eventTask;
                async ValueTask ChainEventTaskWithErrorMessage()
                {
                    await originalChatTask;
                    if (announceFailure)
                        Chat.QueueWatchdogMessage("Startup failed!");
                }
 
                eventTask = ChainEventTaskWithErrorMessage();
                throw;
            }
            finally
            {
                // finish the chat task that's in flight
                try
                {
                    await eventTask;
                }
                catch (OperationCanceledException ex)
                {
                    Logger.LogTrace(ex, "Announcement task canceled!");
                }
            }
 
            Logger.LogInformation("Controller(s) initialized successfully");
 
            if (startMonitor)
            {
                monitorCts = new CancellationTokenSource();
                monitorTask = MonitorLifetimes(monitorCts.Token);
            }
        }
 
        protected async ValueTask<bool> StopMonitor()
        {
            Logger.LogTrace("StopMonitor");
            if (monitorTask == null)
                return false;
            var wasRunning = !monitorTask.IsCompleted;
            monitorCts!.Cancel();
            await monitorTask;
            Logger.LogTrace("Stopped Monitor");
            monitorCts.Dispose();
            monitorTask = null;
            monitorCts = null;
            return wasRunning;
        }
 
        protected async ValueTask CheckLaunchResult(ISessionController controller, string serverName, CancellationToken cancellationToken)
        {
            var launchResult = await controller.LaunchResult.WaitAsync(cancellationToken);
 
            // Dead sessions won't trigger this
            if (launchResult.ExitCode.HasValue) // you killed us ray...
                throw new JobException(
                    ErrorCode.WatchdogStartupFailed,
                    new JobException($"{serverName} failed to start: {launchResult}"));
            if (!launchResult.StartupTime.HasValue)
                throw new JobException(
                    ErrorCode.WatchdogStartupTimeout,
                    new JobException($"{serverName} timed out on startup: {ActiveLaunchParameters.StartupTimeout!.Value}s"));
        }
 
        protected async ValueTask ReattachFailure(CancellationToken cancellationToken)
        {
            // we lost the server, just restart entirely
            // DCT: Operation must always run
            await DisposeAndNullControllers(CancellationToken.None);
            ClientCount = null;
            const string FailReattachMessage = "Unable to properly reattach to server! Restarting watchdog...";
            Logger.LogWarning(FailReattachMessage);
 
            Chat.QueueWatchdogMessage(FailReattachMessage);
            await InitController(ValueTask.CompletedTask, null, cancellationToken);
        }
 
        protected abstract ValueTask DisposeAndNullControllersImpl();
 
        protected async ValueTask DisposeAndNullControllers(CancellationToken cancellationToken)
        {
            Logger.LogTrace("DisposeAndNullControllers");
            using (await SemaphoreSlimContext.Lock(controllerDisposeSemaphore, cancellationToken))
            {
                await DisposeAndNullControllersImpl();
                if (!releaseServers)
                    await SessionPersistor.Clear(cancellationToken);
            }
        }
 
        protected abstract ISessionController? GetActiveController();
 
        protected abstract ValueTask<MonitorAction> HandleMonitorWakeup(
            MonitorActivationReason activationReason,
            CancellationToken cancellationToken);
 
        protected async ValueTask BeforeApplyDmb(Models.CompileJob newCompileJob, CancellationToken cancellationToken)
        {
            if (newCompileJob.Id == ActiveCompileJob?.Id)
            {
                Logger.LogTrace("Same compile job, not sending deployment event");
                return;
            }
 
            var remoteDeploymentManager = remoteDeploymentManagerFactory.CreateRemoteDeploymentManager(
                metadata,
                newCompileJob);
 
            var eventTask = eventConsumer.HandleEvent(
                EventType.DeploymentActivation,
                new List<string?>
                {
                    GameIOManager.ResolvePath(newCompileJob.DirectoryName!.Value.ToString()),
                },
                false,
                cancellationToken);
 
            try
            {
                await remoteDeploymentManager.ApplyDeployment(newCompileJob, cancellationToken);
            }
            catch (Exception ex)
            {
                Logger.LogWarning(ex, "Failed to apply remote deployment!");
            }
 
            await eventTask;
        }
 
        protected async ValueTask HandleEventImpl(EventType eventType, IEnumerable<string> parameters, bool relayToSession, CancellationToken cancellationToken)
        {
            try
            {
                var sessionEventTask = relayToSession ? ((IEventConsumer)this).HandleEvent(eventType, parameters, false, cancellationToken) : ValueTask.CompletedTask;
                var eventConsumerTask = eventConsumer.HandleEvent(eventType, parameters, false, cancellationToken);
                await ValueTaskExtensions.WhenAll(
                    eventConsumerTask,
                    sessionEventTask);
            }
            catch (JobException ex)
            {
                Logger.LogError(ex, "Suppressing exception triggered by event!");
            }
        }
 
        async ValueTask MonitorRestart(CancellationToken cancellationToken)
        {
            Logger.LogTrace("Monitor restart!");
 
            await DisposeAndNullControllers(cancellationToken);
 
            for (var retryAttempts = 1; ; ++retryAttempts)
            {
                Status = WatchdogStatus.Restoring;
                Exception launchException;
                using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken))
                    try
                    {
                        // use LaunchImplNoLock without announcements or restarting the monitor
                        await LaunchNoLock(false, false, false, null, cancellationToken);
                        Status = WatchdogStatus.Online;
                        Logger.LogDebug("Relaunch successful, resuming monitor...");
                        return;
                    }
                    catch (Exception e) when (e is not OperationCanceledException)
                    {
                        launchException = e;
                    }
 
                Logger.LogWarning(launchException, "Failed to automatically restart the watchdog! Attempt: {attemptNumber}", retryAttempts);
                Status = WatchdogStatus.DelayedRestart;
 
                var retryDelay = Math.Min(
                    Convert.ToInt32(
                        Math.Pow(2, retryAttempts)),
                    TimeSpan.FromHours(1).TotalSeconds); // max of one hour, increasing by a power of 2 each time
 
                Chat.QueueWatchdogMessage(
                    $"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}s...");
 
                await AsyncDelayer.Delay(
                    TimeSpan.FromSeconds(retryDelay),
                    cancellationToken);
            }
        }
 
        async Task InitialCheckDmbUpdated(CompileJob currentCompileJob)
        {
            var factoryTask = DmbFactory.OnNewerDmb;
 
            var latestCompileJob = await DmbFactory.LatestCompileJob();
            if (latestCompileJob != null && latestCompileJob.Id != currentCompileJob.Id)
            {
                Logger.LogDebug("Found new CompileJob without waiting");
                return;
            }
 
            await factoryTask;
        }
 
#pragma warning disable CA1502
        async Task MonitorLifetimes(CancellationToken cancellationToken)
        {
            Logger.LogTrace("Entered MonitorLifetimes");
            Status = WatchdogStatus.Online;
            using var cancellationTokenLoggingRegistration = cancellationToken.Register(() => Logger.LogTrace("Monitor cancellationToken triggered"));
 
            // this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
            try
            {
                MonitorAction nextAction = MonitorAction.Continue;
                Task? activeServerLifetime = null,
                    activeServerReboot = null,
                    activeServerStartup = null,
                    serverPrimed = null,
                    activeLaunchParametersChanged = null,
                    newDmbAvailable = null,
                    healthCheck = null;
                ISessionController? lastController = null;
                var ranInitialDmbCheck = false;
                for (ulong iteration = 1; nextAction != MonitorAction.Exit; ++iteration)
                    using (LogContext.PushProperty(SerilogContextHelper.WatchdogMonitorIterationContextProperty, iteration))
                    {
                        var nextMonitorWakeupTcs = new TaskCompletionSource();
                        try
                        {
                            Logger.LogTrace("Iteration {iteration} of monitor loop", iteration);
                            nextAction = MonitorAction.Continue;
 
                            var controller = GetActiveController();
 
                            void UpdateMonitoredTasks()
                            {
                                var sameController = lastController == controller;
                                void TryUpdateTask(ref Task? oldTask, Func<Task> newTaskFactory)
                                {
                                    if (sameController && oldTask?.IsCompleted == true)
                                        return;
 
                                    oldTask = newTaskFactory();
                                }
 
                                controller!.RebootGate = nextMonitorWakeupTcs.Task;
 
                                TryUpdateTask(ref activeServerLifetime, () => controller.Lifetime);
                                TryUpdateTask(ref activeServerReboot, () => controller.OnReboot);
                                TryUpdateTask(ref serverPrimed, () => controller.OnPrime);
                                TryUpdateTask(ref activeServerStartup, () => controller.OnStartup);
 
                                if (!sameController)
                                    lastController = controller;
 
                                TryUpdateTask(ref activeLaunchParametersChanged, () => activeParametersUpdated.Task);
                                TryUpdateTask(
                                    ref newDmbAvailable,
                                    () =>
                                    {
                                        var result = ranInitialDmbCheck
                                            ? DmbFactory.OnNewerDmb
                                            : InitialCheckDmbUpdated(controller.CompileJob);
                                        ranInitialDmbCheck = true;
                                        return result;
                                    });
                            }
 
                            if (controller != null)
                            {
                                UpdateMonitoredTasks();
 
                                var healthCheckSeconds = ActiveLaunchParameters.HealthCheckSeconds!.Value;
                                healthCheck = healthCheckSeconds == 0
                                    || !controller.DMApiAvailable
                                    ? Extensions.TaskExtensions.InfiniteTask
                                    : Task.Delay(
                                        TimeSpan.FromSeconds(healthCheckSeconds),
                                        cancellationToken);
 
                                // cancel waiting if requested
                                var toWaitOn = Task.WhenAny(
                                    activeServerLifetime!,
                                    activeServerReboot!,
                                    activeServerStartup!,
                                    healthCheck,
                                    newDmbAvailable!,
                                    activeLaunchParametersChanged!,
                                    serverPrimed!);
 
                                // wait for something to happen
                                await toWaitOn.WaitAsync(cancellationToken);
                            }
                            else
                            {
                                Logger.LogError("Controller was null on monitor wakeup! Attempting restart...");
                                nextAction = MonitorAction.Restart; // excuse me wtf?
                            }
 
                            cancellationToken.ThrowIfCancellationRequested();
                            Logger.LogTrace("Monitor activated");
 
                            // always run HandleMonitorWakeup from the context of the semaphore lock
                            using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken))
                            {
                                // Set this sooner so chat sends don't hold us up
                                if (activeServerLifetime!.IsCompleted)
                                    Status = WatchdogStatus.Restoring;
 
                                // multiple things may have happened, handle them one at a time
                                for (var moreActivationsToProcess = true; moreActivationsToProcess && (nextAction == MonitorAction.Continue || nextAction == MonitorAction.Skip);)
                                {
                                    MonitorActivationReason activationReason = default; // this will always be assigned before being used
 
                                    bool CheckActivationReason(ref Task? task, MonitorActivationReason testActivationReason)
                                    {
                                        var taskCompleted = task?.IsCompleted == true;
                                        task = null;
                                        if (nextAction == MonitorAction.Skip)
                                            nextAction = MonitorAction.Continue;
                                        else if (taskCompleted)
                                        {
                                            activationReason = testActivationReason;
                                            return true;
                                        }
 
                                        return false;
                                    }
 
                                    // process the tasks in this order and call HandlerMonitorWakup for each depending on the new monitorState
                                    var anyActivation = CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed)
                                        || CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted)
                                        || CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
                                        || CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated)
                                        || CheckActivationReason(ref healthCheck, MonitorActivationReason.HealthCheck)
                                        || CheckActivationReason(ref serverPrimed, MonitorActivationReason.ActiveServerPrimed)
                                        || CheckActivationReason(ref activeServerStartup, MonitorActivationReason.ActiveServerStartup);
 
                                    UpdateMonitoredTasks();
 
                                    if (!anyActivation)
                                        moreActivationsToProcess = false;
                                    else
                                    {
                                        Logger.LogTrace("Reason: {activationReason}", activationReason);
                                        if (activationReason == MonitorActivationReason.HealthCheck)
                                            nextAction = await HandleHealthCheck(
                                                cancellationToken);
                                        else
                                            nextAction = await HandleMonitorWakeup(
                                                activationReason,
                                                cancellationToken);
                                    }
                                }
                            }
 
                            Logger.LogTrace("Next monitor action is to {nextAction}", nextAction);
 
                            // Restart if requested
                            if (nextAction == MonitorAction.Restart)
                            {
                                await MonitorRestart(cancellationToken);
                                nextAction = MonitorAction.Continue;
                            }
                        }
                        catch (Exception e) when (e is not OperationCanceledException)
                        {
                            // really, this should NEVER happen
                            Logger.LogError(
                                e,
                                "Monitor crashed! Iteration: {iteration}",
                                iteration);
 
                            var nextActionMessage = nextAction != MonitorAction.Exit
                                ? "Recovering"
                                : "Shutting down";
                            Chat.QueueWatchdogMessage(
                                $"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}");
 
                            if (disposed)
                                nextAction = MonitorAction.Exit;
                            else if (nextAction != MonitorAction.Exit)
                            {
                                if (GetActiveController()?.Lifetime.IsCompleted != true)
                                    await MonitorRestart(cancellationToken);
                                else
                                    Logger.LogDebug("Server seems to be okay, not restarting");
                                nextAction = MonitorAction.Continue;
                            }
                        }
                        finally
                        {
                            nextMonitorWakeupTcs.SetResult();
                        }
                    }
            }
            catch (OperationCanceledException)
            {
                // stop signal
                Logger.LogDebug("Monitor cancelled");
 
                if (releaseServers)
                {
                    Logger.LogTrace("Detaching server...");
                    var controller = GetActiveController();
                    if (controller != null)
                        await controller.Release();
                    else
                        Logger.LogError("Controller was null on monitor shutdown!");
                }
            }
 
            // DCT: Operation must always run
            await DisposeAndNullControllers(CancellationToken.None);
            Status = WatchdogStatus.Offline;
 
            Logger.LogTrace("Monitor exiting...");
        }
#pragma warning restore CA1502
 
        async ValueTask TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken)
        {
            if (Status == WatchdogStatus.Offline)
                return;
            if (!graceful)
            {
                var eventTask = HandleEventImpl(
                    releaseServers
                        ? EventType.WatchdogDetach
                        : EventType.WatchdogShutdown,
                    Enumerable.Empty<string>(),
                    releaseServers,
                    cancellationToken);
 
                if (announce)
                    Chat.QueueWatchdogMessage("Shutting down...");
 
                await eventTask;
 
                await StopMonitor();
 
                LastLaunchParameters = null;
                return;
            }
 
            // merely set the reboot state
            var toKill = GetActiveController();
            if (toKill != null)
            {
                await toKill.SetRebootState(Session.RebootState.Shutdown, cancellationToken);
                Logger.LogTrace("Graceful termination requested");
            }
            else
                Logger.LogTrace("Could not gracefully terminate as there is no active controller!");
        }
 
        async ValueTask<MonitorAction> HandleHealthCheck(CancellationToken cancellationToken)
        {
            Logger.LogTrace("Sending health check to active server...");
            var activeServer = GetActiveController();
            if (activeServer == null)
                return MonitorAction.Restart; // uhhhh???
 
            var response = await activeServer.SendCommand(new TopicParameters(), cancellationToken);
 
            var shouldShutdown = activeServer.RebootState == Session.RebootState.Shutdown;
            if (response == null)
            {
                switch (++healthChecksMissed)
                {
                    case 1:
                        Logger.LogDebug("DEFCON 4: Game server missed first health check!");
                        break;
                    case 2:
                        const string message2 = "DEFCON 3: Game server has missed 2 health checks!";
                        Logger.LogInformation(message2);
                        Chat.QueueWatchdogMessage(message2);
                        break;
                    case 3:
                        var actionToTake = shouldShutdown
                            ? "shutdown"
                            : "be restarted";
                        const string logTemplate1 = "DEFCON 2: Game server has missed 3 health checks! If it does not respond to the next one, the watchdog will {actionToTake}!";
                        Logger.LogWarning(logTemplate1, actionToTake);
                        Chat.QueueWatchdogMessage(
                            logTemplate1.Replace(
                                "{actionToTake}",
                                actionToTake,
                                StringComparison.Ordinal));
                        break;
                    case 4:
                        var actionTaken = shouldShutdown
                            ? "Shutting down due to graceful termination request"
                            : "Restarting";
                        const string logTemplate2 = "DEFCON 1: Four health checks have been missed! {actionTaken}...";
                        Logger.LogWarning(logTemplate2, actionTaken);
                        Chat.QueueWatchdogMessage(
                            logTemplate2.Replace(
                                "{actionTaken}",
                                actionTaken,
                                StringComparison.Ordinal));
 
                        if (ActiveLaunchParameters.DumpOnHealthCheckRestart!.Value)
                        {
                            Logger.LogDebug("DumpOnHealthCheckRestart enabled.");
                            try
                            {
                                await CreateDumpNoLock(cancellationToken);
                            }
                            catch (JobException ex)
                            {
                                Logger.LogWarning(ex, "Creating dump failed!");
                            }
                            catch (Win32Exception ex)
                            {
                                Logger.LogWarning(ex, "Creating dump failed!");
                            }
                        }
                        else
                            Logger.LogTrace("DumpOnHealthCheckRestart disabled.");
 
                        await DisposeAndNullControllers(cancellationToken);
                        return shouldShutdown ? MonitorAction.Exit : MonitorAction.Restart;
                    default:
                        Logger.LogError("Invalid health checks missed count: {healthChecksMissed}", healthChecksMissed);
                        break;
                }
            }
            else
            {
                healthChecksMissed = 0;
                ClientCount = response.ClientCount;
            }
 
            return MonitorAction.Continue;
        }
 
        void HandleChatResponses(TopicResponse? result)
        {
            if (result?.ChatResponses != null)
            {
                var warnedMissingChannelIds = false;
                foreach (var response in result.ChatResponses
                    .Where(response =>
                    {
                        if (response.ChannelIds == null)
                        {
                            if (!warnedMissingChannelIds)
                            {
                                Logger.LogWarning("DMAPI response contains null channelIds!");
                                warnedMissingChannelIds = true;
                            }
 
                            return false;
                        }
 
                        return true;
                    }))
                    Chat.QueueMessage(
                        response,
                        response.ChannelIds!
                            .Select(channelIdString =>
                            {
                                if (UInt64.TryParse(channelIdString, out var channelId))
                                    return (ulong?)channelId;
                                else
                                    Logger.LogWarning("Could not parse chat response channel ID: {channelID}", channelIdString);
 
                                return null;
                            })
                            .Where(nullableChannelId => nullableChannelId.HasValue)
                            .Select(nullableChannelId => nullableChannelId!.Value));
            }
        }
 
        async ValueTask CreateDumpNoLock(CancellationToken cancellationToken)
        {
            const string DumpDirectory = "ProcessDumps";
 
            var session = GetActiveController();
            if (session?.Lifetime.IsCompleted != false)
                throw new JobException(ErrorCode.GameServerOffline);
 
            var dumpFileExtension = session.DumpFileExtension;
 
            var dumpFileNameTemplate = diagnosticsIOManager.ResolvePath(
                diagnosticsIOManager.ConcatPath(
                    DumpDirectory,
                    $"DreamDaemon-{DateTimeOffset.UtcNow.ToFileStamp()}"));
 
            var dumpFileName = $"{dumpFileNameTemplate}{dumpFileExtension}";
            var iteration = 0;
            while (await diagnosticsIOManager.FileExists(dumpFileName, cancellationToken))
                dumpFileName = $"{dumpFileNameTemplate} ({++iteration}){dumpFileExtension}";
 
            if (iteration == 0)
                await diagnosticsIOManager.CreateDirectory(DumpDirectory, cancellationToken);
 
            if (session.Lifetime.IsCompleted)
                throw new JobException(ErrorCode.GameServerOffline);
 
            Logger.LogInformation("Dumping session to {dumpFileName}...", dumpFileName);
            await session.CreateDump(dumpFileName, ActiveLaunchParameters.Minidumps!.Value, cancellationToken);
        }
    }
}