DmbFactory.cs

Go to the documentation of this file.

using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
 
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
 
using Tgstation.Server.Api.Models;
using Tgstation.Server.Common.Extensions;
using Tgstation.Server.Host.Components.Deployment.Remote;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Database;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.Models;
using Tgstation.Server.Host.Utils;
 
namespace Tgstation.Server.Host.Components.Deployment
{
    sealed class DmbFactory : IDmbFactory, ICompileJobSink
    {
        public Task OnNewerDmb
        {
            get
            {
                lock (jobLockManagers)
                    return newerDmbTcs.Task;
            }
        }
 
        [MemberNotNullWhen(true, nameof(nextLockManager))]
        public bool DmbAvailable => nextLockManager != null;
 
        readonly IDatabaseContextFactory databaseContextFactory;
 
        readonly IIOManager ioManager;
 
        readonly IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory;
 
        readonly ILogger<DmbFactory> logger;
 
        readonly IEventConsumer eventConsumer;
 
        readonly IAsyncDelayer asyncDelayer;
 
        readonly Api.Models.Instance metadata;
 
        readonly CancellationTokenSource cleanupCts;
 
        readonly CancellationTokenSource lockLogCts;
 
        readonly Dictionary<long, DeploymentLockManager> jobLockManagers;
 
        volatile TaskCompletionSource newerDmbTcs;
 
        Task cleanupTask;
 
        DeploymentLockManager? nextLockManager;
 
        bool started;
 
        public DmbFactory(
            IDatabaseContextFactory databaseContextFactory,
            IIOManager ioManager,
            IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
            IEventConsumer eventConsumer,
            IAsyncDelayer asyncDelayer,
            ILogger<DmbFactory> logger,
            Api.Models.Instance metadata)
        {
            this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
            this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
            this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
            this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
            this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
            this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
            this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
 
            cleanupTask = Task.CompletedTask;
            newerDmbTcs = new TaskCompletionSource();
            cleanupCts = new CancellationTokenSource();
            lockLogCts = new CancellationTokenSource();
            jobLockManagers = new Dictionary<long, DeploymentLockManager>();
        }
 
        public void Dispose()
        {
            // we don't dispose nextDmbProvider here, since it might be the only thing we have
            lockLogCts.Dispose();
            cleanupCts.Dispose();
        }
 
        public async ValueTask LoadCompileJob(CompileJob job, Action<bool>? activationAction, CancellationToken cancellationToken)
        {
            ArgumentNullException.ThrowIfNull(job);
 
            var (dmbProvider, lockManager) = await FromCompileJobInternal(job, "Compile job loading", cancellationToken);
            if (dmbProvider == null)
                return;
 
            if (lockManager == null)
                throw new InvalidOperationException($"We did not acquire the first lock for compile job {job.Id}!");
 
            // Do this first, because it's entirely possible when we set the tcs it will immediately need to be applied
            if (started)
            {
                var remoteDeploymentManager = remoteDeploymentManagerFactory.CreateRemoteDeploymentManager(
                    metadata,
                    job);
                await remoteDeploymentManager.StageDeployment(
                    lockManager.CompileJob,
                    activationAction,
                    cancellationToken);
            }
 
            ValueTask dmbDisposeTask;
            lock (jobLockManagers)
            {
                dmbDisposeTask = nextLockManager?.DisposeAsync() ?? ValueTask.CompletedTask;
                nextLockManager = lockManager;
 
                // Oh god dammit
                var temp = Interlocked.Exchange(ref newerDmbTcs, new TaskCompletionSource());
                temp.SetResult();
            }
 
            await dmbDisposeTask;
        }
 
        public IDmbProvider LockNextDmb(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
        {
            if (!DmbAvailable)
                throw new InvalidOperationException("No .dmb available!");
 
            return nextLockManager.AddLock(reason, callerFile, callerLine);
        }
 
        public async Task StartAsync(CancellationToken cancellationToken)
        {
            CompileJob? cj = null;
            await databaseContextFactory.UseContext(
                async (db) =>
                    cj = await db
                        .CompileJobs
                        .AsQueryable()
                        .Where(x => x.Job.Instance!.Id == metadata.Id)
                        .OrderByDescending(x => x.Job.StoppedAt)
                        .FirstOrDefaultAsync(cancellationToken));
 
            try
            {
                if (cj == default(CompileJob))
                    return;
                await LoadCompileJob(cj, null, cancellationToken);
            }
            finally
            {
                started = true;
            }
 
            // we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet
            cleanupTask = Task.WhenAll(cleanupTask, LogLockStatesLoop());
        }
 
        public async Task StopAsync(CancellationToken cancellationToken)
        {
            try
            {
                lockLogCts.Cancel();
 
                lock (jobLockManagers)
                    remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockManagers.Keys);
 
                using (cancellationToken.Register(() => cleanupCts.Cancel()))
                    await cleanupTask;
            }
            finally
            {
                started = false;
            }
        }
 
#pragma warning disable CA1506 // TODO: Decomplexify
        public async ValueTask<IDmbProvider?> FromCompileJob(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
        {
            ArgumentNullException.ThrowIfNull(compileJob);
            ArgumentNullException.ThrowIfNull(reason);
 
            var (dmb, _) = await FromCompileJobInternal(compileJob, reason, cancellationToken, callerFile, callerLine);
 
            return dmb;
        }
 
#pragma warning disable CA1506 // TODO: Decomplexify
        public async ValueTask CleanUnusedCompileJobs(CancellationToken cancellationToken)
        {
            List<long> jobIdsToSkip;
 
            // don't clean locked directories
            lock (jobLockManagers)
                jobIdsToSkip = jobLockManagers.Keys.ToList();
 
            List<string>? jobUidsToNotErase = null;
 
            // find the uids of locked directories
            if (jobIdsToSkip.Count > 0)
            {
                await databaseContextFactory.UseContext(async db =>
                {
                    jobUidsToNotErase = (await db
                            .CompileJobs
                            .AsQueryable()
                            .Where(
                                x => x.Job.Instance!.Id == metadata.Id
                                && jobIdsToSkip.Contains(x.Id!.Value))
                            .Select(x => x.DirectoryName!.Value)
                            .ToListAsync(cancellationToken))
                        .Select(x => x.ToString())
                        .ToList();
                });
            }
            else
                jobUidsToNotErase = new List<string>();
 
            jobUidsToNotErase!.Add(SwappableDmbProvider.LiveGameDirectory);
 
            logger.LogTrace("We will not clean the following directories: {directoriesToNotClean}", String.Join(", ", jobUidsToNotErase));
 
            // cleanup
            var gameDirectory = ioManager.ResolvePath();
            await ioManager.CreateDirectory(gameDirectory, cancellationToken);
            var directories = await ioManager.GetDirectories(gameDirectory, cancellationToken);
            int deleting = 0;
            var tasks = directories.Select<string, ValueTask>(async x =>
            {
                var nameOnly = ioManager.GetFileName(x);
                if (jobUidsToNotErase.Contains(nameOnly))
                    return;
                logger.LogDebug("Cleaning unused game folder: {dirName}...", nameOnly);
                try
                {
                    ++deleting;
                    await DeleteCompileJobContent(x, cancellationToken);
                }
                catch (Exception e) when (e is not OperationCanceledException)
                {
                    logger.LogWarning(e, "Error deleting directory {dirName}!", x);
                }
            }).ToList();
            if (deleting > 0)
                await ValueTaskExtensions.WhenAll(tasks);
        }
#pragma warning restore CA1506
 
        public async ValueTask<CompileJob?> LatestCompileJob()
        {
            if (!DmbAvailable)
                return null;
 
            await using IDmbProvider provider = LockNextDmb("Checking latest CompileJob");
 
            return provider.CompileJob;
        }
 
        public void LogLockStates()
        {
            var builder = new StringBuilder();
 
            lock (jobLockManagers)
                foreach (var lockManager in jobLockManagers.Values)
                    lockManager.LogLockStats(builder);
 
            logger.LogTrace("Periodic deployment log states report:{newLine}{report}", Environment.NewLine, builder);
        }
 
        async ValueTask<(IDmbProvider? DmbProvider, DeploymentLockManager? LockManager)> FromCompileJobInternal(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
        {
            // ensure we have the entire metadata tree
            var compileJobId = compileJob.Require(x => x.Id);
            lock (jobLockManagers)
                if (jobLockManagers.TryGetValue(compileJobId, out var lockManager))
                    return (DmbProvider: lockManager.AddLock(reason, callerFile, callerLine), LockManager: null); // fast path
 
            logger.LogTrace("Loading compile job {id}...", compileJobId);
            await databaseContextFactory.UseContext(
                async db => compileJob = await db
                    .CompileJobs
                    .AsQueryable()
                    .Where(x => x!.Id == compileJobId)
                    .Include(x => x.Job!)
                        .ThenInclude(x => x.StartedBy)
                    .Include(x => x.Job!)
                        .ThenInclude(x => x.Instance)
                    .Include(x => x.RevisionInformation!)
                        .ThenInclude(x => x.PrimaryTestMerge!)
                            .ThenInclude(x => x.MergedBy)
                    .Include(x => x.RevisionInformation!)
                        .ThenInclude(x => x.ActiveTestMerges!)
                            .ThenInclude(x => x.TestMerge!)
                                .ThenInclude(x => x.MergedBy)
                    .FirstAsync(cancellationToken)); // can't wait to see that query
 
            EngineVersion engineVersion;
            if (!EngineVersion.TryParse(compileJob.EngineVersion, out var engineVersionNullable))
            {
                logger.LogError("Error loading compile job, bad engine version: {engineVersion}", compileJob.EngineVersion);
                return (null, null); // omae wa mou shinderu
            }
            else
                engineVersion = engineVersionNullable!;
 
            if (!compileJob.Job.StoppedAt.HasValue)
            {
                // This happens when we're told to load the compile job that is currently finished up
                // It constitutes an API violation if it's returned by the DreamDaemonController so just set it here
                // Bit of a hack, but it works out to be nearly if not the same value that's put in the DB
                logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{id}...", compileJob.Job.Id);
                compileJob.Job.StoppedAt = DateTimeOffset.UtcNow;
            }
 
            var providerSubmitted = false;
            void CleanupAction()
            {
                if (providerSubmitted)
                    CleanRegisteredCompileJob(compileJob);
            }
 
            var newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction));
            try
            {
                const string LegacyADirectoryName = "A";
                const string LegacyBDirectoryName = "B";
 
                var dmbExistsAtRoot = await ioManager.FileExists(
                    ioManager.ConcatPath(
                        newProvider.Directory,
                        newProvider.DmbName),
                    cancellationToken);
 
                if (!dmbExistsAtRoot)
                {
                    logger.LogTrace("Didn't find .dmb at game directory root, checking A/B dirs...");
                    var primaryCheckTask = ioManager.FileExists(
                        ioManager.ConcatPath(
                            newProvider.Directory,
                            LegacyADirectoryName,
                            newProvider.DmbName),
                        cancellationToken);
                    var secondaryCheckTask = ioManager.FileExists(
                        ioManager.ConcatPath(
                            newProvider.Directory,
                            LegacyBDirectoryName,
                            newProvider.DmbName),
                        cancellationToken);
 
                    if (!(await primaryCheckTask && await secondaryCheckTask))
                    {
                        logger.LogWarning("Error loading compile job, .dmb missing!");
                        return (null, null); // omae wa mou shinderu
                    }
 
                    // rebuild the provider because it's using the legacy style directories
                    // Don't dispose it
                    logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName);
#pragma warning disable CA2000 // Dispose objects before losing scope (false positive)
                    newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), LegacyADirectoryName);
#pragma warning restore CA2000 // Dispose objects before losing scope
                }
 
                lock (jobLockManagers)
                {
                    IDmbProvider lockedProvider;
                    if (!jobLockManagers.TryGetValue(compileJobId, out var lockManager))
                    {
                        lockManager = DeploymentLockManager.Create(newProvider, logger, reason, out lockedProvider);
                        jobLockManagers.Add(compileJobId, lockManager);
 
                        providerSubmitted = true;
                    }
                    else
                    {
                        lockedProvider = lockManager.AddLock(reason, callerFile, callerLine); // race condition
                        lockManager = null;
                    }
 
                    return (DmbProvider: lockedProvider, LockManager: lockManager);
                }
            }
            finally
            {
                if (!providerSubmitted)
                    await newProvider.DisposeAsync();
            }
        }
 
        void CleanRegisteredCompileJob(CompileJob job)
        {
            Task HandleCleanup()
            {
                lock (jobLockManagers)
                    jobLockManagers.Remove(job.Require(x => x.Id));
 
                var otherTask = cleanupTask;
 
                async Task WrapThrowableTasks()
                {
                    try
                    {
                        // First kill the GitHub deployment
                        var remoteDeploymentManager = remoteDeploymentManagerFactory.CreateRemoteDeploymentManager(metadata, job);
 
                        var cancellationToken = cleanupCts.Token;
                        var deploymentJob = remoteDeploymentManager.MarkInactive(job, cancellationToken);
 
                        var deleteTask = DeleteCompileJobContent(job.DirectoryName!.Value.ToString(), cancellationToken);
 
                        await ValueTaskExtensions.WhenAll(deleteTask, deploymentJob);
                    }
                    catch (Exception ex) when (ex is not OperationCanceledException)
                    {
                        logger.LogWarning(ex, "Error cleaning up compile job {jobGuid}!", job.DirectoryName);
                    }
                }
 
                return Task.WhenAll(otherTask, WrapThrowableTasks());
            }
 
            lock (cleanupCts)
                cleanupTask = HandleCleanup();
        }
 
        async ValueTask DeleteCompileJobContent(string directory, CancellationToken cancellationToken)
        {
            // Then call the cleanup event, waiting here first
            await eventConsumer.HandleEvent(EventType.DeploymentCleanup, new List<string> { ioManager.ResolvePath(directory) }, true, cancellationToken);
            await ioManager.DeleteDirectory(directory, cancellationToken);
        }
 
        async Task LogLockStatesLoop()
        {
            logger.LogTrace("Entering lock logging loop");
            CancellationToken cancellationToken = lockLogCts.Token;
 
            while (!cancellationToken.IsCancellationRequested)
                try
                {
                    LogLockStates();
                    await asyncDelayer.Delay(TimeSpan.FromMinutes(10), cancellationToken);
                }
                catch (OperationCanceledException ex)
                {
                    logger.LogTrace(ex, "Exiting lock logging loop");
                    break;
                }
        }
    }
}