tgstation-server 6.19.0
The /tg/station 13 server suite
Loading...
Searching...
No Matches
DmbFactory.cs
Go to the documentation of this file.
1using System;
2using System.Collections.Generic;
3using System.Diagnostics.CodeAnalysis;
4using System.Linq;
5using System.Runtime.CompilerServices;
6using System.Text;
7using System.Threading;
8using System.Threading.Tasks;
9
10using Microsoft.EntityFrameworkCore;
11using Microsoft.Extensions.Logging;
12
21
23{
28 {
30 public Task OnNewerDmb
31 {
32 get
33 {
34 lock (jobLockManagers)
35 return newerDmbTcs.Task;
36 }
37 }
38
40 [MemberNotNullWhen(true, nameof(nextLockManager))]
41 public bool DmbAvailable => nextLockManager != null;
42
47
52
57
61 readonly ILogger<DmbFactory> logger;
62
67
72
77
81 readonly CancellationTokenSource cleanupCts;
82
86 readonly CancellationTokenSource lockLogCts;
87
91 readonly Dictionary<long, DeploymentLockManager> jobLockManagers;
92
96 volatile TaskCompletionSource newerDmbTcs;
97
102
107
112
129 ILogger<DmbFactory> logger,
130 Api.Models.Instance metadata)
131 {
132 this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
133 this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
134 this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
135 this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
136 this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
137 this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
138 this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
139
140 cleanupTask = Task.CompletedTask;
141 newerDmbTcs = new TaskCompletionSource();
142 cleanupCts = new CancellationTokenSource();
143 lockLogCts = new CancellationTokenSource();
144 jobLockManagers = new Dictionary<long, DeploymentLockManager>();
145 }
146
148 public void Dispose()
149 {
150 // we don't dispose nextDmbProvider here, since it might be the only thing we have
151 lockLogCts.Dispose();
152 cleanupCts.Dispose();
153 }
154
156 public async ValueTask LoadCompileJob(CompileJob job, Action<bool>? activationAction, CancellationToken cancellationToken)
157 {
158 ArgumentNullException.ThrowIfNull(job);
159
160 var (dmbProvider, lockManager) = await FromCompileJobInternal(job, "Compile job loading", cancellationToken);
161 if (dmbProvider == null)
162 return;
163
164 if (lockManager == null)
165 throw new InvalidOperationException($"We did not acquire the first lock for compile job {job.Id}!");
166
167 // Do this first, because it's entirely possible when we set the tcs it will immediately need to be applied
168 if (started)
169 {
171 metadata,
172 job);
173 await remoteDeploymentManager.StageDeployment(
174 lockManager.CompileJob,
175 activationAction,
176 cancellationToken);
177 }
178
179 ValueTask dmbDisposeTask;
180 lock (jobLockManagers)
181 {
182 dmbDisposeTask = nextLockManager?.DisposeAsync() ?? ValueTask.CompletedTask;
183 nextLockManager = lockManager;
184
185 // Oh god dammit
186 var temp = Interlocked.Exchange(ref newerDmbTcs, new TaskCompletionSource());
187 temp.SetResult();
188 }
189
190 await dmbDisposeTask;
191 }
192
194 public IDmbProvider LockNextDmb(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
195 {
196 if (!DmbAvailable)
197 throw new InvalidOperationException("No .dmb available!");
198
199 return nextLockManager.AddLock(reason, callerFile, callerLine);
200 }
201
203 public async Task StartAsync(CancellationToken cancellationToken)
204 {
205 CompileJob? cj = null;
207 async (db) =>
208 cj = await db
209 .CompileJobs
210 .Where(x => x.Job.Instance!.Id == metadata.Id)
211 .OrderByDescending(x => x.Job.StoppedAt)
212 .FirstOrDefaultAsync(cancellationToken));
213
214 try
215 {
216 if (cj == default(CompileJob))
217 return;
218 await LoadCompileJob(cj, null, cancellationToken);
219 }
220 finally
221 {
222 started = true;
223 }
224
225 // we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet
226 cleanupTask = Task.WhenAll(cleanupTask, LogLockStatesLoop());
227 }
228
230 public async Task StopAsync(CancellationToken cancellationToken)
231 {
232 try
233 {
234 lockLogCts.Cancel();
235
236 lock (jobLockManagers)
238
239 using (cancellationToken.Register(() => cleanupCts.Cancel()))
240 await cleanupTask;
241 }
242 finally
243 {
244 started = false;
245 }
246 }
247
249#pragma warning disable CA1506 // TODO: Decomplexify
250 public async ValueTask<IDmbProvider?> FromCompileJob(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
251 {
252 ArgumentNullException.ThrowIfNull(compileJob);
253 ArgumentNullException.ThrowIfNull(reason);
254
255 var (dmb, _) = await FromCompileJobInternal(compileJob, reason, cancellationToken, callerFile, callerLine);
256
257 return dmb;
258 }
259
261#pragma warning disable CA1506 // TODO: Decomplexify
262 public async ValueTask CleanUnusedCompileJobs(CancellationToken cancellationToken)
263 {
264 List<long> jobIdsToSkip;
265
266 // don't clean locked directories
267 lock (jobLockManagers)
268 jobIdsToSkip = jobLockManagers.Keys.ToList();
269
270 List<string>? jobUidsToNotErase = null;
271
272 // find the uids of locked directories
273 if (jobIdsToSkip.Count > 0)
274 {
275 await databaseContextFactory.UseContext(async db =>
276 {
277 jobUidsToNotErase = (await db
278 .CompileJobs
279 .Where(
280 x => x.Job.Instance!.Id == metadata.Id
281 && jobIdsToSkip.Contains(x.Id!.Value))
282 .Select(x => x.DirectoryName!.Value)
283 .ToListAsync(cancellationToken))
284 .Select(x => x.ToString())
285 .ToList();
286 });
287 }
288 else
289 jobUidsToNotErase = new List<string>();
290
291 jobUidsToNotErase!.Add(SwappableDmbProvider.LiveGameDirectory);
292
293 logger.LogTrace("We will not clean the following directories: {directoriesToNotClean}", String.Join(", ", jobUidsToNotErase));
294
295 // cleanup
296 var gameDirectory = ioManager.ResolvePath();
297 await ioManager.CreateDirectory(gameDirectory, cancellationToken);
298 var directories = await ioManager.GetDirectories(gameDirectory, cancellationToken);
299 int deleting = 0;
300 var tasks = directories.Select<string, ValueTask>(async x =>
301 {
302 var nameOnly = ioManager.GetFileName(x);
303 if (jobUidsToNotErase.Contains(nameOnly))
304 return;
305 logger.LogDebug("Cleaning unused game folder: {dirName}...", nameOnly);
306 try
307 {
308 ++deleting;
309 await DeleteCompileJobContent(x, cancellationToken);
310 }
311 catch (Exception e) when (e is not OperationCanceledException)
312 {
313 logger.LogWarning(e, "Error deleting directory {dirName}!", x);
314 }
315 }).ToList();
316 if (deleting > 0)
317 await ValueTaskExtensions.WhenAll(tasks);
318 }
319#pragma warning restore CA1506
320
322 public async ValueTask<CompileJob?> LatestCompileJob()
323 {
324 if (!DmbAvailable)
325 return null;
326
327 await using IDmbProvider provider = LockNextDmb("Checking latest CompileJob");
328
329 return provider.CompileJob;
330 }
331
333 public void LogLockStates()
334 {
335 var builder = new StringBuilder();
336
337 lock (jobLockManagers)
338 foreach (var lockManager in jobLockManagers.Values)
339 lockManager.LogLockStats(builder);
340
341 logger.LogTrace("Periodic deployment log states report:{newLine}{report}", Environment.NewLine, builder);
342 }
343
353 async ValueTask<(IDmbProvider? DmbProvider, DeploymentLockManager? LockManager)> FromCompileJobInternal(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
354 {
355 // ensure we have the entire metadata tree
356 var compileJobId = compileJob.Require(x => x.Id);
357 lock (jobLockManagers)
358 if (jobLockManagers.TryGetValue(compileJobId, out var lockManager))
359 return (DmbProvider: lockManager.AddLock(reason, callerFile, callerLine), LockManager: null); // fast path
360
361 logger.LogTrace("Loading compile job {id}...", compileJobId);
363 async db => compileJob = await db
364 .CompileJobs
365 .Where(x => x!.Id == compileJobId)
366 .Include(x => x.Job!)
367 .ThenInclude(x => x.StartedBy)
368 .Include(x => x.Job!)
369 .ThenInclude(x => x.Instance)
370 .Include(x => x.RevisionInformation!)
371 .ThenInclude(x => x.PrimaryTestMerge!)
372 .ThenInclude(x => x.MergedBy)
373 .Include(x => x.RevisionInformation!)
374 .ThenInclude(x => x.ActiveTestMerges!)
375 .ThenInclude(x => x.TestMerge!)
376 .ThenInclude(x => x.MergedBy)
377 .FirstAsync(cancellationToken)); // can't wait to see that query
378
379 EngineVersion engineVersion;
380 if (!EngineVersion.TryParse(compileJob.EngineVersion, out var engineVersionNullable))
381 {
382 logger.LogError("Error loading compile job, bad engine version: {engineVersion}", compileJob.EngineVersion);
383 return (null, null); // omae wa mou shinderu
384 }
385 else
386 engineVersion = engineVersionNullable!;
387
388 if (!compileJob.Job.StoppedAt.HasValue)
389 {
390 // This happens when we're told to load the compile job that is currently finished up
391 // It constitutes an API violation if it's returned by the DreamDaemonController so just set it here
392 // Bit of a hack, but it works out to be nearly if not the same value that's put in the DB
393 logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{id}...", compileJob.Job.Id);
394 compileJob.Job.StoppedAt = DateTimeOffset.UtcNow;
395 }
396
397 var providerSubmitted = false;
398 void CleanupAction()
399 {
400 if (providerSubmitted)
401 CleanRegisteredCompileJob(compileJob);
402 }
403
404 var newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction));
405 try
406 {
407 const string LegacyADirectoryName = "A";
408 const string LegacyBDirectoryName = "B";
409
410 var dmbExistsAtRoot = await ioManager.FileExists(
412 newProvider.Directory,
413 newProvider.DmbName),
414 cancellationToken);
415
416 if (!dmbExistsAtRoot)
417 {
418 logger.LogTrace("Didn't find .dmb at game directory root, checking A/B dirs...");
419 var primaryCheckTask = ioManager.FileExists(
421 newProvider.Directory,
422 LegacyADirectoryName,
423 newProvider.DmbName),
424 cancellationToken);
425 var secondaryCheckTask = ioManager.FileExists(
427 newProvider.Directory,
428 LegacyBDirectoryName,
429 newProvider.DmbName),
430 cancellationToken);
431
432 if (!(await primaryCheckTask && await secondaryCheckTask))
433 {
434 logger.LogWarning("Error loading compile job, .dmb missing!");
435 return (null, null); // omae wa mou shinderu
436 }
437
438 // rebuild the provider because it's using the legacy style directories
439 // Don't dispose it
440 logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName);
441#pragma warning disable CA2000 // Dispose objects before losing scope (false positive)
442 newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), LegacyADirectoryName);
443#pragma warning restore CA2000 // Dispose objects before losing scope
444 }
445
446 lock (jobLockManagers)
447 {
448 IDmbProvider lockedProvider;
449 if (!jobLockManagers.TryGetValue(compileJobId, out var lockManager))
450 {
451 lockManager = DeploymentLockManager.Create(newProvider, logger, reason, out lockedProvider);
452 jobLockManagers.Add(compileJobId, lockManager);
453
454 providerSubmitted = true;
455 }
456 else
457 {
458 lockedProvider = lockManager.AddLock(reason, callerFile, callerLine); // race condition
459 lockManager = null;
460 }
461
462 return (DmbProvider: lockedProvider, LockManager: lockManager);
463 }
464 }
465 finally
466 {
467 if (!providerSubmitted)
468 await newProvider.DisposeAsync();
469 }
470 }
471
477 {
478 Task HandleCleanup()
479 {
480 lock (jobLockManagers)
481 jobLockManagers.Remove(job.Require(x => x.Id));
482
483 var otherTask = cleanupTask;
484
485 async Task WrapThrowableTasks()
486 {
487 try
488 {
489 // First kill the GitHub deployment
491
492 var cancellationToken = cleanupCts.Token;
493 var deploymentJob = remoteDeploymentManager.MarkInactive(job, cancellationToken);
494
495 var deleteTask = DeleteCompileJobContent(job.DirectoryName!.Value.ToString(), cancellationToken);
496
497 await ValueTaskExtensions.WhenAll(deleteTask, deploymentJob);
498 }
499 catch (Exception ex) when (ex is not OperationCanceledException)
500 {
501 logger.LogWarning(ex, "Error cleaning up compile job {jobGuid}!", job.DirectoryName);
502 }
503 }
504
505 return Task.WhenAll(otherTask, WrapThrowableTasks());
506 }
507
508 lock (cleanupCts)
509 cleanupTask = HandleCleanup();
510 }
511
518 async ValueTask DeleteCompileJobContent(string directory, CancellationToken cancellationToken)
519 {
520 // Then call the cleanup event, waiting here first
521 await eventConsumer.HandleEvent(EventType.DeploymentCleanup, new List<string> { ioManager.ResolvePath(directory) }, true, cancellationToken);
522 await ioManager.DeleteDirectory(directory, cancellationToken);
523 }
524
529 async Task LogLockStatesLoop()
530 {
531 logger.LogTrace("Entering lock logging loop");
532 CancellationToken cancellationToken = lockLogCts.Token;
533
534 while (!cancellationToken.IsCancellationRequested)
535 try
536 {
538 await asyncDelayer.Delay(TimeSpan.FromMinutes(10), cancellationToken);
539 }
540 catch (OperationCanceledException ex)
541 {
542 logger.LogTrace(ex, "Exiting lock logging loop");
543 break;
544 }
545 }
546 }
547}
Information about an engine installation.
static bool TryParse(string input, out EngineVersion? engineVersion)
Attempts to parse a stringified EngineVersion.
virtual ? long Id
The ID of the entity.
Definition EntityId.cs:14
Metadata about a server instance.
Definition Instance.cs:9
Guid? DirectoryName
The Game folder the results were compiled into.
Definition CompileJob.cs:29
DateTimeOffset? StoppedAt
When the Job stopped.
Definition Job.cs:49
Extension methods for the ValueTask and ValueTask<TResult> classes.
static async ValueTask WhenAll(IEnumerable< ValueTask > tasks)
Fully await a given list of tasks .
IDmbProvider AddLock(string reason, [CallerFilePath] string? callerFile=null, [CallerLineNumber]int callerLine=default)
Add a lock to the managed IDmbProvider.
static DeploymentLockManager Create(IDmbProvider dmbProvider, ILogger logger, string initialLockReason, out IDmbProvider firstLock, [CallerFilePath] string? callerFile=null, [CallerLineNumber] int callerLine=default)
Create a DeploymentLockManager.
readonly IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory
The IRemoteDeploymentManagerFactory for the DmbFactory.
Definition DmbFactory.cs:56
async ValueTask CleanUnusedCompileJobs(CancellationToken cancellationToken)
Deletes all compile jobs that are inactive in the Game folder.A ValueTask representing the running op...
DmbFactory(IDatabaseContextFactory databaseContextFactory, IIOManager ioManager, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IEventConsumer eventConsumer, IAsyncDelayer asyncDelayer, ILogger< DmbFactory > logger, Api.Models.Instance metadata)
Initializes a new instance of the DmbFactory class.
readonly IIOManager ioManager
The IIOManager for the DmbFactory.
Definition DmbFactory.cs:51
IDmbProvider LockNextDmb(string reason, [CallerFilePath] string? callerFile=null, [CallerLineNumber] int callerLine=default)
Gets the next IDmbProvider. DmbAvailable is a precondition.A new IDmbProvider.
async ValueTask LoadCompileJob(CompileJob job, Action< bool >? activationAction, CancellationToken cancellationToken)
Load a new job into the ICompileJobSink.A ValueTask representing the running operation.
readonly IAsyncDelayer asyncDelayer
The IAsyncDelayer for the DmbFactory.
Definition DmbFactory.cs:71
readonly IEventConsumer eventConsumer
The IEventConsumer for the DmbFactory.
Definition DmbFactory.cs:66
readonly CancellationTokenSource lockLogCts
The CancellationTokenSource for LogLockStatesLoop.
Definition DmbFactory.cs:86
bool DmbAvailable
If LockNextDmb will succeed.
Definition DmbFactory.cs:41
volatile TaskCompletionSource newerDmbTcs
TaskCompletionSource resulting in the latest DmbProvider yet to exist.
Definition DmbFactory.cs:96
readonly ILogger< DmbFactory > logger
The ILogger for the DmbFactory.
Definition DmbFactory.cs:61
void CleanRegisteredCompileJob(CompileJob job)
Delete the Api.Models.Internal.CompileJob.DirectoryName of job .
async Task StopAsync(CancellationToken cancellationToken)
readonly Api.Models.Instance metadata
The Api.Models.Instance for the DmbFactory.
Definition DmbFactory.cs:76
async ValueTask<(IDmbProvider? DmbProvider, DeploymentLockManager? LockManager)> FromCompileJobInternal(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile=null, [CallerLineNumber] int callerLine=default)
Gets a IDmbProvider and potentially the DeploymentLockManager for a given CompileJob.
readonly Dictionary< long, DeploymentLockManager > jobLockManagers
Map of CompileJob.JobIds to locks on them.
Definition DmbFactory.cs:91
DeploymentLockManager? nextLockManager
The DeploymentLockManager for the latest DmbProvider.
Task OnNewerDmb
Get a Task that completes when the result of a call to LockNextDmb will be different than the previou...
Definition DmbFactory.cs:31
async ValueTask< CompileJob?> LatestCompileJob()
Gets the latest CompileJob.A ValueTask<TResult> resulting in the latest CompileJob or null if none ar...
void LogLockStates()
Log the states of all active IDmbProviders.
readonly IDatabaseContextFactory databaseContextFactory
The IDatabaseContextFactory for the DmbFactory.
Definition DmbFactory.cs:46
readonly CancellationTokenSource cleanupCts
The CancellationTokenSource for cleanupTask.
Definition DmbFactory.cs:81
bool started
If the DmbFactory is "started" via IComponentService.
async ValueTask DeleteCompileJobContent(string directory, CancellationToken cancellationToken)
Handles cleaning the resources of a CompileJob.
async Task StartAsync(CancellationToken cancellationToken)
async ValueTask< IDmbProvider?> FromCompileJob(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile=null, [CallerLineNumber] int callerLine=default)
Gets a IDmbProvider for a given CompileJob.A ValueTask<TResult> resulting in a new IDmbProvider repre...
async Task LogLockStatesLoop()
Lock all DeploymentLockManagers states.
Task cleanupTask
Task representing calls to CleanRegisteredCompileJob(CompileJob).
A IDmbProvider that uses filesystem links to change directory structure underneath the server process...
const string LiveGameDirectory
The directory where the BaseProvider is symlinked to.
Job Job
See CompileJobResponse.Job.
Definition CompileJob.cs:16
string EngineVersion
The Version the CompileJob was made with in string form.
Definition CompileJob.cs:33
Runs a given disposeAction on Dispose.
Provides absolute paths to the latest compiled .dmbs.
IRemoteDeploymentManager CreateRemoteDeploymentManager(Api.Models.Instance metadata, RemoteGitProvider remoteGitProvider)
Creates a IRemoteDeploymentManager for a given remoteGitProvider .
void ForgetLocalStateForCompileJobs(IEnumerable< long > compileJobsIds)
Cause the IRemoteDeploymentManagerFactory to drop any local state is has for the given compileJobsIds...
ValueTask MarkInactive(CompileJob compileJob, CancellationToken cancellationToken)
Mark the deplotment for a given compileJob as inactive.
ValueTask StageDeployment(CompileJob compileJob, Action< bool >? activationCallback, CancellationToken cancellationToken)
Stage a given compileJob 's deployment.
Consumes EventTypes and takes the appropriate actions.
ValueTask HandleEvent(EventType eventType, IEnumerable< string?> parameters, bool deploymentPipeline, CancellationToken cancellationToken)
Handle a given eventType .
Factory for scoping usage of IDatabaseContexts. Meant for use by Components.
ValueTask UseContext(Func< IDatabaseContext, ValueTask > operation)
Run an operation in the scope of an IDatabaseContext.
Interface for using filesystems.
Definition IIOManager.cs:14
string GetFileName(string path)
Gets the file name portion of a path .
string ResolvePath()
Retrieve the full path of the current working directory.
string ConcatPath(params string[] paths)
Combines an array of strings into a path.
Task< IReadOnlyList< string > > GetDirectories(string path, CancellationToken cancellationToken)
Returns full directory names in a given path .
Task CreateDirectory(string path, CancellationToken cancellationToken)
Create a directory at path .
Task DeleteDirectory(string path, CancellationToken cancellationToken)
Recursively delete a directory, removes and does not enter any symlinks encounterd.
Task< bool > FileExists(string path, CancellationToken cancellationToken)
Check that the file at path exists.
ValueTask Delay(TimeSpan timeSpan, CancellationToken cancellationToken)
Create a Task that completes after a given timeSpan .
EventType
Types of events. Mirror in tgs.dm. Prefer last listed name for script.
Definition EventType.cs:7