PosixProcessFeatures.cs

Go to the documentation of this file.

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
 
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Mono.Unix;
using Mono.Unix.Native;
 
using Tgstation.Server.Api.Models;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.Jobs;
 
namespace Tgstation.Server.Host.System
{
    sealed class PosixProcessFeatures : IProcessFeatures, IHostedService
    {
        const short SelfOomAdjust = 1;
 
        const short ChildProcessOomAdjust = SelfOomAdjust + 1;
 
        readonly Lazy<IProcessExecutor> lazyLoadedProcessExecutor;
 
        readonly IIOManager ioManager;
 
        readonly ILogger<PosixProcessFeatures> logger;
 
        short baselineOomAdjust;
 
        public PosixProcessFeatures(Lazy<IProcessExecutor> lazyLoadedProcessExecutor, IIOManager ioManager, ILogger<PosixProcessFeatures> logger)
        {
            this.lazyLoadedProcessExecutor = lazyLoadedProcessExecutor ?? throw new ArgumentNullException(nameof(lazyLoadedProcessExecutor));
            this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
            this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
        }
 
        public void ResumeProcess(global::System.Diagnostics.Process process)
        {
            var result = Syscall.kill(process.Id, Signum.SIGCONT);
            if (result != 0)
                throw new UnixIOException(Stdlib.GetLastError());
        }
 
        public void SuspendProcess(global::System.Diagnostics.Process process)
        {
            var result = Syscall.kill(process.Id, Signum.SIGSTOP);
            if (result != 0)
                throw new UnixIOException(Stdlib.GetLastError());
        }
 
        public string GetExecutingUsername(global::System.Diagnostics.Process process)
            => throw new NotSupportedException();
 
        public async ValueTask CreateDump(global::System.Diagnostics.Process process, string outputFile, bool minidump, CancellationToken cancellationToken)
        {
            ArgumentNullException.ThrowIfNull(process);
            ArgumentNullException.ThrowIfNull(outputFile);
 
            string? gcorePath = null;
            foreach (var path in GetPotentialGCorePaths())
                if (await ioManager.FileExists(path, cancellationToken))
                {
                    gcorePath = path;
                    break;
                }
 
            if (gcorePath == null)
                throw new JobException(ErrorCode.MissingGCore);
 
            int pid;
            try
            {
                if (process.HasExited)
                    throw new JobException(ErrorCode.GameServerOffline);
 
                pid = process.Id;
            }
            catch (InvalidOperationException ex)
            {
                throw new JobException(ErrorCode.GameServerOffline, ex);
            }
 
            string? output;
            int exitCode;
            await using (var gcoreProc = await lazyLoadedProcessExecutor.Value.LaunchProcess(
                gcorePath,
                Environment.CurrentDirectory,
                $"{(!minidump ? "-a " : String.Empty)}-o {outputFile} {process.Id}",
                cancellationToken,
                readStandardHandles: true,
                noShellExecute: true))
            {
                using (cancellationToken.Register(() => gcoreProc.Terminate()))
                    exitCode = (await gcoreProc.Lifetime).Value;
 
                output = await gcoreProc.GetCombinedOutput(cancellationToken);
                logger.LogDebug("gcore output:{newline}{output}", Environment.NewLine, output);
            }
 
            if (exitCode != 0)
                throw new JobException(
                    ErrorCode.GCoreFailure,
                    new JobException(
                        $"Exit Code: {exitCode}{Environment.NewLine}Output:{Environment.NewLine}{output}"));
 
            // gcore outputs name.pid so remove the pid part
            var generatedGCoreFile = $"{outputFile}.{pid}";
            await ioManager.MoveFile(generatedGCoreFile, outputFile, cancellationToken);
        }
 
        public async ValueTask<int> HandleProcessStart(global::System.Diagnostics.Process process, CancellationToken cancellationToken)
        {
            ArgumentNullException.ThrowIfNull(process);
            var pid = process.Id;
            try
            {
                // make sure all processes we spawn are killed _before_ us
                await AdjustOutOfMemoryScore(pid, ChildProcessOomAdjust, cancellationToken);
            }
            catch (Exception ex) when (ex is not OperationCanceledException)
            {
                logger.LogWarning(ex, "Failed to adjust OOM killer score for pid {pid}!", pid);
            }
 
            return pid;
        }
 
        public async Task StartAsync(CancellationToken cancellationToken)
        {
            // let this all throw
            string originalString;
            {
                // can't use ReadAllBytes here, /proc files have 0 length so the buffer is initialized to empty
                // https://stackoverflow.com/questions/12237712/how-can-i-show-the-size-of-files-in-proc-it-should-not-be-size-zero
                await using var fileStream = ioManager.CreateAsyncReadStream(
                    "/proc/self/oom_score_adj", true, true);
                using var reader = new StreamReader(fileStream, Encoding.UTF8, leaveOpen: true);
                originalString = await reader.ReadToEndAsync(cancellationToken);
            }
 
            var trimmedString = originalString.Trim();
 
            logger.LogTrace("Original oom_score_adj is \"{original}\"", trimmedString);
 
            var originalOomAdjust = Int16.Parse(trimmedString, CultureInfo.InvariantCulture);
            baselineOomAdjust = Math.Clamp(originalOomAdjust, (short)-1000, (short)1000);
 
            if (baselineOomAdjust == 1000)
                if (originalOomAdjust != baselineOomAdjust)
                    logger.LogWarning("oom_score_adj is at it's limit of 1000 (Clamped from {original}). TGS cannot guarantee the kill order of its parent/child processes!", originalOomAdjust);
                else
                    logger.LogWarning("oom_score_adj is at it's limit of 1000. TGS cannot guarantee the kill order of its parent/child processes!");
 
            try
            {
                // we do not want to be killed before the host watchdog
                await AdjustOutOfMemoryScore(null, SelfOomAdjust, cancellationToken);
            }
            catch (Exception ex) when (ex is not OperationCanceledException)
            {
                logger.LogWarning(ex, "Could not increase oom_score_adj!");
            }
        }
 
        public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
 
        ValueTask AdjustOutOfMemoryScore(int? pid, short adjustment, CancellationToken cancellationToken)
        {
            var adjustedValue = Math.Clamp(baselineOomAdjust + adjustment, -1000, 1000);
 
            var pidStr = pid.HasValue
                ? pid.Value.ToString(CultureInfo.InvariantCulture)
                : "self";
            logger.LogTrace(
                "Setting oom_score_adj of {pid} to {adjustment}...", pidStr, adjustedValue);
            return ioManager.WriteAllBytes(
                $"/proc/{pidStr}/oom_score_adj",
                Encoding.UTF8.GetBytes(adjustedValue.ToString(CultureInfo.InvariantCulture)),
                cancellationToken);
        }
 
        IEnumerable<string> GetPotentialGCorePaths()
        {
            var enviromentPath = Environment.GetEnvironmentVariable("PATH");
            IEnumerable<string> enumerator;
            if (enviromentPath == null)
                enumerator = Enumerable.Empty<string>();
            else
            {
                var paths = enviromentPath.Split(';');
                enumerator = paths
                    .Select(x => x.Split(':'))
                    .SelectMany(x => x);
            }
 
            var exeName = "gcore";
 
            enumerator = enumerator
                .Concat(new List<string>(2)
                {
                    "/usr/bin",
                    "/usr/share/bin",
                    "/bin",
                });
 
            enumerator = enumerator.Select(x => ioManager.ConcatPath(x, exeName));
 
            return enumerator;
        }
    }
}