Merge pull request #2377 from KazWolfe/assert-blame

feat: Identify the plugin causing an assertion failure

.github/workflows/update-submodules.yml

@@ -41,6 +41,7 @@ jobs:
         git config --global user.email noreply@github.com
         git config --global pull.rebase false
     - name: Update submodule
+      id: update-submodule
       run: |
         git checkout -b ${{ matrix.submodule.branch-prefix }}-${{ matrix.branches }}
         git reset --hard origin/${{ matrix.branches }}
@@ -49,9 +50,19 @@ jobs:
         git reset --hard origin/${{ matrix.submodule.branch }}
         cd ../..
         git add ${{ matrix.submodule.path }}
+
+        if [[ -z "$(git status --porcelain --untracked-files=no)" ]]; then   
+          echo "No changes detected!"
+          echo "SUBMIT_PR=false" >> "$GITHUB_OUTPUT"
+          exit 0
+        fi
+        
         git commit --message "Update ${{ matrix.submodule.name }}"
         git push origin ${{ matrix.submodule.branch-prefix }}-${{ matrix.branches }} --force
+        echo "SUBMIT_PR=true" >> "$GITHUB_OUTPUT"
+        
     - name: Create PR
+      if: ${{ steps.update-submodule.outputs.SUBMIT_PR == 'true' }}
       run: |
         echo ${{ secrets.UPDATE_PAT }} | gh auth login --with-token
         prNumber=$(gh pr list --base ${{ matrix.branches }} --head ${{ matrix.submodule.branch-prefix }}-${{ matrix.branches }} --state open --json number --template "{{range .}}{{.number}}{{end}}")

Dalamud/Interface/Internal/Asserts/AssertHandler.cs

@@ -1,9 +1,11 @@
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Windows.Forms;
 
+using Dalamud.Plugin.Internal;
 using Dalamud.Utility;
 
 using Serilog;
@@ -55,7 +57,8 @@ internal class AssertHandler : IDisposable
     /// </summary>
     public unsafe void Setup()
     {
-        CustomNativeFunctions.igCustom_SetAssertCallback(Marshal.GetFunctionPointerForDelegate(this.callback).ToPointer());
+        CustomNativeFunctions.igCustom_SetAssertCallback(
+            Marshal.GetFunctionPointerForDelegate(this.callback).ToPointer());
     }
 
     /// <summary>
@@ -72,16 +75,53 @@ internal class AssertHandler : IDisposable
         this.Shutdown();
     }
 
+    private static string? ExtractImguiFunction(StackTrace stackTrace)
+    {
+        var frame = stackTrace.GetFrames()
+                              .FirstOrDefault(f => f.GetMethod()?.DeclaringType?.Namespace == "Dalamud.Bindings.ImGui");
+        if (frame == null)
+            return null;
+
+        var method = frame.GetMethod();
+        if (method == null)
+            return null;
+
+        return $"{method.Name}({string.Join(", ", method.GetParameters().Select(p => p.Name))})";
+    }
+
+    private static StackTrace GenerateStackTrace()
+    {
+        var trace = DiagnosticUtil.GetUsefulTrace(new StackTrace(true));
+        var frames = trace.GetFrames().ToList();
+
+        // Remove everything that happens in the assert context.
+        var lastAssertIdx = frames.FindLastIndex(f => f.GetMethod()?.DeclaringType == typeof(AssertHandler));
+        if (lastAssertIdx >= 0)
+        {
+            frames.RemoveRange(0, lastAssertIdx + 1);
+        }
+
+        var firstInterfaceManagerIdx = frames.FindIndex(f => f.GetMethod()?.DeclaringType == typeof(InterfaceManager));
+        if (firstInterfaceManagerIdx >= 0)
+        {
+            frames.RemoveRange(firstInterfaceManagerIdx, frames.Count - firstInterfaceManagerIdx);
+        }
+
+        return new StackTrace(frames);
+    }
+
+
     private unsafe void OnImGuiAssert(void* pExpr, void* pFile, int line)
     {
         var expr = Marshal.PtrToStringAnsi(new IntPtr(pExpr));
         var file = Marshal.PtrToStringAnsi(new IntPtr(pFile));
         if (expr == null || file == null)
         {
-            Log.Warning("ImGui assertion failed: {Expr} at {File}:{Line} (failed to parse)",
-                        expr,
-                        file,
-                        line);
+            Log.Warning(
+                "ImGui assertion failed: {Expr} at {File}:{Line} (failed to parse)",
+                expr,
+                file,
+                line);
             return;
         }
 
@@ -93,7 +133,7 @@ internal class AssertHandler : IDisposable
         if (!this.ShowAsserts && !this.everShownAssertThisSession)
             return;
 
-        Lazy<string> stackTrace = new(() => DiagnosticUtil.GetUsefulTrace(new StackTrace()).ToString());
+        Lazy<StackTrace> stackTrace = new(GenerateStackTrace);
 
         if (!this.EnableVerboseLogging)
         {
@@ -103,11 +143,12 @@ internal class AssertHandler : IDisposable
 
                 if (count <= HideThreshold || count % HidePrintEvery == 0)
                 {
-                    Log.Warning("ImGui assertion failed: {Expr} at {File}:{Line} (repeated {Count} times)",
-                                expr,
-                                file,
-                                line,
-                                count);
+                    Log.Warning(
+                        "ImGui assertion failed: {Expr} at {File}:{Line} (repeated {Count} times)",
+                        expr,
+                        file,
+                        line,
+                        count);
                 }
             }
             else
@@ -117,11 +158,12 @@ internal class AssertHandler : IDisposable
         }
         else
         {
-            Log.Warning("ImGui assertion failed: {Expr} at {File}:{Line}\n{StackTrace:l}",
-                        expr,
-                        file,
-                        line,
-                        stackTrace.Value);
+            Log.Warning(
+                "ImGui assertion failed: {Expr} at {File}:{Line}\n{StackTrace:l}",
+                expr,
+                file,
+                line,
+                stackTrace.Value.ToString());
         }
 
         if (!this.ShowAsserts)
@@ -145,7 +187,8 @@ internal class AssertHandler : IDisposable
         }
 
         // grab the stack trace now that we've decided to show UI.
-        _ = stackTrace.Value;
+        var responsiblePlugin = Service<PluginManager>.GetNullable()?.FindCallingPlugin(stackTrace.Value);
+        var responsibleMethodCall = ExtractImguiFunction(stackTrace.Value);
 
         var gitHubUrl = GetRepoUrl();
         var showOnGitHubButton = new TaskDialogButton
@@ -175,12 +218,37 @@ internal class AssertHandler : IDisposable
         var ignoreButton = TaskDialogButton.Ignore;
 
         TaskDialogButton? result = null;
+
         void DialogThreadStart()
         {
             // TODO(goat): This is probably not gonna work if we showed the loading dialog
             // this session since it already loaded visual styles...
             Application.EnableVisualStyles();
 
+            string text;
+            if (responsiblePlugin != null)
+            {
+                text = $"The plugin \"{responsiblePlugin.Name}\" appears to have caused an ImGui assertion failure. " +
+                       $"Please report this problem to the plugin's developer.\n\n";
+            }
+            else
+            {
+                text = "Some code in a plugin or Dalamud itself has caused an ImGui assertion failure. " +
+                       "Please report this problem in the Dalamud discord.\n\n";
+            }
+
+            text += $"You may attempt to continue running the game, but Dalamud UI elements may not work " +
+                    $"correctly, or the game may crash after resuming.\n\n";
+
+            if (responsibleMethodCall != null)
+            {
+                text += $"Assertion failed: {expr} when performing {responsibleMethodCall}\n{file}:{line}";
+            }
+            else
+            {
+                text += $"Assertion failed: {expr}\nAt: {file}:{line}";
+            }
+
             var page = new TaskDialogPage
             {
                 Heading = "ImGui assertion failed",
@@ -189,9 +257,9 @@ internal class AssertHandler : IDisposable
                 {
                     CollapsedButtonText = "Show stack trace",
                     ExpandedButtonText = "Hide stack trace",
-                    Text = stackTrace.Value,
+                    Text = stackTrace.Value.ToString(),
                 },
-                Text = $"Some code in a plugin or Dalamud itself has caused an internal assertion in ImGui to fail. The game will most likely crash now.\n\n{expr}\nAt: {file}:{line}",
+                Text = text,
                 Icon = TaskDialogIcon.Warning,
                 Buttons =
                 [

Dalamud/Interface/Internal/PluginCategoryManager.cs

@@ -294,7 +294,7 @@ internal class PluginCategoryManager
                 }
             }
 
-            if (PluginManager.HasTestingVersion(manifest) || manifest.IsTestingExclusive)
+            if (manifest.IsTestingExclusive || manifest.IsAvailableForTesting)
                 categoryList.Add(CategoryKind.AvailableForTesting);
 
             // always add, even if empty

Dalamud/Interface/Internal/Windows/PluginInstaller/PluginInstallerWindow.cs

@@ -2454,10 +2454,11 @@ internal class PluginInstallerWindow : Window, IDisposable
         var configuration = Service<DalamudConfiguration>.Get();
         var pluginManager = Service<PluginManager>.Get();
 
+        var canUseTesting = pluginManager.CanUseTesting(manifest);
         var useTesting = pluginManager.UseTesting(manifest);
         var wasSeen = this.WasPluginSeen(manifest.InternalName);
 
-        var effectiveApiLevel = useTesting && manifest.TestingDalamudApiLevel != null ? manifest.TestingDalamudApiLevel.Value : manifest.DalamudApiLevel;
+        var effectiveApiLevel = useTesting ? manifest.TestingDalamudApiLevel.Value : manifest.DalamudApiLevel;
         var isOutdated = effectiveApiLevel < PluginManager.DalamudApiLevel;
 
         var isIncompatible = manifest.MinimumDalamudVersion != null &&
@@ -2487,7 +2488,7 @@ internal class PluginInstallerWindow : Window, IDisposable
         {
             label += Locs.PluginTitleMod_TestingExclusive;
         }
-        else if (configuration.DoPluginTest && PluginManager.HasTestingVersion(manifest))
+        else if (canUseTesting)
         {
             label += Locs.PluginTitleMod_TestingAvailable;
         }
@@ -2593,8 +2594,7 @@ internal class PluginInstallerWindow : Window, IDisposable
         var configuration = Service<DalamudConfiguration>.Get();
         var pluginManager = Service<PluginManager>.Get();
 
-        var hasTestingVersionAvailable = configuration.DoPluginTest &&
-                                         PluginManager.HasTestingVersion(manifest);
+        var hasTestingVersionAvailable = configuration.DoPluginTest && manifest.IsAvailableForTesting;
 
         if (ImGui.BeginPopupContextItem("ItemContextMenu"u8))
         {
@@ -2689,8 +2689,7 @@ internal class PluginInstallerWindow : Window, IDisposable
             label += Locs.PluginTitleMod_TestingVersion;
         }
 
-        var hasTestingAvailable = this.pluginListAvailable.Any(x => x.InternalName == plugin.InternalName &&
-                                                                               x.IsAvailableForTesting);
+        var hasTestingAvailable = this.pluginListAvailable.Any(x => x.InternalName == plugin.InternalName && x.IsAvailableForTesting);
         if (hasTestingAvailable && configuration.DoPluginTest && testingOptIn == null)
         {
             label += Locs.PluginTitleMod_TestingAvailable;
@@ -3784,16 +3783,7 @@ internal class PluginInstallerWindow : Window, IDisposable
 
     private bool IsManifestFiltered(IPluginManifest manifest)
     {
-        var hasSearchString = !string.IsNullOrWhiteSpace(this.searchText);
-        var oldApi = (manifest.TestingDalamudApiLevel == null
-                            || manifest.TestingDalamudApiLevel < PluginManager.DalamudApiLevel)
-                          && manifest.DalamudApiLevel < PluginManager.DalamudApiLevel;
-        var installed = this.IsManifestInstalled(manifest).IsInstalled;
-
-        if (oldApi && !hasSearchString && !installed)
-            return true;
-
-        if (!hasSearchString)
+        if (string.IsNullOrWhiteSpace(this.searchText))
             return false;
 
         return this.GetManifestSearchScore(manifest) < 1;

Dalamud/Interface/Internal/Windows/SelfTest/SelfTestWindow.cs

@@ -6,11 +6,13 @@ using System.Numerics;
 using Dalamud.Bindings.ImGui;
 using Dalamud.Interface.Colors;
 using Dalamud.Interface.Components;
-using Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 using Dalamud.Interface.Utility;
 using Dalamud.Interface.Utility.Raii;
 using Dalamud.Interface.Windowing;
 using Dalamud.Logging.Internal;
+using Dalamud.Plugin.SelfTest;
+using Dalamud.Plugin.SelfTest.Internal;
+using Dalamud.Utility;
 using Lumina.Excel.Sheets;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest;
@@ -22,49 +24,14 @@ internal class SelfTestWindow : Window
 {
     private static readonly ModuleLog Log = new("AGING");
 
-    private readonly List<ISelfTestStep> steps =
-    [
-        new LoginEventSelfTestStep(),
-        new WaitFramesSelfTestStep(1000),
-        new FrameworkTaskSchedulerSelfTestStep(),
-        new EnterTerritorySelfTestStep(148, "Central Shroud"),
-        new ItemPayloadSelfTestStep(),
-        new ContextMenuSelfTestStep(),
-        new NamePlateSelfTestStep(),
-        new ActorTableSelfTestStep(),
-        new FateTableSelfTestStep(),
-        new AetheryteListSelfTestStep(),
-        new ConditionSelfTestStep(),
-        new ToastSelfTestStep(),
-        new TargetSelfTestStep(),
-        new KeyStateSelfTestStep(),
-        new GamepadStateSelfTestStep(),
-        new ChatSelfTestStep(),
-        new HoverSelfTestStep(),
-        new LuminaSelfTestStep<Item>(true),
-        new LuminaSelfTestStep<Level>(true),
-        new LuminaSelfTestStep<Lumina.Excel.Sheets.Action>(true),
-        new LuminaSelfTestStep<Quest>(true),
-        new LuminaSelfTestStep<TerritoryType>(false),
-        new AddonLifecycleSelfTestStep(),
-        new PartyFinderSelfTestStep(),
-        new HandledExceptionSelfTestStep(),
-        new DutyStateSelfTestStep(),
-        new GameConfigSelfTestStep(),
-        new MarketBoardSelfTestStep(),
-        new SheetRedirectResolverSelfTestStep(),
-        new NounProcessorSelfTestStep(),
-        new SeStringEvaluatorSelfTestStep(),
-        new CompletionSelfTestStep(),
-        new LogoutEventSelfTestStep()
-    ];
+    private readonly SelfTestRegistry selfTestRegistry;
 
-    private readonly Dictionary<int, (SelfTestStepResult Result, TimeSpan? Duration)> testIndexToResult = new();
+    private List<SelfTestWithResults> visibleSteps = new();
 
     private bool selfTestRunning = false;
-    private int currentStep = 0;
-    private int scrollToStep = -1;
-    private DateTimeOffset lastTestStart;
+    private SelfTestGroup? currentTestGroup = null;
+    private SelfTestWithResults? currentStep = null;
+    private SelfTestWithResults? scrollToStep = null;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="SelfTestWindow"/> class.
@@ -72,6 +39,7 @@ internal class SelfTestWindow : Window
     public SelfTestWindow()
         : base("Dalamud Self-Test", ImGuiWindowFlags.NoScrollbar | ImGuiWindowFlags.NoScrollWithMouse)
     {
+        this.selfTestRegistry = Service<SelfTestRegistry>.Get();
         this.Size = new Vector2(800, 800);
         this.SizeCondition = ImGuiCond.FirstUseEver;
 
@@ -81,6 +49,55 @@ internal class SelfTestWindow : Window
     /// <inheritdoc/>
     public override void Draw()
     {
+        // Initialize to first group if not set (first time drawing)
+        if (this.currentTestGroup == null)
+        {
+            this.currentTestGroup = this.selfTestRegistry.SelfTestGroups.FirstOrDefault(); // Should always be "Dalamud"
+            if (this.currentTestGroup != null)
+            {
+                this.SelectTestGroup(this.currentTestGroup);
+            }
+        }
+
+        // Update visible steps based on current group
+        if (this.currentTestGroup != null)
+        {
+            this.visibleSteps = this.selfTestRegistry.SelfTests
+                .Where(test => test.Group == this.currentTestGroup.Name).ToList();
+
+            // Stop tests if no steps available or if current step is no longer valid
+            if (this.visibleSteps.Count == 0 || (this.currentStep != null && !this.visibleSteps.Contains(this.currentStep)))
+            {
+                this.StopTests();
+            }
+        }
+
+        using (var dropdown = ImRaii.Combo("###SelfTestGroupCombo"u8, this.currentTestGroup?.Name ?? string.Empty))
+        {
+            if (dropdown)
+            {
+                foreach (var testGroup in this.selfTestRegistry.SelfTestGroups)
+                {
+                    if (ImGui.Selectable(testGroup.Name))
+                    {
+                        this.SelectTestGroup(testGroup);
+                    }
+
+                    if (!testGroup.Loaded)
+                    {
+                        ImGui.SameLine();
+                        this.DrawUnloadedIcon();
+                    }
+                }
+            }
+        }
+
+        if (this.currentTestGroup?.Loaded == false)
+        {
+            ImGui.SameLine();
+            this.DrawUnloadedIcon();
+        }
+
         if (this.selfTestRunning)
         {
             if (ImGuiComponents.IconButton(FontAwesomeIcon.Stop))
@@ -92,13 +109,10 @@ internal class SelfTestWindow : Window
 
             if (ImGuiComponents.IconButton(FontAwesomeIcon.StepForward))
             {
-                this.testIndexToResult[this.currentStep] = (SelfTestStepResult.NotRan, null);
-                this.steps[this.currentStep].CleanUp();
-                this.currentStep++;
+                this.currentStep.Reset();
+                this.MoveToNextTest();
                 this.scrollToStep = this.currentStep;
-                this.lastTestStart = DateTimeOffset.Now;
-
-                if (this.currentStep >= this.steps.Count)
+                if (this.currentStep == null)
                 {
                     this.StopTests();
                 }
@@ -106,38 +120,50 @@ internal class SelfTestWindow : Window
         }
         else
         {
+            var canRunTests = this.currentTestGroup?.Loaded == true && this.visibleSteps.Count > 0;
+
+            using var disabled = ImRaii.Disabled(!canRunTests);
             if (ImGuiComponents.IconButton(FontAwesomeIcon.Play))
             {
                 this.selfTestRunning = true;
-                this.currentStep = 0;
+                this.currentStep = this.visibleSteps.FirstOrDefault();
                 this.scrollToStep = this.currentStep;
-                this.testIndexToResult.Clear();
-                this.lastTestStart = DateTimeOffset.Now;
+                foreach (var test in this.visibleSteps)
+                {
+                    test.Reset();
+                }
             }
         }
 
         ImGui.SameLine();
 
-        ImGui.Text($"Step: {this.currentStep} / {this.steps.Count}");
+        var stepNumber = this.currentStep != null ? this.visibleSteps.IndexOf(this.currentStep) : 0;
+        ImGui.Text($"Step: {stepNumber} / {this.visibleSteps.Count}"); 
 
         ImGui.Spacing();
 
+        if (this.currentTestGroup?.Loaded == false)
+        {
+            ImGui.TextColoredWrapped(ImGuiColors.DalamudGrey, $"Plugin '{this.currentTestGroup.Name}' is unloaded. No tests available.");
+            ImGui.Spacing();
+        }
+
         this.DrawResultTable();
 
         ImGui.Spacing();
 
-        if (this.currentStep >= this.steps.Count)
+        if (this.currentStep == null)
         {
             if (this.selfTestRunning)
             {
                 this.StopTests();
             }
 
-            if (this.testIndexToResult.Any(x => x.Value.Result == SelfTestStepResult.Fail))
+            if (this.visibleSteps.Any(test => test.Result == SelfTestStepResult.Fail))
             {
                 ImGui.TextColoredWrapped(ImGuiColors.DalamudRed, "One or more checks failed!"u8);
             }
-            else
+            else if (this.visibleSteps.All(test => test.Result == SelfTestStepResult.Pass))
             {
                 ImGui.TextColoredWrapped(ImGuiColors.HealerGreen, "All checks passed!"u8);
             }
@@ -153,30 +179,14 @@ internal class SelfTestWindow : Window
         using var resultChild = ImRaii.Child("SelfTestResultChild"u8, ImGui.GetContentRegionAvail());
         if (!resultChild) return;
 
-        var step = this.steps[this.currentStep];
-        ImGui.Text($"Current: {step.Name}");
+        ImGui.Text($"Current: {this.currentStep.Name}");
 
         ImGuiHelpers.ScaledDummy(10);
 
-        SelfTestStepResult result;
-        try
+        this.currentStep.DrawAndStep();
+        if (this.currentStep.Result != SelfTestStepResult.Waiting)
         {
-            result = step.RunStep();
-        }
-        catch (Exception ex)
-        {
-            Log.Error(ex, $"Step failed: {step.Name}");
-            result = SelfTestStepResult.Fail;
-        }
-
-        if (result != SelfTestStepResult.Waiting)
-        {
-            var duration = DateTimeOffset.Now - this.lastTestStart;
-            this.testIndexToResult[this.currentStep] = (result, duration);
-            this.currentStep++;
-            this.scrollToStep = this.currentStep;
-
-            this.lastTestStart = DateTimeOffset.Now;
+            this.MoveToNextTest();
         }
     }
 
@@ -202,85 +212,62 @@ internal class SelfTestWindow : Window
         ImGui.TableSetupScrollFreeze(0, 1);
         ImGui.TableHeadersRow();
 
-        for (var i = 0; i < this.steps.Count; i++)
+        foreach (var (step, index) in this.visibleSteps.WithIndex())
         {
-            var step = this.steps[i];
             ImGui.TableNextRow();
 
-            if (this.selfTestRunning && this.currentStep == i)
+            if (this.selfTestRunning && this.currentStep == step)
             {
                 ImGui.TableSetBgColor(ImGuiTableBgTarget.RowBg0, ImGui.GetColorU32(ImGuiCol.TableRowBgAlt));
             }
 
             ImGui.TableSetColumnIndex(0);
             ImGui.AlignTextToFramePadding();
-            ImGui.Text(i.ToString());
+            ImGui.Text(index.ToString());
 
-            if (this.selfTestRunning && this.scrollToStep == i)
+            if (this.selfTestRunning && this.scrollToStep == step)
             {
                 ImGui.SetScrollHereY();
-                this.scrollToStep = -1;
+                this.scrollToStep = null;
             }
 
             ImGui.TableSetColumnIndex(1);
             ImGui.AlignTextToFramePadding();
             ImGui.Text(step.Name);
 
-            if (this.testIndexToResult.TryGetValue(i, out var result))
+            ImGui.TableSetColumnIndex(2);
+            ImGui.AlignTextToFramePadding();
+            switch (step.Result)
             {
-                ImGui.TableSetColumnIndex(2);
-                ImGui.AlignTextToFramePadding();
-
-                switch (result.Result)
-                {
-                    case SelfTestStepResult.Pass:
-                        ImGui.TextColored(ImGuiColors.HealerGreen, "PASS"u8);
-                        break;
-                    case SelfTestStepResult.Fail:
-                        ImGui.TextColored(ImGuiColors.DalamudRed, "FAIL"u8);
-                        break;
-                    default:
-                        ImGui.TextColored(ImGuiColors.DalamudGrey, "NR"u8);
-                        break;
-                }
-
-                ImGui.TableSetColumnIndex(3);
-                if (result.Duration.HasValue)
-                {
-                    ImGui.AlignTextToFramePadding();
-                    ImGui.Text(this.FormatTimeSpan(result.Duration.Value));
-                }
-            }
-            else
-            {
-                ImGui.TableSetColumnIndex(2);
-                ImGui.AlignTextToFramePadding();
-                if (this.selfTestRunning && this.currentStep == i)
-                {
+                case SelfTestStepResult.Pass:
+                    ImGui.TextColored(ImGuiColors.HealerGreen, "PASS"u8);
+                    break;
+                case SelfTestStepResult.Fail:
+                    ImGui.TextColored(ImGuiColors.DalamudRed, "FAIL"u8);
+                    break;
+                case SelfTestStepResult.Waiting:
                     ImGui.TextColored(ImGuiColors.DalamudGrey, "WAIT"u8);
-                }
-                else
-                {
+                    break;
+                default:
                     ImGui.TextColored(ImGuiColors.DalamudGrey, "NR"u8);
-                }
+                    break;
+            }
 
-                ImGui.TableSetColumnIndex(3);
+            ImGui.TableSetColumnIndex(3);
+            if (step.Duration.HasValue)
+            {
                 ImGui.AlignTextToFramePadding();
-                if (this.selfTestRunning && this.currentStep == i)
-                {
-                    ImGui.Text(this.FormatTimeSpan(DateTimeOffset.Now - this.lastTestStart));
-                }
+                ImGui.Text(this.FormatTimeSpan(step.Duration.Value));
             }
 
             ImGui.TableSetColumnIndex(4);
-            using var id = ImRaii.PushId($"selfTest{i}");
+            using var id = ImRaii.PushId($"selfTest{index}");
             if (ImGuiComponents.IconButton(FontAwesomeIcon.FastForward))
             {
                 this.StopTests();
-                this.testIndexToResult.Remove(i);
-                this.currentStep = i;
+                this.currentStep = step;
+                this.currentStep.Reset();
                 this.selfTestRunning = true;
-                this.lastTestStart = DateTimeOffset.Now;
             }
 
             if (ImGui.IsItemHovered())
@@ -293,12 +280,14 @@ internal class SelfTestWindow : Window
     private void StopTests()
     {
         this.selfTestRunning = false;
+        this.currentStep = null;
+        this.scrollToStep = null;
 
-        foreach (var agingStep in this.steps)
+        foreach (var agingStep in this.visibleSteps)
         {
             try
             {
-                agingStep.CleanUp();
+                agingStep.Finish();
             }
             catch (Exception ex)
             {
@@ -307,6 +296,46 @@ internal class SelfTestWindow : Window
         }
     }
 
+    /// <summary>
+    /// Makes <paramref name="testGroup"/> the active test group.
+    /// </summary>
+    /// <param name="testGroup">The test group to make active.</param>
+    private void SelectTestGroup(SelfTestGroup testGroup)
+    {
+        this.currentTestGroup = testGroup;
+        this.StopTests();
+    }
+
+    /// <summary>
+    /// Move `currentTest` to the next test. If there are no tests left, set `currentTest` to null.
+    /// </summary>
+    private void MoveToNextTest()
+    {
+        if (this.currentStep == null)
+        {
+            this.currentStep = this.visibleSteps.FirstOrDefault();
+            return;
+        }
+
+        var currentIndex = this.visibleSteps.IndexOf(this.currentStep);
+        this.currentStep = this.visibleSteps.ElementAtOrDefault(currentIndex + 1);
+    }
+
+    /// <summary>
+    /// Draws the unloaded plugin icon with tooltip.
+    /// </summary>
+    private void DrawUnloadedIcon()
+    {
+        ImGui.PushFont(UiBuilder.IconFont);
+        ImGui.TextColored(ImGuiColors.DalamudGrey, FontAwesomeIcon.Unlink.ToIconString());
+        ImGui.PopFont();
+
+        if (ImGui.IsItemHovered())
+        {
+            ImGui.SetTooltip("Plugin is unloaded");
+        }
+    }
+
     private string FormatTimeSpan(TimeSpan ts)
     {
         var str = ts.ToString("g", CultureInfo.InvariantCulture);

Dalamud/Interface/Internal/Windows/SelfTest/Steps/ActorTableSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.ClientState.Objects;
+using Dalamud.Plugin.SelfTest;
 using Dalamud.Utility;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;

Dalamud/Interface/Internal/Windows/SelfTest/Steps/AddonLifecycleSelfTestStep.cs

@@ -3,6 +3,7 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.Addon.Lifecycle;
 using Dalamud.Game.Addon.Lifecycle.AddonArgTypes;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/AetheryteListSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.ClientState.Aetherytes;
+using Dalamud.Plugin.SelfTest;
 using Dalamud.Utility;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;

Dalamud/Interface/Internal/Windows/SelfTest/Steps/ChatSelfTestStep.cs

@@ -2,6 +2,7 @@ using Dalamud.Bindings.ImGui;
 using Dalamud.Game.Gui;
 using Dalamud.Game.Text;
 using Dalamud.Game.Text.SeStringHandling;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/CompletionSelfTestStep.cs

@@ -1,6 +1,7 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.Command;
 using Dalamud.Interface.Utility;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/ConditionSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.ClientState.Conditions;
+using Dalamud.Plugin.SelfTest;
 using Serilog;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;

Dalamud/Interface/Internal/Windows/SelfTest/Steps/ContextMenuSelfTestStep.cs

@@ -8,6 +8,7 @@ using Dalamud.Game.ClientState.Objects.SubKinds;
 using Dalamud.Game.Gui.ContextMenu;
 using Dalamud.Game.Text;
 using Dalamud.Game.Text.SeStringHandling;
+using Dalamud.Plugin.SelfTest;
 using Lumina.Excel;
 using Lumina.Excel.Sheets;
 using Serilog;

Dalamud/Interface/Internal/Windows/SelfTest/Steps/DalamudSelfTest.cs

@@ -0,0 +1,51 @@
+using Dalamud.Plugin.SelfTest.Internal;
+using Lumina.Excel.Sheets;
+
+namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
+
+/// <summary>
+/// Class handling Dalamud self-test registration.
+/// </summary>
+[ServiceManager.EarlyLoadedService]
+internal class DalamudSelfTest : IServiceType
+{
+    [ServiceManager.ServiceConstructor]
+    private DalamudSelfTest(SelfTestRegistry registry)
+    {
+        registry.RegisterDalamudSelfTestSteps([
+            new LoginEventSelfTestStep(),
+            new WaitFramesSelfTestStep(1000),
+            new FrameworkTaskSchedulerSelfTestStep(),
+            new EnterTerritorySelfTestStep(148, "Central Shroud"),
+            new ItemPayloadSelfTestStep(),
+            new ContextMenuSelfTestStep(),
+            new NamePlateSelfTestStep(),
+            new ActorTableSelfTestStep(),
+            new FateTableSelfTestStep(),
+            new AetheryteListSelfTestStep(),
+            new ConditionSelfTestStep(),
+            new ToastSelfTestStep(),
+            new TargetSelfTestStep(),
+            new KeyStateSelfTestStep(),
+            new GamepadStateSelfTestStep(),
+            new ChatSelfTestStep(),
+            new HoverSelfTestStep(),
+            new LuminaSelfTestStep<Item>(true),
+            new LuminaSelfTestStep<Level>(true),
+            new LuminaSelfTestStep<Lumina.Excel.Sheets.Action>(true),
+            new LuminaSelfTestStep<Quest>(true),
+            new LuminaSelfTestStep<TerritoryType>(false),
+            new AddonLifecycleSelfTestStep(),
+            new PartyFinderSelfTestStep(),
+            new HandledExceptionSelfTestStep(),
+            new DutyStateSelfTestStep(),
+            new GameConfigSelfTestStep(),
+            new MarketBoardSelfTestStep(),
+            new SheetRedirectResolverSelfTestStep(),
+            new NounProcessorSelfTestStep(),
+            new SeStringEvaluatorSelfTestStep(),
+            new CompletionSelfTestStep(),
+            new LogoutEventSelfTestStep()
+        ]);
+    }
+}

Dalamud/Interface/Internal/Windows/SelfTest/Steps/DutyStateSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.DutyState;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/EnterTerritorySelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.ClientState;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/FateTableSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.ClientState.Fates;
+using Dalamud.Plugin.SelfTest;
 using Dalamud.Utility;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;

Dalamud/Interface/Internal/Windows/SelfTest/Steps/FrameworkTaskSchedulerSelfTestStep.cs

@@ -1,6 +1,7 @@
 using System.Threading.Tasks;
 
 using Dalamud.Game;
+using Dalamud.Plugin.SelfTest;
 using Dalamud.Utility;
 
 using Microsoft.VisualBasic.Logging;

Dalamud/Interface/Internal/Windows/SelfTest/Steps/GameConfigSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.Config;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/GamepadStateSelfTestStep.cs

@@ -2,7 +2,7 @@ using System.Linq;
 
 using Dalamud.Game.ClientState.GamePad;
 using Dalamud.Interface.Utility;
-
+using Dalamud.Plugin.SelfTest;
 using Lumina.Text.Payloads;
 
 using LSeStringBuilder = Lumina.Text.SeStringBuilder;

Dalamud/Interface/Internal/Windows/SelfTest/Steps/HandledExceptionSelfTestStep.cs

@@ -1,5 +1,7 @@
 using System.Runtime.InteropServices;
 
+using Dalamud.Plugin.SelfTest;
+
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 
 /// <summary>

Dalamud/Interface/Internal/Windows/SelfTest/Steps/HoverSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.Gui;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/ItemPayloadSelfTestStep.cs

@@ -1,6 +1,7 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.Gui;
 using Dalamud.Game.Text.SeStringHandling;
+using Dalamud.Plugin.SelfTest;
 using Dalamud.Utility;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;

Dalamud/Interface/Internal/Windows/SelfTest/Steps/KeyStateSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.ClientState.Keys;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/LoginEventSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.ClientState;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/LogoutEventSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.ClientState;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/LuminaSelfTestStep.cs

@@ -1,4 +1,5 @@
 using Dalamud.Data;
+using Dalamud.Plugin.SelfTest;
 using Dalamud.Utility;
 using Lumina.Excel;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/MarketBoardSelfTestStep.cs

@@ -5,6 +5,7 @@ using Dalamud.Bindings.ImGui;
 using Dalamud.Game.MarketBoard;
 using Dalamud.Game.Network.Structures;
 using Dalamud.Interface.Utility;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/NamePlateSelfTestStep.cs

@@ -4,6 +4,7 @@ using Dalamud.Bindings.ImGui;
 using Dalamud.Game.Gui.NamePlate;
 using Dalamud.Game.Text.SeStringHandling;
 using Dalamud.Game.Text.SeStringHandling.Payloads;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/NounProcessorSelfTestStep.cs

@@ -2,6 +2,7 @@ using Dalamud.Bindings.ImGui;
 using Dalamud.Game;
 using Dalamud.Game.Text.Noun;
 using Dalamud.Game.Text.Noun.Enums;
+using Dalamud.Plugin.SelfTest;
 
 using LSheets = Lumina.Excel.Sheets;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/PartyFinderSelfTestStep.cs

@@ -1,6 +1,7 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.Gui.PartyFinder;
 using Dalamud.Game.Gui.PartyFinder.Types;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/SeStringEvaluatorSelfTestStep.cs

@@ -3,7 +3,7 @@ using Dalamud.Configuration.Internal;
 using Dalamud.Game.ClientState;
 using Dalamud.Game.Text.Evaluator;
 using Dalamud.Game.Text.SeStringHandling.Payloads;
-
+using Dalamud.Plugin.SelfTest;
 using Lumina.Text.ReadOnly;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;

Dalamud/Interface/Internal/Windows/SelfTest/Steps/SheetRedirectResolverSelfTestStep.cs

@@ -3,6 +3,7 @@ using System.Runtime.InteropServices;
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game;
 using Dalamud.Game.Text.Evaluator.Internal;
+using Dalamud.Plugin.SelfTest;
 using FFXIVClientStructs.FFXIV.Client.System.String;
 using FFXIVClientStructs.FFXIV.Client.UI.Misc;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/TargetSelfTestStep.cs

@@ -2,6 +2,7 @@ using Dalamud.Bindings.ImGui;
 using Dalamud.Game.ClientState.Objects;
 using Dalamud.Game.ClientState.Objects.SubKinds;
 using Dalamud.Game.ClientState.Objects.Types;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/ToastSelfTestStep.cs

@@ -1,5 +1,6 @@
 using Dalamud.Bindings.ImGui;
 using Dalamud.Game.Gui.Toast;
+using Dalamud.Plugin.SelfTest;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 

Dalamud/Interface/Internal/Windows/SelfTest/Steps/WaitFramesSelfTestStep.cs

@@ -1,3 +1,5 @@
+using Dalamud.Plugin.SelfTest;
+
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
 
 /// <summary>

Dalamud/Plugin/DalamudPluginInterface.cs

@@ -16,6 +16,7 @@ using Dalamud.Game.Text.Sanitizer;
 using Dalamud.Interface;
 using Dalamud.Interface.Internal;
 using Dalamud.Interface.Internal.Windows.PluginInstaller;
+using Dalamud.Interface.Internal.Windows.SelfTest;
 using Dalamud.Interface.Internal.Windows.Settings;
 using Dalamud.IoC.Internal;
 using Dalamud.Plugin.Internal;
@@ -25,6 +26,7 @@ using Dalamud.Plugin.Internal.Types.Manifest;
 using Dalamud.Plugin.Ipc;
 using Dalamud.Plugin.Ipc.Exceptions;
 using Dalamud.Plugin.Ipc.Internal;
+using Dalamud.Plugin.Services;
 
 using Serilog;
 

Dalamud/Plugin/IDalamudPluginInterface.cs

@@ -15,6 +15,7 @@ using Dalamud.Plugin.Internal.Types.Manifest;
 using Dalamud.Plugin.Ipc;
 using Dalamud.Plugin.Ipc.Exceptions;
 using Dalamud.Plugin.Ipc.Internal;
+using Dalamud.Plugin.Services;
 
 namespace Dalamud.Plugin;
 

Dalamud/Plugin/Internal/PluginManager.cs

@@ -278,26 +278,6 @@ internal class PluginManager : IInternalDisposableService
         return !manifest.IsHide;
     }
 
-    /// <summary>
-    /// Check if a manifest even has an available testing version.
-    /// </summary>
-    /// <param name="manifest">The manifest to test.</param>
-    /// <returns>Whether a testing version is available.</returns>
-    public static bool HasTestingVersion(IPluginManifest manifest)
-    {
-        var av = manifest.AssemblyVersion;
-        var tv = manifest.TestingAssemblyVersion;
-        var hasTv = tv != null;
-
-        if (hasTv)
-        {
-            return tv > av &&
-                   manifest.TestingDalamudApiLevel == DalamudApiLevel;
-        }
-
-        return false;
-    }
-
     /// <summary>
     /// Get a disposable that will lock plugin lists while it is not disposed.
     /// You must NEVER use this in async code.
@@ -371,6 +351,20 @@ internal class PluginManager : IInternalDisposableService
         return this.configuration.PluginTestingOptIns!.Any(x => x.InternalName == manifest.InternalName);
     }
 
+    /// <summary>
+    /// For a given manifest, determine if the testing version can be used over the normal version.
+    /// The higher of the two versions is calculated after checking other settings.
+    /// </summary>
+    /// <param name="manifest">Manifest to check.</param>
+    /// <returns>A value indicating whether testing can be used.</returns>
+    public bool CanUseTesting(IPluginManifest manifest)
+    {
+        if (!this.configuration.DoPluginTest)
+            return false;
+
+        return manifest.IsTestingExclusive || manifest.IsAvailableForTesting;
+    }
+
     /// <summary>
     /// For a given manifest, determine if the testing version should be used over the normal version.
     /// The higher of the two versions is calculated after checking other settings.
@@ -379,16 +373,7 @@ internal class PluginManager : IInternalDisposableService
     /// <returns>A value indicating whether testing should be used.</returns>
     public bool UseTesting(IPluginManifest manifest)
     {
-        if (!this.configuration.DoPluginTest)
-            return false;
-
-        if (!this.HasTestingOptIn(manifest))
-            return false;
-
-        if (manifest.IsTestingExclusive)
-            return true;
-
-        return HasTestingVersion(manifest);
+        return this.CanUseTesting(manifest) && this.HasTestingOptIn(manifest);
     }
 
     /// <inheritdoc/>
@@ -1208,9 +1193,18 @@ internal class PluginManager : IInternalDisposableService
             return false;
 
         // API level - we keep the API before this in the installer to show as "outdated"
-        var effectiveApiLevel = this.UseTesting(manifest) && manifest.TestingDalamudApiLevel != null ? manifest.TestingDalamudApiLevel.Value : manifest.DalamudApiLevel;
-        if (effectiveApiLevel < DalamudApiLevel - 1 && !this.LoadAllApiLevels)
-            return false;
+        if (!this.LoadAllApiLevels)
+        {
+            var effectiveDalamudApiLevel =
+                this.CanUseTesting(manifest) &&
+                manifest.TestingDalamudApiLevel.HasValue &&
+                manifest.TestingDalamudApiLevel.Value > manifest.DalamudApiLevel
+                    ? manifest.TestingDalamudApiLevel.Value
+                    : manifest.DalamudApiLevel;
+
+            if (effectiveDalamudApiLevel < PluginManager.DalamudApiLevel - 1)
+                return false;
+        }
 
         // Banned
         if (this.IsManifestBanned(manifest))

Dalamud/Plugin/Internal/Types/LocalPlugin.cs

@@ -301,7 +301,7 @@ internal class LocalPlugin : IAsyncDisposable
                 throw new PluginPreconditionFailedException($"Unable to load {this.Name}, game is newer than applicable version {this.manifest.ApplicableVersion}");
 
             // We want to allow loading dev plugins with a lower API level than the current Dalamud API level, for ease of development
-            if (this.manifest.EffectiveApiLevel < PluginManager.DalamudApiLevel && !pluginManager.LoadAllApiLevels && !this.IsDev)
+            if (!pluginManager.LoadAllApiLevels && !this.IsDev && this.manifest.EffectiveApiLevel < PluginManager.DalamudApiLevel)
                 throw new PluginPreconditionFailedException($"Unable to load {this.Name}, incompatible API level {this.manifest.EffectiveApiLevel}");
 
             // We might want to throw here?

Dalamud/Plugin/Internal/Types/Manifest/IPluginManifest.cs

@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 
 namespace Dalamud.Plugin.Internal.Types.Manifest;
 
@@ -118,4 +118,9 @@ public interface IPluginManifest
     /// Gets an URL for the plugin's icon.
     /// </summary>
     public string? IconUrl { get; }
+
+    /// <summary>
+    /// Gets a value indicating whether this plugin is eligible for testing.
+    /// </summary>
+    public bool IsAvailableForTesting { get; }
 }

Dalamud/Plugin/Internal/Types/Manifest/RemotePluginManifest.cs

@@ -21,9 +21,4 @@ internal record RemotePluginManifest : PluginManifest
     /// Gets or sets the changelog to be shown when obtaining the testing version of the plugin.
     /// </summary>
     public string? TestingChangelog { get; set; }
-
-    /// <summary>
-    /// Gets a value indicating whether this plugin is eligible for testing.
-    /// </summary>
-    public bool IsAvailableForTesting => this.TestingAssemblyVersion != null && this.TestingAssemblyVersion > this.AssemblyVersion;
 }

Dalamud/Plugin/Internal/Types/PluginManifest.cs

@@ -160,4 +160,11 @@ internal record PluginManifest : IPluginManifest
     /// <inheritdoc/>
     [JsonProperty("_Dip17Channel")]
     public string? Dip17Channel { get; init; }
+
+    /// <inheritdoc/>
+    [JsonIgnore]
+    public bool IsAvailableForTesting
+        => this.TestingAssemblyVersion != null &&
+           this.TestingAssemblyVersion > this.AssemblyVersion &&
+           this.TestingDalamudApiLevel == PluginManager.DalamudApiLevel;
 }

Dalamud/Plugin/SelfTest/ISelfTestRegistry.cs

@@ -0,0 +1,54 @@
+using System.Collections.Generic;
+
+namespace Dalamud.Plugin.SelfTest;
+
+/// <summary>
+/// Interface for registering and unregistering self-test steps from plugins.
+/// </summary>
+/// <example>
+/// Registering custom self-test steps for your plugin:
+/// <code>
+/// [PluginService]
+/// public ISelfTestRegistry SelfTestRegistry { get; init; }
+///
+/// // In your plugin initialization
+/// this.SelfTestRegistry.RegisterTestSteps([
+///     new MyCustomSelfTestStep(),
+///     new AnotherSelfTestStep()
+/// ]);
+/// </code>
+///
+/// Creating a custom self-test step:
+/// <code>
+/// public class MyCustomSelfTestStep : ISelfTestStep
+/// {
+///     public string Name => "My Custom Test";
+///
+///     public SelfTestStepResult RunStep()
+///     {
+///         // Your test logic here
+///         if (/* test condition passes */)
+///             return SelfTestStepResult.Pass;
+///
+///         if (/* test condition fails */)
+///             return SelfTestStepResult.Fail;
+///
+///         // Still waiting for test to complete
+///         return SelfTestStepResult.Waiting;
+///     }
+///
+///     public void CleanUp()
+///     {
+///         // Clean up any resources used by the test
+///     }
+/// }
+/// </code>
+/// </example>
+public interface ISelfTestRegistry
+{
+    /// <summary>
+    /// Registers the self-test steps for this plugin.
+    /// </summary>
+    /// <param name="steps">The test steps to register.</param>
+    public void RegisterTestSteps(IEnumerable<ISelfTestStep> steps);
+}

Dalamud/Plugin/SelfTest/ISelfTestStep.cs

@@ -1,9 +1,9 @@
-namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
+namespace Dalamud.Plugin.SelfTest;
 
 /// <summary>
 /// Interface for test implementations.
 /// </summary>
-internal interface ISelfTestStep
+public interface ISelfTestStep
 {
     /// <summary>
     /// Gets the name of the test.

Dalamud/Plugin/SelfTest/Internal/SelfTestGroup.cs

@@ -0,0 +1,28 @@
+namespace Dalamud.Plugin.SelfTest.Internal;
+
+/// <summary>
+/// Represents a self-test group with its loaded/unloaded state.
+/// </summary>
+internal class SelfTestGroup
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SelfTestGroup"/> class.
+    /// </summary>
+    /// <param name="name">The name of the test group.</param>
+    /// <param name="loaded">Whether the group is currently loaded.</param>
+    public SelfTestGroup(string name, bool loaded = true)
+    {
+        this.Name = name;
+        this.Loaded = loaded;
+    }
+
+    /// <summary>
+    /// Gets the name of the test group.
+    /// </summary>
+    public string Name { get; }
+
+    /// <summary>
+    /// Gets or sets a value indicating whether this test group is currently loaded.
+    /// </summary>
+    public bool Loaded { get; set; }
+}

Dalamud/Plugin/SelfTest/Internal/SelfTestRegistry.cs

@@ -0,0 +1,124 @@
+using System.Collections.Generic;
+using System.Linq;
+
+using Dalamud.Logging.Internal;
+using Dalamud.Plugin.Internal.Types;
+
+namespace Dalamud.Plugin.SelfTest.Internal;
+
+/// <summary>
+/// Registry for self-tests that can be run in the SelfTest window.
+/// </summary>
+[ServiceManager.EarlyLoadedService]
+internal class SelfTestRegistry : IServiceType
+{
+    /// <summary>
+    /// The name of the Dalamud test group.
+    /// </summary>
+    public const string DalamudTestGroup = "Dalamud";
+
+    private static readonly ModuleLog Log = new("SelfTestRegistry");
+
+    private List<SelfTestWithResults> dalamudSelfTests = new();
+    private List<SelfTestWithResults> pluginSelfTests = new();
+    private Dictionary<string, SelfTestGroup> allGroups = new();
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SelfTestRegistry"/> class.
+    /// </summary>
+    [ServiceManager.ServiceConstructor]
+    public SelfTestRegistry()
+    {
+    }
+
+    /// <summary>
+    /// Gets all available self test groups.
+    /// </summary>
+    public IEnumerable<SelfTestGroup> SelfTestGroups
+    {
+        get
+        {
+            // Always return Dalamud group first, then plugin groups
+            if (this.allGroups.TryGetValue(DalamudTestGroup, out var dalamudGroup))
+            {
+                yield return dalamudGroup;
+            }
+
+            foreach (var group in this.allGroups.Values)
+            {
+                if (group.Name != DalamudTestGroup)
+                {
+                    yield return group;
+                }
+            }
+        }
+    }
+
+    /// <summary>
+    /// Gets all self tests from all groups.
+    /// </summary>
+    public IEnumerable<SelfTestWithResults> SelfTests => this.dalamudSelfTests.Concat(this.pluginSelfTests);
+
+    /// <summary>
+    /// Registers Dalamud self test steps.
+    /// </summary>
+    /// <param name="steps">The steps to register.</param>
+    public void RegisterDalamudSelfTestSteps(IEnumerable<ISelfTestStep> steps)
+    {
+        // Ensure Dalamud group exists and is loaded
+        if (!this.allGroups.ContainsKey(DalamudTestGroup))
+        {
+            this.allGroups[DalamudTestGroup] = new SelfTestGroup(DalamudTestGroup, loaded: true);
+        }
+        else
+        {
+            this.allGroups[DalamudTestGroup].Loaded = true;
+        }
+
+        this.dalamudSelfTests.AddRange(steps.Select(step => SelfTestWithResults.FromDalamudStep(step)));
+    }
+
+    /// <summary>
+    /// Registers plugin self test steps.
+    /// </summary>
+    /// <param name="plugin">The plugin registering the tests.</param>
+    /// <param name="steps">The steps to register.</param>
+    public void RegisterPluginSelfTestSteps(LocalPlugin plugin, IEnumerable<ISelfTestStep> steps)
+    {
+        // Ensure plugin group exists and is loaded
+        if (!this.allGroups.ContainsKey(plugin.InternalName))
+        {
+            this.allGroups[plugin.InternalName] = new SelfTestGroup(plugin.InternalName, loaded: true);
+        }
+        else
+        {
+            this.allGroups[plugin.InternalName].Loaded = true;
+        }
+
+        this.pluginSelfTests.AddRange(steps.Select(step => SelfTestWithResults.FromPluginStep(plugin, step)));
+    }
+
+    /// <summary>
+    /// Unregisters all self test steps for a plugin.
+    /// </summary>
+    /// <param name="plugin">The plugin to unregister tests for.</param>
+    public void UnregisterPluginSelfTestSteps(LocalPlugin plugin)
+    {
+        // Clean up existing tests for this plugin
+        this.pluginSelfTests.ForEach(test =>
+        {
+            if (test.Plugin == plugin)
+            {
+                test.Unload();
+            }
+        });
+
+        this.pluginSelfTests.RemoveAll(test => test.Plugin == plugin);
+
+        // Mark group as unloaded if it exists
+        if (this.allGroups.ContainsKey(plugin.InternalName))
+        {
+            this.allGroups[plugin.InternalName].Loaded = false;
+        }
+    }
+}

Dalamud/Plugin/SelfTest/Internal/SelfTestRegistryPluginScoped.cs

@@ -0,0 +1,51 @@
+using System.Collections.Generic;
+
+using Dalamud.IoC;
+using Dalamud.IoC.Internal;
+using Dalamud.Plugin.Internal.Types;
+
+namespace Dalamud.Plugin.SelfTest.Internal;
+
+/// <summary>
+/// Plugin-scoped version of SelfTestRegistry.
+/// </summary>
+[PluginInterface]
+[ServiceManager.ScopedService]
+[ResolveVia<ISelfTestRegistry>]
+internal class SelfTestRegistryPluginScoped : ISelfTestRegistry, IInternalDisposableService
+{
+    [ServiceManager.ServiceDependency]
+    private readonly SelfTestRegistry selfTestRegistry = Service<SelfTestRegistry>.Get();
+
+    private readonly LocalPlugin plugin;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SelfTestRegistryPluginScoped"/> class.
+    /// </summary>
+    /// <param name="plugin">The plugin this service belongs to.</param>
+    [ServiceManager.ServiceConstructor]
+    public SelfTestRegistryPluginScoped(LocalPlugin plugin)
+    {
+        this.plugin = plugin;
+    }
+
+    /// <summary>
+    /// Gets the plugin name.
+    /// </summary>
+    public string PluginName { get; private set; }
+
+    /// <summary>
+    /// Registers test steps for this plugin.
+    /// </summary>
+    /// <param name="steps">The test steps to register.</param>
+    public void RegisterTestSteps(IEnumerable<ISelfTestStep> steps)
+    {
+        this.selfTestRegistry.RegisterPluginSelfTestSteps(this.plugin, steps);
+    }
+
+    /// <inheritdoc/>
+    void IInternalDisposableService.DisposeService()
+    {
+        this.selfTestRegistry.UnregisterPluginSelfTestSteps(this.plugin);
+    }
+}

Dalamud/Plugin/SelfTest/Internal/SelfTestWithResults.cs

@@ -0,0 +1,173 @@
+using Dalamud.Logging.Internal;
+using Dalamud.Plugin.Internal.Types;
+
+namespace Dalamud.Plugin.SelfTest.Internal;
+
+/// <summary>
+/// A self test step with result tracking.
+/// </summary>
+internal class SelfTestWithResults
+{
+    private static readonly ModuleLog Log = new("SelfTest");
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SelfTestWithResults"/> class.
+    /// </summary>
+    /// <param name="plugin">The plugin providing this test.</param>
+    /// <param name="group">The test group name.</param>
+    /// <param name="step">The test step.</param>
+    public SelfTestWithResults(LocalPlugin plugin, string group, ISelfTestStep step)
+    {
+        this.Plugin = plugin;
+        this.Group = group;
+        this.Step = step;
+    }
+
+    /// <summary>
+    /// Gets the test group name.
+    /// </summary>
+    public string Group { get; private set; }
+
+    /// <summary>
+    /// Gets the plugin that defined these tests. <c>null</c> for Dalamud tests.
+    /// </summary>
+    public LocalPlugin? Plugin { get; private set; }
+
+    /// <summary>
+    /// Gets the test name.
+    /// </summary>
+    public string Name { get => this.Step.Name; }
+
+    /// <summary>
+    /// Gets a value indicating whether the test has run and finished.
+    /// </summary>
+    public bool Finished => this.Result == SelfTestStepResult.Fail || this.Result == SelfTestStepResult.Pass;
+
+    /// <summary>
+    /// Gets a value indicating whether the plugin that provided this test has been unloaded.
+    /// </summary>
+    public bool Unloaded => this.Step == null;
+
+    /// <summary>
+    /// Gets the most recent result of running this test.
+    /// </summary>
+    public SelfTestStepResult Result { get; private set; } = SelfTestStepResult.NotRan;
+
+    /// <summary>
+    /// Gets the last time this test was started.
+    /// </summary>
+    public DateTimeOffset? StartTime { get; private set; } = null;
+
+    /// <summary>
+    /// Gets how long it took (or is taking) for this test to execute.
+    /// </summary>
+    public TimeSpan? Duration { get; private set; } = null;
+
+    /// <summary>
+    /// Gets or sets the Step that our results are for.
+    ///
+    /// If <c>null</c> it means the Plugin that provided this test has been unloaded and we can't use this test anymore.
+    /// </summary>
+    private ISelfTestStep? Step { get; set; }
+
+    /// <summary>
+    /// Creates a SelfTestWithResults from a Dalamud step.
+    /// </summary>
+    /// <param name="step">The step to wrap.</param>
+    /// <returns>A new SelfTestWithResults instance.</returns>
+    public static SelfTestWithResults FromDalamudStep(ISelfTestStep step)
+    {
+        return new SelfTestWithResults(plugin: null, group: "Dalamud", step: step);
+    }
+
+    /// <summary>
+    /// Creates a SelfTestWithResults from a plugin step.
+    /// </summary>
+    /// <param name="plugin">The plugin providing the step.</param>
+    /// <param name="step">The step to wrap.</param>
+    /// <returns>A new SelfTestWithResults instance.</returns>
+    public static SelfTestWithResults FromPluginStep(LocalPlugin plugin, ISelfTestStep step)
+    {
+        return new SelfTestWithResults(plugin: plugin, group: plugin.InternalName, step: step);
+    }
+
+    /// <summary>
+    /// Reset the test.
+    /// </summary>
+    public void Reset()
+    {
+        this.Result = SelfTestStepResult.NotRan;
+        this.StartTime = null;
+        this.Duration = null;
+    }
+
+    /// <summary>
+    /// Finish the currently running test and clean up any state. This preserves test run results.
+    /// </summary>
+    public void Finish()
+    {
+        if (this.Step == null)
+        {
+            return;
+        }
+
+        if (this.Result == SelfTestStepResult.NotRan)
+        {
+            return;
+        }
+
+        this.Step.CleanUp();
+    }
+
+    /// <summary>
+    /// Steps the state of this Self Test. This should be called every frame that we care about the results of this test.
+    /// </summary>
+    public void DrawAndStep()
+    {
+        // If we've been unloaded then there's nothing to do.
+        if (this.Step == null)
+        {
+            return;
+        }
+
+        // If we have already finished then there's nothing to do
+        if (this.Finished)
+        {
+            return;
+        }
+
+        // Otherwise, we assume that calling this functions means we are running the test.
+        if (this.Result == SelfTestStepResult.NotRan)
+        {
+            this.StartTime = DateTimeOffset.Now;
+            this.Result = SelfTestStepResult.Waiting;
+        }
+
+        try
+        {
+            this.Result = this.Step.RunStep();
+        }
+        catch (Exception ex)
+        {
+            Log.Error(ex, $"Step failed: {this.Name} ({this.Group})");
+            this.Result = SelfTestStepResult.Fail;
+        }
+
+        this.Duration = DateTimeOffset.Now - this.StartTime;
+
+        // If we ran and finished we need to clean up
+        if (this.Finished)
+        {
+            this.Finish();
+        }
+    }
+
+    /// <summary>
+    /// Unloads the test and cleans up.
+    /// </summary>
+    public void Unload()
+    {
+        this.Finish();
+        this.Step = null;
+    }
+}

Dalamud/Plugin/SelfTest/SelfTestStepResult.cs

@@ -1,9 +1,9 @@
-namespace Dalamud.Interface.Internal.Windows.SelfTest;
+namespace Dalamud.Plugin.SelfTest;
 
 /// <summary>
 /// Enum declaring result states of tests.
 /// </summary>
-internal enum SelfTestStepResult
+public enum SelfTestStepResult
 {
     /// <summary>
     /// Test was not ran.

Dalamud/Utility/Util.cs

@@ -18,6 +18,7 @@ using Dalamud.Game;
 using Dalamud.Game.ClientState.Objects.SubKinds;
 using Dalamud.Game.ClientState.Objects.Types;
 using Dalamud.Interface.Colors;
+using Dalamud.Interface.Internal;
 using Dalamud.Interface.Utility;
 using Dalamud.Interface.Utility.Raii;
 using Dalamud.Support;
@@ -28,8 +29,6 @@ using Windows.Win32.System.Memory;
 using Windows.Win32.System.Ole;
 using Windows.Win32.UI.WindowsAndMessaging;
 
-using Dalamud.Interface.Internal;
-
 using FLASHWINFO = Windows.Win32.UI.WindowsAndMessaging.FLASHWINFO;
 using HWND = Windows.Win32.Foundation.HWND;
 using MEMORY_BASIC_INFORMATION = Windows.Win32.System.Memory.MEMORY_BASIC_INFORMATION;

lib/FFXIVClientStructs

@@ -1 +1 @@
-Subproject commit 2dfa068007695c0177dbb362034c6a26e036069f
+Subproject commit 2b2d67e374b9f117d3d8038070bd80909de3941d