diff --git a/Dalamud/Interface/Internal/Windows/SelfTest/Steps/SeStringEvaluatorSelfTestStep.cs b/Dalamud/Interface/Internal/Windows/SelfTest/Steps/SeStringEvaluatorSelfTestStep.cs
index 8e66dd5cf..9853e31d4 100644
--- a/Dalamud/Interface/Internal/Windows/SelfTest/Steps/SeStringEvaluatorSelfTestStep.cs
+++ b/Dalamud/Interface/Internal/Windows/SelfTest/Steps/SeStringEvaluatorSelfTestStep.cs
@@ -1,6 +1,9 @@
 using Dalamud.Bindings.ImGui;
+using Dalamud.Configuration.Internal;
 using Dalamud.Game.ClientState;
 using Dalamud.Game.Text.Evaluator;
+using Dalamud.Game.Text.SeStringHandling.Payloads;
+
 using Lumina.Text.ReadOnly;
 
 namespace Dalamud.Interface.Internal.Windows.SelfTest.Steps;
@@ -75,6 +78,55 @@ internal class SeStringEvaluatorSelfTestStep : ISelfTestStep
                     return SelfTestStepResult.Waiting;
                 }
 
+                this.step++;
+                break;
+
+            case 2:
+                ImGui.Text("Checking AutoTranslatePayload.Text results..."u8);
+
+                var config = Service<DalamudConfiguration>.Get();
+                var originalLanguageOverride = config.LanguageOverride;
+
+                Span<(string Language, uint Group, uint Key, string ExpectedText)> tests = [
+                    ("en", 49u, 209u, " albino karakul "), // Mount
+                    ("en", 62u, 116u, " /echo "), // TextCommand - testing Command
+                    ("en", 62u, 143u, " /dutyfinder "), // TextCommand - testing Alias over Command
+                    ("en", 65u, 67u, " Minion of Light "), // Companion - testing noun handling for the german language (special case)
+                    ("en", 71u, 7u, " Phantom Geomancer "), // MKDSupportJob
+
+                    ("de", 49u, 209u, " Albino-Karakul "), // Mount
+                    ("de", 62u, 115u, " /freiegesellschaft "), // TextCommand - testing Alias over Command
+                    ("de", 62u, 116u, " /echo "), // TextCommand - testing Command
+                    ("de", 65u, 67u, " Begleiter des Lichts "), // Companion - testing noun handling for the german language (special case)
+                    ("de", 71u, 7u, " Phantom-Geomant "), // MKDSupportJob
+                ];
+
+                try
+                {
+                    foreach (var (language, group, key, expectedText) in tests)
+                    {
+                        config.LanguageOverride = language;
+
+                        var payload = new AutoTranslatePayload(group, key);
+
+                        if (payload.Text != expectedText)
+                        {
+                            ImGui.Text($"Test failed for Group {group}, Key {key}");
+                            ImGui.Text($"Expected: {expectedText}");
+                            ImGui.Text($"Got: {payload.Text}");
+
+                            if (ImGui.Button("Continue"u8))
+                                return SelfTestStepResult.Fail;
+
+                            return SelfTestStepResult.Waiting;
+                        }
+                    }
+                }
+                finally
+                {
+                    config.LanguageOverride = originalLanguageOverride;
+                }
+
                 return SelfTestStepResult.Pass;
         }