Update Lumina and use MacroStringParser (#2033)

No longer requires hook and main thread requirements on compiling macro
strings.

Needs lookup table fixing on Lumina; using reflection to fix for the
time being.

Dalamud.CorePlugin/Dalamud.CorePlugin.csproj

@@ -27,7 +27,7 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="Lumina" Version="4.1.1" />
+        <PackageReference Include="Lumina" Version="4.2.1" />
         <PackageReference Include="Lumina.Excel" Version="7.0.1" />
         <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
         <PackageReference Include="StyleCop.Analyzers" Version="1.2.0-beta.333">

Dalamud/Dalamud.csproj

@@ -71,7 +71,7 @@
         <PackageReference Include="goaaats.Reloaded.Hooks" Version="4.2.0-goat.4" />
         <PackageReference Include="goaaats.Reloaded.Assembler" Version="1.0.14-goat.2" />
         <PackageReference Include="JetBrains.Annotations" Version="2024.2.0" />
-        <PackageReference Include="Lumina" Version="4.1.1" />
+        <PackageReference Include="Lumina" Version="4.2.1" />
         <PackageReference Include="Lumina.Excel" Version="7.0.1" />
         <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="9.0.0-preview.1.24081.5" />
         <PackageReference Include="Microsoft.Windows.CsWin32" Version="0.3.46-beta">

Dalamud/Game/Text/SeStringHandling/Payloads/DalamudLinkPayload.cs

@@ -37,17 +37,19 @@ public class DalamudLinkPayload : Payload
     /// <inheritdoc/>
     protected override byte[] EncodeImpl()
     {
-        return new Lumina.Text.SeStringBuilder()
+        var ssb = Lumina.Text.SeStringBuilder.SharedPool.Get();
-               .BeginMacro(MacroCode.Link)
+        var res = ssb.BeginMacro(MacroCode.Link)
-               .AppendIntExpression((int)EmbeddedInfoType.DalamudLink - 1)
+                     .AppendIntExpression((int)EmbeddedInfoType.DalamudLink - 1)
-               .AppendUIntExpression(this.CommandId)
+                     .AppendUIntExpression(this.CommandId)
-               .AppendIntExpression(this.Extra1)
+                     .AppendIntExpression(this.Extra1)
-               .AppendIntExpression(this.Extra2)
+                     .AppendIntExpression(this.Extra2)
-               .BeginStringExpression()
+                     .BeginStringExpression()
-               .Append(JsonConvert.SerializeObject(new[] { this.Plugin, this.ExtraString }))
+                     .Append(JsonConvert.SerializeObject(new[] { this.Plugin, this.ExtraString }))
-               .EndExpression()
+                     .EndExpression()
-               .EndMacro()
+                     .EndMacro()
-               .ToArray();
+                     .ToArray();
+        Lumina.Text.SeStringBuilder.SharedPool.Return(ssb);
+        return res;
     }
 
     /// <inheritdoc/>

Dalamud/Interface/ImGuiSeStringRenderer/Internal/SeStringRenderer.cs

@@ -1,4 +1,3 @@
-using System.Buffers;
 using System.Collections.Generic;
 using System.Numerics;
 using System.Runtime.InteropServices;
@@ -16,18 +15,16 @@ using Dalamud.Utility;
 
 using FFXIVClientStructs.FFXIV.Client.System.String;
 using FFXIVClientStructs.FFXIV.Client.UI;
-using FFXIVClientStructs.FFXIV.Client.UI.Misc;
 
 using ImGuiNET;
 
 using Lumina.Excel.GeneratedSheets2;
+using Lumina.Text.Parse;
 using Lumina.Text.Payloads;
 using Lumina.Text.ReadOnly;
 
 using static Dalamud.Game.Text.SeStringHandling.BitmapFontIcon;
 
-using SeStringBuilder = Lumina.Text.SeStringBuilder;
-
 namespace Dalamud.Interface.ImGuiSeStringRenderer.Internal;
 
 /// <summary>Draws SeString.</summary>
@@ -46,30 +43,12 @@ internal unsafe class SeStringRenderer : IInternalDisposableService
     /// of this placeholder. On its own, usually displayed like <c>[OBJ]</c>.</summary>
     private const int ObjectReplacementCharacter = '\uFFFC';
 
-    /// <summary>SeString to return instead, if macro encoder has failed and could not provide us the reason.</summary>
-    private static readonly ReadOnlySeString MacroEncoderEncodeStringError =
-        new SeStringBuilder()
-            .BeginMacro(MacroCode.ColorType).AppendIntExpression(508).EndMacro()
-            .BeginMacro(MacroCode.EdgeColorType).AppendIntExpression(509).EndMacro()
-            .Append(
-                "<encode failed, and error message generation failed because the part that caused the error was too long>"u8)
-            .BeginMacro(MacroCode.EdgeColorType).AppendIntExpression(0).EndMacro()
-            .BeginMacro(MacroCode.ColorType).AppendIntExpression(0).EndMacro()
-            .ToReadOnlySeString();
-
     [ServiceManager.ServiceDependency]
     private readonly GameConfig gameConfig = Service<GameConfig>.Get();
 
     /// <summary>Cache of compiled SeStrings from <see cref="CompileAndCache"/>.</summary>
     private readonly ConcurrentLru<string, ReadOnlySeString> cache = new(1024);
 
-    /// <summary>Sets the global invalid parameter handler. Used to suppress <c>vsprintf_s</c> from raising.</summary>
-    /// <remarks>There exists a thread local version of this, but as the game-provided implementation is what
-    /// effectively is a screaming tool that the game has a bug, it should be safe to fail in any means.</remarks>
-    private readonly delegate* unmanaged<
-        delegate* unmanaged<char*, char*, char*, int, nuint, void>,
-        delegate* unmanaged<char*, char*, char*, int, nuint, void>> setInvalidParameterHandler;
-
     /// <summary>Parsed <c>gfdata.gfd</c> file, containing bitmap font icon lookup table.</summary>
     private readonly GfdFile gfd;
 
@@ -90,14 +69,6 @@ internal unsafe class SeStringRenderer : IInternalDisposableService
         this.colorStackSet = new(
             dm.Excel.GetSheet<UIColor>() ?? throw new InvalidOperationException("Failed to access UIColor sheet."));
         this.gfd = dm.GetFile<GfdFile>("common/font/gfdata.gfd")!;
-
-        // SetUnhandledExceptionFilter(who cares);
-        // _set_purecall_handler(() => *(int*)0 = 0xff14);
-        // _set_invalid_parameter_handler(() => *(int*)0 = 0xff14);
-        var f = sigScanner.ScanText(
-                    "ff 15 ?? ?? ?? ?? 48 8d 0d ?? ?? ?? ?? e8 ?? ?? ?? ?? 48 8d 0d ?? ?? ?? ?? e8 ?? ?? ?? ??") + 26;
-        fixed (void* p = &this.setInvalidParameterHandler)
-            *(nint*)p = *(int*)f + f + 4;
     }
 
     /// <summary>Finalizes an instance of the <see cref="SeStringRenderer"/> class.</summary>
@@ -106,72 +77,16 @@ internal unsafe class SeStringRenderer : IInternalDisposableService
     /// <inheritdoc/>
     void IInternalDisposableService.DisposeService() => this.ReleaseUnmanagedResources();
 
-    /// <summary>Compiles a SeString from a text macro representation.</summary>
-    /// <param name="text">SeString text macro representation.</param>
-    /// <returns>Compiled SeString.</returns>
-    public ReadOnlySeString Compile(ReadOnlySpan<byte> text)
-    {
-        // MacroEncoder looks stateful; disallowing calls from off main threads for now.
-        ThreadSafety.AssertMainThread();
-
-        var prev = this.setInvalidParameterHandler(&MsvcrtInvalidParameterHandlerDetour);
-        try
-        {
-            using var tmp = new Utf8String();
-            RaptureTextModule.Instance()->MacroEncoder.EncodeString(&tmp, text);
-            return new(tmp.AsSpan().ToArray());
-        }
-        catch (Exception)
-        {
-            return MacroEncoderEncodeStringError;
-        }
-        finally
-        {
-            this.setInvalidParameterHandler(prev);
-        }
-
-        [UnmanagedCallersOnly]
-        static void MsvcrtInvalidParameterHandlerDetour(char* a, char* b, char* c, int d, nuint e) =>
-            throw new InvalidOperationException();
-    }
-
-    /// <summary>Compiles a SeString from a text macro representation.</summary>
-    /// <param name="text">SeString text macro representation.</param>
-    /// <returns>Compiled SeString.</returns>
-    public ReadOnlySeString Compile(ReadOnlySpan<char> text)
-    {
-        var len = Encoding.UTF8.GetByteCount(text);
-        if (len >= 1024)
-        {
-            var buf = ArrayPool<byte>.Shared.Rent(len + 1);
-            buf[Encoding.UTF8.GetBytes(text, buf)] = 0;
-            var res = this.Compile(buf);
-            ArrayPool<byte>.Shared.Return(buf);
-            return res;
-        }
-        else
-        {
-            Span<byte> buf = stackalloc byte[len + 1];
-            buf[Encoding.UTF8.GetBytes(text, buf)] = 0;
-            return this.Compile(buf);
-        }
-    }
-
     /// <summary>Compiles and caches a SeString from a text macro representation.</summary>
     /// <param name="text">SeString text macro representation.
     /// Newline characters will be normalized to newline payloads.</param>
     /// <returns>Compiled SeString.</returns>
-    public ReadOnlySeString CompileAndCache(string text)
+    public ReadOnlySeString CompileAndCache(string text) =>
-    {
+        this.cache.GetOrAdd(
-        // MacroEncoder looks stateful; disallowing calls from off main threads for now.
-        // Note that this is replicated in context.Compile. Only access cache from the main thread.
-        ThreadSafety.AssertMainThread();
-
-        return this.cache.GetOrAdd(
             text,
-            static (text, context) => context.Compile(text.ReplaceLineEndings("<br>")),
+            static text => ReadOnlySeString.FromMacroString(
-            this);
+                text.ReplaceLineEndings("<br>"),
-    }
+                new() { ExceptionMode = MacroStringParseExceptionMode.EmbedError }));
 
     /// <summary>Compiles and caches a SeString from a text macro representation, and then draws it.</summary>
     /// <param name="text">SeString text macro representation.

Dalamud/Utility/SeStringExtensions.cs

@@ -1,4 +1,4 @@
-using Dalamud.Interface.ImGuiSeStringRenderer.Internal;
+using Lumina.Text.Parse;
 
 using DSeString = Dalamud.Game.Text.SeStringHandling.SeString;
 using DSeStringBuilder = Dalamud.Game.Text.SeStringHandling.SeStringBuilder;
@@ -24,44 +24,44 @@ public static class SeStringExtensions
     /// <param name="ssb">Target SeString builder.</param>
     /// <param name="macroString">Macro string in UTF-8 to compile and append to <paramref name="ssb"/>.</param>
     /// <returns><c>this</c> for method chaining.</returns>
-    /// <remarks>Must be called from the main thread.</remarks>
+    [Obsolete($"Use {nameof(LSeStringBuilder)}.{nameof(LSeStringBuilder.AppendMacroString)} directly instead.")]
-    public static LSeStringBuilder AppendMacroString(this LSeStringBuilder ssb, ReadOnlySpan<byte> macroString)
+    [Api11ToDo("Remove")]
-    {
+    public static LSeStringBuilder AppendMacroString(this LSeStringBuilder ssb, ReadOnlySpan<byte> macroString) =>
-        ThreadSafety.AssertMainThread();
+        ssb.AppendMacroString(macroString, new() { ExceptionMode = MacroStringParseExceptionMode.EmbedError });
-        return ssb.Append(Service<SeStringRenderer>.Get().Compile(macroString));
-    }
 
     /// <summary>Compiles and appends a macro string.</summary>
     /// <param name="ssb">Target SeString builder.</param>
     /// <param name="macroString">Macro string in UTF-16 to compile and append to <paramref name="ssb"/>.</param>
     /// <returns><c>this</c> for method chaining.</returns>
-    /// <remarks>Must be called from the main thread.</remarks>
+    [Obsolete($"Use {nameof(LSeStringBuilder)}.{nameof(LSeStringBuilder.AppendMacroString)} directly instead.")]
-    public static LSeStringBuilder AppendMacroString(this LSeStringBuilder ssb, ReadOnlySpan<char> macroString)
+    [Api11ToDo("Remove")]
-    {
+    public static LSeStringBuilder AppendMacroString(this LSeStringBuilder ssb, ReadOnlySpan<char> macroString) =>
-        ThreadSafety.AssertMainThread();
+        ssb.AppendMacroString(macroString, new() { ExceptionMode = MacroStringParseExceptionMode.EmbedError });
-        return ssb.Append(Service<SeStringRenderer>.Get().Compile(macroString));
-    }
 
     /// <summary>Compiles and appends a macro string.</summary>
     /// <param name="ssb">Target SeString builder.</param>
     /// <param name="macroString">Macro string in UTF-8 to compile and append to <paramref name="ssb"/>.</param>
     /// <returns><c>this</c> for method chaining.</returns>
-    /// <remarks>Must be called from the main thread.</remarks>
     public static DSeStringBuilder AppendMacroString(this DSeStringBuilder ssb, ReadOnlySpan<byte> macroString)
     {
-        ThreadSafety.AssertMainThread();
+        var lssb = LSeStringBuilder.SharedPool.Get();
-        return ssb.Append(DSeString.Parse(Service<SeStringRenderer>.Get().Compile(macroString)));
+        lssb.AppendMacroString(macroString, new() { ExceptionMode = MacroStringParseExceptionMode.EmbedError });
+        ssb.Append(DSeString.Parse(lssb.ToReadOnlySeString().Data.Span));
+        LSeStringBuilder.SharedPool.Return(lssb);
+        return ssb;
     }
 
     /// <summary>Compiles and appends a macro string.</summary>
     /// <param name="ssb">Target SeString builder.</param>
     /// <param name="macroString">Macro string in UTF-16 to compile and append to <paramref name="ssb"/>.</param>
     /// <returns><c>this</c> for method chaining.</returns>
-    /// <remarks>Must be called from the main thread.</remarks>
     public static DSeStringBuilder AppendMacroString(this DSeStringBuilder ssb, ReadOnlySpan<char> macroString)
     {
-        ThreadSafety.AssertMainThread();
+        var lssb = LSeStringBuilder.SharedPool.Get();
-        return ssb.Append(DSeString.Parse(Service<SeStringRenderer>.Get().Compile(macroString)));
+        lssb.AppendMacroString(macroString, new() { ExceptionMode = MacroStringParseExceptionMode.EmbedError });
+        ssb.Append(DSeString.Parse(lssb.ToReadOnlySeString().Data.Span));
+        LSeStringBuilder.SharedPool.Return(lssb);
+        return ssb;
     }
 
     /// <summary>