Add optional vtable swapchain hook mode

Dalamud/Configuration/Internal/DalamudConfiguration.cs

@@ -8,6 +8,7 @@ using System.Runtime.InteropServices;
 using Dalamud.Game.Text;
 using Dalamud.Interface;
 using Dalamud.Interface.FontIdentifier;
+using Dalamud.Interface.Internal;
 using Dalamud.Interface.Internal.ReShadeHandling;
 using Dalamud.Interface.Style;
 using Dalamud.IoC.Internal;
@@ -445,6 +446,9 @@ internal sealed class DalamudConfiguration : IInternalDisposableService
     /// <summary>Gets or sets the mode specifying how to handle ReShade.</summary>
     public ReShadeHandlingMode ReShadeHandlingMode { get; set; } = ReShadeHandlingMode.ReShadeAddon;
 
+    /// <summary>Gets or sets the swap chain hook mode.</summary>
+    public SwapChainHelper.HookMode SwapChainHookMode { get; set; } = SwapChainHelper.HookMode.ByteCode;
+
     /// <summary>
     /// Gets or sets hitch threshold for game network up in milliseconds.
     /// </summary>

Dalamud/Hooking/Internal/ObjectVTableHook.cs

@@ -0,0 +1,286 @@
+using System.Linq;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+using Dalamud.Utility;
+
+using Serilog;
+
+namespace Dalamud.Hooking.Internal;
+
+/// <summary>Manages a hook that works by replacing the vtable of target object.</summary>
+internal unsafe class ObjectVTableHook : IDisposable
+{
+    private readonly nint** ppVtbl;
+    private readonly int numMethods;
+
+    private readonly nint* pVtblOriginal;
+    private readonly nint[] vtblOverriden;
+
+    /// <summary>Extra data for overriden vtable entries, primarily for keeping references to delegates that are used
+    /// with <see cref="Marshal.GetFunctionPointerForDelegate"/>.</summary>
+    private readonly object?[] vtblOverridenTag;
+
+    private bool released;
+
+    /// <summary>Initializes a new instance of the <see cref="ObjectVTableHook"/> class.</summary>
+    /// <param name="ppVtbl">Address to vtable. Usually the address of the object itself.</param>
+    /// <param name="numMethods">Number of methods in this vtable.</param>
+    public ObjectVTableHook(nint ppVtbl, int numMethods)
+    {
+        this.ppVtbl = (nint**)ppVtbl;
+        this.numMethods = numMethods;
+        this.vtblOverridenTag = new object?[numMethods];
+        this.pVtblOriginal = *this.ppVtbl;
+        this.vtblOverriden = GC.AllocateArray<nint>(numMethods, true);
+        this.OriginalVTableSpan.CopyTo(this.vtblOverriden);
+    }
+
+    /// <summary>Initializes a new instance of the <see cref="ObjectVTableHook"/> class.</summary>
+    /// <param name="ppVtbl">Address to vtable. Usually the address of the object itself.</param>
+    /// <param name="numMethods">Number of methods in this vtable.</param>
+    public ObjectVTableHook(void* ppVtbl, int numMethods)
+        : this((nint)ppVtbl, numMethods)
+    {
+    }
+
+    /// <summary>Finalizes an instance of the <see cref="ObjectVTableHook"/> class.</summary>
+    ~ObjectVTableHook() => this.ReleaseUnmanagedResources();
+
+    /// <summary>Gets the span view of original vtable.</summary>
+    public ReadOnlySpan<nint> OriginalVTableSpan => new(this.pVtblOriginal, this.numMethods);
+
+    /// <summary>Gets the span view of overriden vtable.</summary>
+    public ReadOnlySpan<nint> OverridenVTableSpan => this.vtblOverriden.AsSpan();
+
+    /// <summary>Disables the hook.</summary>
+    public void Disable()
+    {
+        // already disabled
+        if (*this.ppVtbl == this.pVtblOriginal)
+            return;
+
+        if (*this.ppVtbl != Unsafe.AsPointer(ref this.vtblOverriden[0]))
+        {
+            Log.Warning(
+                "[{who}]: the object was hooked by something else; disabling may result in a crash.",
+                this.GetType().Name);
+        }
+
+        *this.ppVtbl = this.pVtblOriginal;
+    }
+
+    /// <inheritdoc />
+    public void Dispose()
+    {
+        this.ReleaseUnmanagedResources();
+        GC.SuppressFinalize(this);
+    }
+
+    /// <summary>Enables the hook.</summary>
+    public void Enable()
+    {
+        // already enabled
+        if (*this.ppVtbl == Unsafe.AsPointer(ref this.vtblOverriden[0]))
+            return;
+
+        if (*this.ppVtbl != this.pVtblOriginal)
+        {
+            Log.Warning(
+                "[{who}]: the object was hooked by something else; enabling may result in a crash.",
+                this.GetType().Name);
+        }
+
+        *this.ppVtbl = (nint*)Unsafe.AsPointer(ref this.vtblOverriden[0]);
+    }
+
+    /// <summary>Gets the original method address of the given method index.</summary>
+    /// <param name="methodIndex">Index of the method.</param>
+    /// <returns>Address of the original method.</returns>
+    public nint GetOriginalMethodAddress(int methodIndex)
+    {
+        this.EnsureMethodIndex(methodIndex);
+        return this.pVtblOriginal[methodIndex];
+    }
+
+    /// <summary>Gets the original method of the given method index, as a delegate of given type.</summary>
+    /// <param name="methodIndex">Index of the method.</param>
+    /// <typeparam name="T">Type of delegate.</typeparam>
+    /// <returns>Delegate to the original method.</returns>
+    public T GetOriginalMethodDelegate<T>(int methodIndex)
+        where T : Delegate
+    {
+        this.EnsureMethodIndex(methodIndex);
+        return Marshal.GetDelegateForFunctionPointer<T>(this.pVtblOriginal[methodIndex]);
+    }
+
+    /// <summary>Resets a method to the original function.</summary>
+    /// <param name="methodIndex">Index of the method.</param>
+    public void ResetVtableEntry(int methodIndex)
+    {
+        this.EnsureMethodIndex(methodIndex);
+        this.vtblOverriden[methodIndex] = this.pVtblOriginal[methodIndex];
+        this.vtblOverridenTag[methodIndex] = null;
+    }
+
+    /// <summary>Sets a method in vtable to the given address of function.</summary>
+    /// <param name="methodIndex">Index of the method.</param>
+    /// <param name="pfn">Address of the detour function.</param>
+    /// <param name="refkeep">Additional reference to keep in memory.</param>
+    public void SetVtableEntry(int methodIndex, nint pfn, object? refkeep)
+    {
+        this.EnsureMethodIndex(methodIndex);
+        this.vtblOverriden[methodIndex] = pfn;
+        this.vtblOverridenTag[methodIndex] = refkeep;
+    }
+
+    /// <summary>Sets a method in vtable to the given delegate.</summary>
+    /// <param name="methodIndex">Index of the method.</param>
+    /// <param name="detourDelegate">Detour delegate.</param>
+    /// <typeparam name="T">Type of delegate.</typeparam>
+    public void SetVtableEntry<T>(int methodIndex, T detourDelegate)
+        where T : Delegate =>
+        this.SetVtableEntry(methodIndex, Marshal.GetFunctionPointerForDelegate(detourDelegate), detourDelegate);
+
+    /// <summary>Sets a method in vtable to the given delegate.</summary>
+    /// <param name="methodIndex">Index of the method.</param>
+    /// <param name="detourDelegate">Detour delegate.</param>
+    /// <param name="originalMethodDelegate">Original method delegate.</param>
+    /// <typeparam name="T">Type of delegate.</typeparam>
+    public void SetVtableEntry<T>(int methodIndex, T detourDelegate, out T originalMethodDelegate)
+        where T : Delegate
+    {
+        originalMethodDelegate = this.GetOriginalMethodDelegate<T>(methodIndex);
+        this.SetVtableEntry(methodIndex, Marshal.GetFunctionPointerForDelegate(detourDelegate), detourDelegate);
+    }
+
+    /// <summary>Creates a new instance of <see cref="Hook{T}"/> that manages one entry in the vtable hook.</summary>
+    /// <param name="methodIndex">Index of the method.</param>
+    /// <param name="detourDelegate">Detour delegate.</param>
+    /// <typeparam name="T">Type of delegate.</typeparam>
+    /// <returns>A new instance of <see cref="Hook{T}"/>.</returns>
+    /// <remarks>Even if a single hook is enabled, without <see cref="Enable"/>, the hook will remain disabled.
+    /// </remarks>
+    public Hook<T> CreateHook<T>(int methodIndex, T detourDelegate) where T : Delegate =>
+        new SingleHook<T>(this, methodIndex, detourDelegate);
+
+    private void EnsureMethodIndex(int methodIndex)
+    {
+        ArgumentOutOfRangeException.ThrowIfNegative(methodIndex);
+        ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(methodIndex, this.numMethods);
+    }
+
+    private void ReleaseUnmanagedResources()
+    {
+        if (!this.released)
+        {
+            this.Disable();
+            this.released = true;
+        }
+    }
+
+    private sealed class SingleHook<T>(ObjectVTableHook hook, int methodIndex, T detourDelegate)
+        : Hook<T>((nint)hook.ppVtbl)
+        where T : Delegate
+    {
+        /// <inheritdoc/>
+        public override T Original { get; } = hook.GetOriginalMethodDelegate<T>(methodIndex);
+
+        /// <inheritdoc/>
+        public override bool IsEnabled =>
+            hook.OriginalVTableSpan[methodIndex] != hook.OverridenVTableSpan[methodIndex];
+
+        /// <inheritdoc/>
+        public override string BackendName => nameof(ObjectVTableHook);
+
+        /// <inheritdoc/>
+        public override void Enable() => hook.SetVtableEntry(methodIndex, detourDelegate);
+
+        /// <inheritdoc/>
+        public override void Disable() => hook.ResetVtableEntry(methodIndex);
+    }
+}
+
+/// <summary>Typed version of <see cref="ObjectVTableHook"/>.</summary>
+/// <typeparam name="TVTable">VTable struct.</typeparam>
+internal unsafe class ObjectVTableHook<TVTable> : ObjectVTableHook
+    where TVTable : unmanaged
+{
+    private static readonly string[] Fields =
+        typeof(TVTable).GetFields(BindingFlags.Instance | BindingFlags.Public).Select(x => x.Name).ToArray();
+
+    /// <summary>Initializes a new instance of the <see cref="ObjectVTableHook{TVTable}"/> class.</summary>
+    /// <param name="ppVtbl">Address to vtable. Usually the address of the object itself.</param>
+    public ObjectVTableHook(void* ppVtbl)
+        : base(ppVtbl, Fields.Length)
+    {
+    }
+
+    /// <summary>Gets the original vtable.</summary>
+    public ref readonly TVTable OriginalVTable => ref MemoryMarshal.Cast<nint, TVTable>(this.OriginalVTableSpan)[0];
+
+    /// <summary>Gets the overriden vtable.</summary>
+    public ref readonly TVTable OverridenVTable => ref MemoryMarshal.Cast<nint, TVTable>(this.OverridenVTableSpan)[0];
+
+    /// <summary>Gets the index of the method by method name.</summary>
+    /// <param name="methodName">Name of the method.</param>
+    /// <returns>Index of the method.</returns>
+    public int GetMethodIndex(string methodName) => Fields.IndexOf(methodName);
+
+    /// <summary>Gets the original method address of the given method index.</summary>
+    /// <param name="methodName">Name of the method.</param>
+    /// <returns>Address of the original method.</returns>
+    public nint GetOriginalMethodAddress(string methodName) =>
+        this.GetOriginalMethodAddress(this.GetMethodIndex(methodName));
+
+    /// <summary>Gets the original method of the given method index, as a delegate of given type.</summary>
+    /// <param name="methodName">Name of the method.</param>
+    /// <typeparam name="T">Type of delegate.</typeparam>
+    /// <returns>Delegate to the original method.</returns>
+    public T GetOriginalMethodDelegate<T>(string methodName)
+        where T : Delegate
+        => this.GetOriginalMethodDelegate<T>(this.GetMethodIndex(methodName));
+
+    /// <summary>Resets a method to the original function.</summary>
+    /// <param name="methodName">Name of the method.</param>
+    public void ResetVtableEntry(string methodName)
+        => this.ResetVtableEntry(this.GetMethodIndex(methodName));
+
+    /// <summary>Sets a method in vtable to the given address of function.</summary>
+    /// <param name="methodName">Name of the method.</param>
+    /// <param name="pfn">Address of the detour function.</param>
+    /// <param name="refkeep">Additional reference to keep in memory.</param>
+    public void SetVtableEntry(string methodName, nint pfn, object? refkeep)
+        => this.SetVtableEntry(this.GetMethodIndex(methodName), pfn, refkeep);
+
+    /// <summary>Sets a method in vtable to the given delegate.</summary>
+    /// <param name="methodName">Name of the method.</param>
+    /// <param name="detourDelegate">Detour delegate.</param>
+    /// <typeparam name="T">Type of delegate.</typeparam>
+    public void SetVtableEntry<T>(string methodName, T detourDelegate)
+        where T : Delegate =>
+        this.SetVtableEntry(
+            this.GetMethodIndex(methodName),
+            Marshal.GetFunctionPointerForDelegate(detourDelegate),
+            detourDelegate);
+
+    /// <summary>Sets a method in vtable to the given delegate.</summary>
+    /// <param name="methodName">Name of the method.</param>
+    /// <param name="detourDelegate">Detour delegate.</param>
+    /// <param name="originalMethodDelegate">Original method delegate.</param>
+    /// <typeparam name="T">Type of delegate.</typeparam>
+    public void SetVtableEntry<T>(string methodName, T detourDelegate, out T originalMethodDelegate)
+        where T : Delegate
+        => this.SetVtableEntry(this.GetMethodIndex(methodName), detourDelegate, out originalMethodDelegate);
+
+    /// <summary>Creates a new instance of <see cref="Hook{T}"/> that manages one entry in the vtable hook.</summary>
+    /// <param name="methodName">Name of the method.</param>
+    /// <param name="detourDelegate">Detour delegate.</param>
+    /// <typeparam name="T">Type of delegate.</typeparam>
+    /// <returns>A new instance of <see cref="Hook{T}"/>.</returns>
+    /// <remarks>Even if a single hook is enabled, without <see cref="ObjectVTableHook.Enable"/>, the hook will remain
+    /// disabled.</remarks>
+    public Hook<T> CreateHook<T>(string methodName, T detourDelegate) where T : Delegate =>
+        this.CreateHook(this.GetMethodIndex(methodName), detourDelegate);
+}

Dalamud/Interface/Internal/InterfaceManager.cs

@@ -35,6 +35,7 @@ using JetBrains.Annotations;
 
 using PInvoke;
 
+using TerraFX.Interop.DirectX;
 using TerraFX.Interop.Windows;
 
 // general dev notes, here because it's easiest
@@ -94,6 +95,7 @@ internal partial class InterfaceManager : IInternalDisposableService
     private Hook<SetCursorDelegate>? setCursorHook;
     private Hook<DxgiPresentDelegate>? dxgiPresentHook;
     private Hook<ResizeBuffersDelegate>? resizeBuffersHook;
+    private ObjectVTableHook<IDXGISwapChain.Vtbl<IDXGISwapChain>>? swapChainHook;
     private ReShadeAddonInterface? reShadeAddonInterface;
 
     private IFontAtlas? dalamudAtlas;
@@ -308,6 +310,7 @@ internal partial class InterfaceManager : IInternalDisposableService
             Interlocked.Exchange(ref this.setCursorHook, null)?.Dispose();
             Interlocked.Exchange(ref this.dxgiPresentHook, null)?.Dispose();
             Interlocked.Exchange(ref this.resizeBuffersHook, null)?.Dispose();
+            Interlocked.Exchange(ref this.swapChainHook, null)?.Dispose();
             Interlocked.Exchange(ref this.reShadeAddonInterface, null)?.Dispose();
         }
     }
@@ -769,12 +772,13 @@ internal partial class InterfaceManager : IInternalDisposableService
                 Log.Verbose("Unwrapped ReShade.");
         }
 
+        ResizeBuffersDelegate resizeBuffersDelegate;
+        DxgiPresentDelegate? dxgiPresentDelegate;
         if (this.dalamudConfiguration.ReShadeHandlingMode == ReShadeHandlingMode.ReShadeAddon &&
             ReShadeAddonInterface.TryRegisterAddon(out this.reShadeAddonInterface))
         {
-            this.resizeBuffersHook = Hook<ResizeBuffersDelegate>.FromAddress(
+            resizeBuffersDelegate = this.AsReShadeAddonResizeBuffersDetour;
-                (nint)SwapChainHelper.GameDeviceSwapChainVtbl->ResizeBuffers,
+            dxgiPresentDelegate = null;
-                this.AsReShadeAddonResizeBuffersDetour);
 
             Log.Verbose(
                 "Registered as a ReShade({name}: 0x{addr:X}) addon.",
@@ -786,21 +790,55 @@ internal partial class InterfaceManager : IInternalDisposableService
         }
         else
         {
-            this.resizeBuffersHook = Hook<ResizeBuffersDelegate>.FromAddress(
+            resizeBuffersDelegate = this.AsHookResizeBuffersDetour;
-                (nint)SwapChainHelper.GameDeviceSwapChainVtbl->ResizeBuffers,
+            dxgiPresentDelegate = this.PresentDetour;
-                this.AsHookResizeBuffersDetour);
-            
-            this.dxgiPresentHook = Hook<DxgiPresentDelegate>.FromAddress(
-                (nint)SwapChainHelper.GameDeviceSwapChainVtbl->Present,
-                this.PresentDetour);
         }
 
-        Log.Verbose($"IDXGISwapChain::ResizeBuffers address: {Util.DescribeAddress(this.resizeBuffersHook.Address)}");
+        switch (this.dalamudConfiguration.SwapChainHookMode)
+        {
+            case SwapChainHelper.HookMode.ByteCode: 
+            default:
+            {
+                this.resizeBuffersHook = Hook<ResizeBuffersDelegate>.FromAddress(
+                    (nint)SwapChainHelper.GameDeviceSwapChainVtbl->ResizeBuffers,
+                    resizeBuffersDelegate);
+
+                if (dxgiPresentDelegate is not null)
+                {
+                    this.dxgiPresentHook = Hook<DxgiPresentDelegate>.FromAddress(
+                        (nint)SwapChainHelper.GameDeviceSwapChainVtbl->Present,
+                        dxgiPresentDelegate);
+                }
+
+                break;
+            }
+
+            case SwapChainHelper.HookMode.VTable:
+            {
+                this.swapChainHook = new(SwapChainHelper.GameDeviceSwapChain);
+                this.resizeBuffersHook = this.swapChainHook.CreateHook(
+                    nameof(IDXGISwapChain.ResizeBuffers),
+                    resizeBuffersDelegate);
+
+                if (dxgiPresentDelegate is not null)
+                {
+                    this.dxgiPresentHook = this.swapChainHook.CreateHook(
+                        nameof(IDXGISwapChain.Present),
+                        dxgiPresentDelegate);
+                }
+
+                break;
+            }
+        }
+
+        Log.Verbose(
+            $"IDXGISwapChain::ResizeBuffers address: {Util.DescribeAddress(this.resizeBuffersHook.Address)}");
         Log.Verbose($"IDXGISwapChain::Present address: {Util.DescribeAddress(this.dxgiPresentHook?.Address ?? 0)}");
 
         this.setCursorHook.Enable();
-        this.dxgiPresentHook?.Enable();
         this.resizeBuffersHook.Enable();
+        this.dxgiPresentHook?.Enable();
+        this.swapChainHook?.Enable();
     }
 
     private IntPtr SetCursorDetour(IntPtr hCursor)

Dalamud/Interface/Internal/ReShadeHandling/ReShadeUnwrapper.cs

@@ -78,28 +78,24 @@ internal static unsafe class ReShadeUnwrapper
     {
         foreach (ProcessModule processModule in Process.GetCurrentProcess().Modules)
         {
-            if (ptr < processModule.BaseAddress || ptr >= processModule.BaseAddress + processModule.ModuleMemorySize)
+            if (ptr < processModule.BaseAddress ||
+                ptr >= processModule.BaseAddress + processModule.ModuleMemorySize ||
+                !HasProcExported(processModule, "ReShadeRegisterAddon"u8) ||
+                !HasProcExported(processModule, "ReShadeUnregisterAddon"u8) ||
+                !HasProcExported(processModule, "ReShadeRegisterEvent"u8) ||
+                !HasProcExported(processModule, "ReShadeUnregisterEvent"u8))
                 continue;
 
-            fixed (byte* pfn0 = "ReShadeRegisterAddon"u8)
-            fixed (byte* pfn1 = "ReShadeUnregisterAddon"u8)
-            fixed (byte* pfn2 = "ReShadeRegisterEvent"u8)
-            fixed (byte* pfn3 = "ReShadeUnregisterEvent"u8)
-            {
-                if (GetProcAddress((HMODULE)processModule.BaseAddress, (sbyte*)pfn0) == 0)
-                    continue;
-                if (GetProcAddress((HMODULE)processModule.BaseAddress, (sbyte*)pfn1) == 0)
-                    continue;
-                if (GetProcAddress((HMODULE)processModule.BaseAddress, (sbyte*)pfn2) == 0)
-                    continue;
-                if (GetProcAddress((HMODULE)processModule.BaseAddress, (sbyte*)pfn3) == 0)
-                    continue;
-            }
-
             return true;
         }
 
         return false;
+
+        static bool HasProcExported(ProcessModule m, ReadOnlySpan<byte> name)
+        {
+            fixed (byte* p = name)
+                return GetProcAddress((HMODULE)m.BaseAddress, (sbyte*)p) != 0;
+        }
     }
 
     private static bool IsReShadedComObject<T>(T* obj)

Dalamud/Interface/Internal/SwapChainHelper.cs

@@ -14,6 +14,16 @@ internal static unsafe class SwapChainHelper
 {
     private static IDXGISwapChain* foundGameDeviceSwapChain;
 
+    /// <summary>Describes how to hook <see cref="IDXGISwapChain"/> methods.</summary>
+    public enum HookMode
+    {
+        /// <summary>Hooks by rewriting the native bytecode.</summary>
+        ByteCode,
+
+        /// <summary>Hooks by providing an alternative vtable.</summary>
+        VTable,
+    }
+
     /// <summary>Gets the game's active instance of IDXGISwapChain that is initialized.</summary>
     /// <value>Address of the game's instance of IDXGISwapChain, or <c>null</c> if not available (yet.)</value>
     public static IDXGISwapChain* GameDeviceSwapChain

Dalamud/Interface/Internal/Windows/Settings/Tabs/SettingsTabExperimental.cs

@@ -98,7 +98,7 @@ public class SettingsTabExperimental : SettingsTab
             {
                 ReShadeHandlingMode.ReShadeAddon => Loc.Localize(
                     "DalamudSettingsReShadeHandlingModeReShadeAddonDescription",
-                    "Dalamud will register itself as a ReShade addon. Most compatibility is expected, but multi-monitor window option won't work too well."),
+                    "Dalamud will register itself as a ReShade addon. Most compatibility is expected, but multi-monitor window option will require reloading ReShade every time a new window is opened, or even may not work at all."),
                 ReShadeHandlingMode.UnwrapReShade => Loc.Localize(
                     "DalamudSettingsReShadeHandlingModeUnwrapReShadeDescription",
                     "Dalamud will exclude itself from all ReShade handling. Multi-monitor windows should work fine with this mode, but it may not be supported and crash in future ReShade versions."),
@@ -109,6 +109,17 @@ public class SettingsTabExperimental : SettingsTab
             },
         },
 
+        new GapSettingsEntry(5, true),
+
+        new EnumSettingsEntry<SwapChainHelper.HookMode>(
+            Loc.Localize("DalamudSettingsSwapChainHookMode", "Swap chain hooking mode"),
+            Loc.Localize(
+                "DalamudSettingsSwapChainHookModeHint",
+                "Depending on addons aside from Dalamud you use, you may have to use different options for Dalamud and other addons to cooperate.\nRestart is required for changes to take effect."),
+            c => c.SwapChainHookMode,
+            (v, c) => c.SwapChainHookMode = v,
+            fallbackValue: SwapChainHelper.HookMode.ByteCode),
+
         /* Disabling profiles after they've been enabled doesn't make much sense, at least not if the user has already created profiles.
         new GapSettingsEntry(5, true),