Merge branch 'refs/heads/pmgr/master'

This commit is contained in:
Ottermandias 2024-07-21 23:26:21 +02:00
commit 07382537a0
10 changed files with 501 additions and 25 deletions

@ -1 +1 @@
Subproject commit 9f1816f1b75003d01c5576769831c10f3d8948a7
Subproject commit f13818fd85b436d0a0f66293fe7c6b60d4bffe3c

View file

@ -0,0 +1,13 @@
using Iced.Intel;
namespace Penumbra.Interop.Hooks.ResourceLoading;
public class MappedCodeReader(UnmanagedMemoryAccessor data, long offset) : CodeReader
{
public override int ReadByte() {
if (offset >= data.Capacity)
return -1;
return data.ReadByte(offset++);
}
}

View file

@ -0,0 +1,30 @@
using Penumbra.GameData;
namespace Penumbra.Interop.Hooks.ResourceLoading;
public sealed class PapHandler(PapRewriter.PapResourceHandlerPrototype papResourceHandler) : IDisposable
{
private readonly PapRewriter _papRewriter = new(papResourceHandler);
public void Enable()
{
ReadOnlySpan<string> signatures =
[
Sigs.LoadAlwaysResidentMotionPacks,
Sigs.LoadWeaponDependentResidentMotionPacks,
Sigs.LoadInitialResidentMotionPacks,
Sigs.LoadMotionPacks,
Sigs.LoadMotionPacks2,
Sigs.LoadMigratoryMotionPack,
];
var stopwatch = Stopwatch.StartNew();
foreach (var sig in signatures)
_papRewriter.Rewrite(sig);
Penumbra.Log.Debug(
$"[PapHandler] Rewrote {signatures.Length} .pap functions for inlined GetResourceAsync in {stopwatch.ElapsedMilliseconds} ms.");
}
public void Dispose()
=> _papRewriter.Dispose();
}

View file

@ -0,0 +1,167 @@
using Dalamud.Hooking;
using Iced.Intel;
using OtterGui;
using Penumbra.String.Classes;
namespace Penumbra.Interop.Hooks.ResourceLoading;
public sealed class PapRewriter(PapRewriter.PapResourceHandlerPrototype papResourceHandler) : IDisposable
{
public unsafe delegate int PapResourceHandlerPrototype(void* self, byte* path, int length);
private readonly PeSigScanner _scanner = new();
private readonly Dictionary<nint, AsmHook> _hooks = [];
private readonly List<nint> _nativeAllocList = [];
public void Rewrite(string sig)
{
if (!_scanner.TryScanText(sig, out var address))
throw new Exception($"Signature [{sig}] could not be found.");
var funcInstructions = _scanner.GetFunctionInstructions(address).ToArray();
var hookPoints = ScanPapHookPoints(funcInstructions).ToList();
foreach (var hookPoint in hookPoints)
{
var stackAccesses = ScanStackAccesses(funcInstructions, hookPoint).ToList();
var stringAllocation = NativeAlloc(Utf8GamePath.MaxGamePathLength);
// We'll need to grab our true hook point; the location where we can change the path at our leisure.
// This is going to be the first call instruction after our 'hookPoint', so, we'll find that.
// Pretty scuffed, this might need a refactoring at some point.
// We're doing it by skipping to our hookPoint's address in the list of instructions inside the function; then getting next CALL
var skipIndex = funcInstructions.IndexOf(instr => instr.IP == hookPoint.IP) + 1;
var detourPoint = funcInstructions.Skip(skipIndex)
.First(instr => instr.Mnemonic == Mnemonic.Call);
// We'll also remove all the 'hookPoints' from 'stackAccesses'.
// We're handling the char *path redirection here, so we don't want this to hit the later code
foreach (var hp in hookPoints)
stackAccesses.RemoveAll(instr => instr.IP == hp.IP);
var detourPointer = Marshal.GetFunctionPointerForDelegate(papResourceHandler);
var targetRegister = hookPoint.Op0Register.ToString().ToLower();
var hookAddress = new IntPtr((long)detourPoint.IP);
var caveAllocation = NativeAlloc(16);
var hook = new AsmHook(
hookAddress,
[
"use64",
$"mov {targetRegister}, 0x{stringAllocation:x8}", // Move our char *path into the relevant register (rdx)
// After this asm stub, we have a call to Crc32(); since r9 is a volatile, unused register, we can use it ourselves
// We're essentially storing the original 2 arguments ('this', 'path'), in case they get mangled in our call
// We technically don't need to save rdx ('path'), since it'll be stringLoc, but eh
$"mov r9, 0x{caveAllocation:x8}",
"mov [r9], rcx",
"mov [r9+0x8], rdx",
// We can use 'rax' here too since it's also volatile, and it'll be overwritten by Crc32()'s return anyway
$"mov rax, 0x{detourPointer:x8}", // Get a pointer to our detour in place
"call rax", // Call detour
// Do the reverse process and retrieve the stored stuff
$"mov r9, 0x{caveAllocation:x8}",
"mov rcx, [r9]",
"mov rdx, [r9+0x8]",
// Plop 'rax' (our return value, the path size) into r8, so it's the third argument for the subsequent Crc32() call
"mov r8, rax",
], "Pap Redirection"
);
_hooks.Add(hookAddress, hook);
hook.Enable();
// Now we're adjusting every single reference to the stack allocated 'path' to our substantially bigger 'stringLoc'
UpdatePathAddresses(stackAccesses, stringAllocation);
}
}
private void UpdatePathAddresses(IEnumerable<Instruction> stackAccesses, nint stringAllocation)
{
foreach (var stackAccess in stackAccesses)
{
var hookAddress = new IntPtr((long)stackAccess.IP + stackAccess.Length);
// Hook already exists, means there's reuse of the same stack address across 2 GetResourceAsync; just skip
if (_hooks.ContainsKey(hookAddress))
continue;
var targetRegister = stackAccess.Op0Register.ToString().ToLower();
var hook = new AsmHook(
hookAddress,
[
"use64",
$"mov {targetRegister}, 0x{stringAllocation:x8}",
], "Pap Stack Accesses"
);
_hooks.Add(hookAddress, hook);
hook.Enable();
}
}
private static IEnumerable<Instruction> ScanStackAccesses(IEnumerable<Instruction> instructions, Instruction hookPoint)
{
return instructions.Where(instr =>
instr.Code == hookPoint.Code
&& instr.Op0Kind == hookPoint.Op0Kind
&& instr.Op1Kind == hookPoint.Op1Kind
&& instr.MemoryBase == hookPoint.MemoryBase
&& instr.MemoryDisplacement64 == hookPoint.MemoryDisplacement64)
.GroupBy(instr => instr.IP)
.Select(grp => grp.First());
}
// This is utterly fucked and hardcoded, but, again, it works
// Might be a neat idea for a more versatile kind of signature though
private static IEnumerable<Instruction> ScanPapHookPoints(Instruction[] funcInstructions)
{
for (var i = 0; i < funcInstructions.Length - 8; i++)
{
if (funcInstructions.AsSpan(i, 8) is
[
{ Code : Code.Lea_r64_m },
{ Code : Code.Lea_r64_m },
{ Mnemonic: Mnemonic.Call },
{ Code : Code.Lea_r64_m },
{ Mnemonic: Mnemonic.Call },
{ Code : Code.Lea_r64_m },
..,
]
)
yield return funcInstructions[i];
}
}
private unsafe nint NativeAlloc(nuint size)
{
var caveLoc = (nint)NativeMemory.Alloc(size);
_nativeAllocList.Add(caveLoc);
return caveLoc;
}
private static unsafe void NativeFree(nint mem)
=> NativeMemory.Free((void*)mem);
public void Dispose()
{
_scanner.Dispose();
foreach (var hook in _hooks.Values)
{
hook.Disable();
hook.Dispose();
}
_hooks.Clear();
foreach (var mem in _nativeAllocList)
NativeFree(mem);
_nativeAllocList.Clear();
}
}

View file

@ -0,0 +1,188 @@
using System.IO.MemoryMappedFiles;
using Iced.Intel;
using PeNet;
using Decoder = Iced.Intel.Decoder;
namespace Penumbra.Interop.Hooks.ResourceLoading;
// A good chunk of this was blatantly stolen from Dalamud's SigScanner 'cause Winter could not be faffed, Winter will definitely not rewrite it later
public unsafe class PeSigScanner : IDisposable
{
private readonly MemoryMappedFile _file;
private readonly MemoryMappedViewAccessor _textSection;
private readonly nint _moduleBaseAddress;
private readonly uint _textSectionVirtualAddress;
public PeSigScanner()
{
var mainModule = Process.GetCurrentProcess().MainModule!;
var fileName = mainModule.FileName;
_moduleBaseAddress = mainModule.BaseAddress;
if (fileName == null)
throw new Exception("Unable to obtain main module path. This should not happen.");
_file = MemoryMappedFile.CreateFromFile(fileName, FileMode.Open, null, 0, MemoryMappedFileAccess.Read);
using var fileStream = _file.CreateViewStream(0, 0, MemoryMappedFileAccess.Read);
var pe = new PeFile(fileStream);
var textSection = pe.ImageSectionHeaders!.First(header => header.Name == ".text");
var textSectionStart = textSection.PointerToRawData;
var textSectionSize = textSection.SizeOfRawData;
_textSectionVirtualAddress = textSection.VirtualAddress;
_textSection = _file.CreateViewAccessor(textSectionStart, textSectionSize, MemoryMappedFileAccess.Read);
}
private nint ScanText(string signature)
{
var scanRet = Scan(_textSection, signature);
if (*(byte*)scanRet is 0xE8 or 0xE9)
scanRet = ReadJmpCallSig(scanRet);
return scanRet;
}
private static nint ReadJmpCallSig(nint sigLocation)
{
var jumpOffset = *(int*)(sigLocation + 1);
return sigLocation + 5 + jumpOffset;
}
public bool TryScanText(string signature, out nint result)
{
try
{
result = ScanText(signature);
return true;
}
catch (KeyNotFoundException)
{
result = nint.Zero;
return false;
}
}
private nint Scan(MemoryMappedViewAccessor section, string signature)
{
var (needle, mask) = ParseSignature(signature);
var index = IndexOf(section, needle, mask);
if (index < 0)
throw new KeyNotFoundException($"Can't find a signature of {signature}");
return (nint)(_moduleBaseAddress + index - section.PointerOffset + _textSectionVirtualAddress);
}
private static (byte[] Needle, bool[] Mask) ParseSignature(string signature)
{
signature = signature.Replace(" ", string.Empty);
if (signature.Length % 2 != 0)
throw new ArgumentException("Signature without whitespaces must be divisible by two.", nameof(signature));
var needleLength = signature.Length / 2;
var needle = new byte[needleLength];
var mask = new bool[needleLength];
for (var i = 0; i < needleLength; i++)
{
var hexString = signature.Substring(i * 2, 2);
if (hexString is "??" or "**")
{
needle[i] = 0;
mask[i] = true;
continue;
}
needle[i] = byte.Parse(hexString, NumberStyles.AllowHexSpecifier);
mask[i] = false;
}
return (needle, mask);
}
private static int IndexOf(MemoryMappedViewAccessor section, byte[] needle, bool[] mask)
{
if (needle.Length > section.Capacity)
return -1;
var badShift = BuildBadCharTable(needle, mask);
var last = needle.Length - 1;
var offset = 0;
var maxOffset = section.Capacity - needle.Length;
byte* buffer = null;
section.SafeMemoryMappedViewHandle.AcquirePointer(ref buffer);
try
{
while (offset <= maxOffset)
{
int position;
for (position = last; needle[position] == *(buffer + position + offset) || mask[position]; position--)
{
if (position == 0)
return offset;
}
offset += badShift[*(buffer + offset + last)];
}
}
finally
{
section.SafeMemoryMappedViewHandle.ReleasePointer();
}
return -1;
}
private static int[] BuildBadCharTable(byte[] needle, bool[] mask)
{
int idx;
var last = needle.Length - 1;
var badShift = new int[256];
for (idx = last; idx > 0 && !mask[idx]; --idx)
{ }
var diff = last - idx;
if (diff == 0)
diff = 1;
for (idx = 0; idx <= 255; ++idx)
badShift[idx] = diff;
for (idx = last - diff; idx < last; ++idx)
badShift[needle[idx]] = last - idx;
return badShift;
}
// Detects function termination; this is done in a really stupid way that will possibly break if looked at wrong, but it'll work for now
// If this shits itself, go bother Winter to implement proper CFG + basic block detection
public IEnumerable<Instruction> GetFunctionInstructions(nint address)
{
var fileOffset = address - _textSectionVirtualAddress - _moduleBaseAddress;
var codeReader = new MappedCodeReader(_textSection, fileOffset);
var decoder = Decoder.Create(64, codeReader, (ulong)address.ToInt64());
do
{
decoder.Decode(out var instr);
// Yes, this is catastrophically bad, but it works for some cases okay
if (instr.Mnemonic == Mnemonic.Int3)
break;
yield return instr;
} while (true);
}
public void Dispose()
{
_textSection.Dispose();
_file.Dispose();
}
}

View file

@ -16,8 +16,10 @@ public unsafe class ResourceLoader : IDisposable, IService
private readonly ResourceService _resources;
private readonly FileReadService _fileReadService;
private readonly TexMdlService _texMdlService;
private readonly PapHandler _papHandler;
private ResolveData _resolvedData = ResolveData.Invalid;
private ResolveData _resolvedData = ResolveData.Invalid;
public event Action<Utf8GamePath>? PapRequested;
public ResourceLoader(ResourceService resources, FileReadService fileReadService, TexMdlService texMdlService)
{
@ -30,6 +32,33 @@ public unsafe class ResourceLoader : IDisposable, IService
_resources.ResourceHandleIncRef += IncRefProtection;
_resources.ResourceHandleDecRef += DecRefProtection;
_fileReadService.ReadSqPack += ReadSqPackDetour;
_papHandler = new PapHandler(PapResourceHandler);
_papHandler.Enable();
}
private int PapResourceHandler(void* self, byte* path, int length)
{
if (!Utf8GamePath.FromPointer(path, out var gamePath))
return length;
var (resolvedPath, _) = _incMode.Value
? (null, ResolveData.Invalid)
: _resolvedData.Valid
? (_resolvedData.ModCollection.ResolvePath(gamePath), _resolvedData)
: ResolvePath(gamePath, ResourceCategory.Chara, ResourceType.Pap);
if (!resolvedPath.HasValue || !Utf8GamePath.FromByteString(resolvedPath.Value.InternalName, out var utf8ResolvedPath))
{
PapRequested?.Invoke(gamePath);
return length;
}
NativeMemory.Copy(utf8ResolvedPath.Path.Path, path, (nuint)utf8ResolvedPath.Length);
path[utf8ResolvedPath.Length] = 0;
PapRequested?.Invoke(gamePath);
return utf8ResolvedPath.Length;
}
/// <summary> Load a resource for a given path and a specific collection. </summary>
@ -84,6 +113,7 @@ public unsafe class ResourceLoader : IDisposable, IService
_resources.ResourceHandleIncRef -= IncRefProtection;
_resources.ResourceHandleDecRef -= DecRefProtection;
_fileReadService.ReadSqPack -= ReadSqPackDetour;
_papHandler.Dispose();
}
private void ResourceHandler(ref ResourceCategory category, ref ResourceType type, ref int hash, ref Utf8GamePath path,

View file

@ -23,10 +23,10 @@
<DefineConstants>PROFILING;</DefineConstants>
</PropertyGroup>
<ItemGroup>
<Compile Remove="Mods\Subclasses\**" />
<EmbeddedResource Remove="Mods\Subclasses\**" />
<None Remove="Mods\Subclasses\**" />
<ItemGroup>
<Compile Remove="Mods\Subclasses\**" />
<EmbeddedResource Remove="Mods\Subclasses\**" />
<None Remove="Mods\Subclasses\**" />
</ItemGroup>
<ItemGroup>
@ -68,6 +68,10 @@
<HintPath>$(DalamudLibPath)Newtonsoft.Json.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="Iced">
<HintPath>$(DalamudLibPath)Iced.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="OtterTex.dll">
<HintPath>lib\OtterTex.dll</HintPath>
</Reference>
@ -79,6 +83,7 @@
<PackageReference Include="SharpCompress" Version="0.33.0" />
<PackageReference Include="SharpGLTF.Core" Version="1.0.0-alpha0030" />
<PackageReference Include="SharpGLTF.Toolkit" Version="1.0.0-alpha0030" />
<PackageReference Include="PeNet" Version="4.0.5" />
</ItemGroup>
<ItemGroup>

View file

@ -36,31 +36,47 @@ public sealed class ResourceWatcher : IDisposable, ITab, IUiService
private Regex? _logRegex;
private int _newMaxEntries;
public unsafe ResourceWatcher(ActorManager actors, Configuration config, ResourceService resources, ResourceLoader loader, ResourceHandleDestructor destructor)
public unsafe ResourceWatcher(ActorManager actors, Configuration config, ResourceService resources, ResourceLoader loader,
ResourceHandleDestructor destructor)
{
_actors = actors;
_config = config;
_ephemeral = config.Ephemeral;
_resources = resources;
_destructor = destructor;
_loader = loader;
_table = new ResourceWatcherTable(config.Ephemeral, _records);
_resources.ResourceRequested += OnResourceRequested;
_actors = actors;
_config = config;
_ephemeral = config.Ephemeral;
_resources = resources;
_destructor = destructor;
_loader = loader;
_table = new ResourceWatcherTable(config.Ephemeral, _records);
_resources.ResourceRequested += OnResourceRequested;
_destructor.Subscribe(OnResourceDestroyed, ResourceHandleDestructor.Priority.ResourceWatcher);
_loader.ResourceLoaded += OnResourceLoaded;
_loader.FileLoaded += OnFileLoaded;
_loader.ResourceLoaded += OnResourceLoaded;
_loader.FileLoaded += OnFileLoaded;
_loader.PapRequested += OnPapRequested;
UpdateFilter(_ephemeral.ResourceLoggingFilter, false);
_newMaxEntries = _config.MaxResourceWatcherRecords;
}
private void OnPapRequested(Utf8GamePath original)
{
if (_ephemeral.EnableResourceLogging && FilterMatch(original.Path, out var match))
Penumbra.Log.Information($"[ResourceLoader] [REQ] {match} was requested asynchronously.");
if (!_ephemeral.EnableResourceWatcher)
return;
var record = Record.CreateRequest(original.Path, false);
if (!_ephemeral.OnlyAddMatchingResources || _table.WouldBeVisible(record))
_newRecords.Enqueue(record);
}
public unsafe void Dispose()
{
Clear();
_records.TrimExcess();
_resources.ResourceRequested -= OnResourceRequested;
_resources.ResourceRequested -= OnResourceRequested;
_destructor.Unsubscribe(OnResourceDestroyed);
_loader.ResourceLoaded -= OnResourceLoaded;
_loader.FileLoaded -= OnFileLoaded;
_loader.ResourceLoaded -= OnResourceLoaded;
_loader.FileLoaded -= OnFileLoaded;
_loader.PapRequested -= OnPapRequested;
}
private void Clear()
@ -200,8 +216,7 @@ public sealed class ResourceWatcher : IDisposable, ITab, IUiService
private unsafe void OnResourceRequested(ref ResourceCategory category, ref ResourceType type, ref int hash, ref Utf8GamePath path,
Utf8GamePath original,
GetResourceParameters* parameters, ref bool sync, ref ResourceHandle* returnValue)
Utf8GamePath original, GetResourceParameters* parameters, ref bool sync, ref ResourceHandle* returnValue)
{
if (_ephemeral.EnableResourceLogging && FilterMatch(original.Path, out var match))
Penumbra.Log.Information($"[ResourceLoader] [REQ] {match} was requested {(sync ? "synchronously." : "asynchronously.")}");

View file

@ -11,6 +11,16 @@
"Unosquare.Swan.Lite": "3.0.0"
}
},
"PeNet": {
"type": "Direct",
"requested": "[4.0.5, )",
"resolved": "4.0.5",
"contentHash": "/OUfRnMG8STVuK8kTpdfm+WQGTDoUiJnI845kFw4QrDmv2gYourmSnH84pqVjHT1YHBSuRfCzfioIpHGjFJrGA==",
"dependencies": {
"PeNet.Asn1": "2.0.1",
"System.Security.Cryptography.Pkcs": "8.0.0"
}
},
"SharpCompress": {
"type": "Direct",
"requested": "[0.33.0, )",
@ -56,6 +66,11 @@
"resolved": "8.0.0",
"contentHash": "cjWrLkJXK0rs4zofsK4bSdg+jhDLTaxrkXu4gS6Y7MAlCvRyNNgwY/lJi5RDlQOnSZweHqoyvgvbdvQsRIW+hg=="
},
"PeNet.Asn1": {
"type": "Transitive",
"resolved": "2.0.1",
"contentHash": "YR2O2YokSAYB+7CXkCDN3bd6/p0K3/AicCPkOJHKUz500v1D/hulCuVlggguqNc3M0LgSfOZKGvVYg2ud1GA9A=="
},
"SharpGLTF.Runtime": {
"type": "Transitive",
"resolved": "1.0.0-alpha0030",
@ -64,6 +79,19 @@
"SharpGLTF.Core": "1.0.0-alpha0030"
}
},
"System.Formats.Asn1": {
"type": "Transitive",
"resolved": "8.0.0",
"contentHash": "AJukBuLoe3QeAF+mfaRKQb2dgyrvt340iMBHYv+VdBzCUM06IxGlvl0o/uPOS7lHnXPN6u8fFRHSHudx5aTi8w=="
},
"System.Security.Cryptography.Pkcs": {
"type": "Transitive",
"resolved": "8.0.0",
"contentHash": "ULmp3xoOwNYjOYp4JZ2NK/6NdTgiN1GQXzVVN1njQ7LOZ0d0B9vyMnhyqbIi9Qw4JXj1JgCsitkTShboHRx7Eg==",
"dependencies": {
"System.Formats.Asn1": "8.0.0"
}
},
"System.ValueTuple": {
"type": "Transitive",
"resolved": "4.5.0",
@ -94,7 +122,7 @@
"type": "Project",
"dependencies": {
"OtterGui": "[1.0.0, )",
"Penumbra.Api": "[5.0.0, )",
"Penumbra.Api": "[5.2.0, )",
"Penumbra.String": "[1.0.4, )"
}
},

View file

@ -6,7 +6,7 @@
"Description": "Runtime mod loader and manager.",
"InternalName": "Penumbra",
"AssemblyVersion": "1.1.1.2",
"TestingAssemblyVersion": "1.2.0.11",
"TestingAssemblyVersion": "1.2.0.12",
"RepoUrl": "https://github.com/xivdev/Penumbra",
"ApplicableVersion": "any",
"DalamudApiLevel": 9,
@ -19,7 +19,7 @@
"LoadRequiredState": 2,
"LoadSync": true,
"DownloadLinkInstall": "https://github.com/xivdev/Penumbra/releases/download/1.1.1.2/Penumbra.zip",
"DownloadLinkTesting": "https://github.com/xivdev/Penumbra/releases/download/testing_1.2.0.11/Penumbra.zip",
"DownloadLinkTesting": "https://github.com/xivdev/Penumbra/releases/download/testing_1.2.0.12/Penumbra.zip",
"DownloadLinkUpdate": "https://github.com/xivdev/Penumbra/releases/download/1.1.1.2/Penumbra.zip",
"IconUrl": "https://raw.githubusercontent.com/xivdev/Penumbra/master/images/icon.png"
}