ToxicMushroom/Penumbra
1
0
Fork 0
mirror of https://github.com/xivdev/Penumbra.git synced 2025-12-12 18:27:24 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rework Interop/Loader Services.

Browse source
This commit is contained in:
Ottermandias 2023-03-16 15:15:42 +01:00
parent 99fd4b7806
commit 0df12a34cb
32 changed files with 1137 additions and 1421 deletions
Download patch file Download diff file Expand all files Collapse all files

2
OtterGui

@ -1 +1 @@
Subproject commit 3d346700e8800c045aa19d70d516d8a4fda2f2ee
Subproject commit df1cd8b02d729b2e7f585c301105b37c70d81c3e

2
Penumbra.String

@ -1 +1 @@
Subproject commit 2f999713c5b692fead3fb28c39002d1cd82c9261
Subproject commit 81f384cf96a9257b1ee2c7019772f30df78ba417

25
Penumbra/Collections/ModCollection.Cache.Access.cs
View file

@ -4,10 +4,12 @@ using Penumbra.Meta.Manager;
using Penumbra.Mods;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using Penumbra.Interop;
using Penumbra.Meta.Files;
using Penumbra.Meta.Manipulations;
using Penumbra.String.Classes;
@ -43,14 +45,10 @@ public partial class ModCollection
public void Apply(Mod.TemporaryMod tempMod, bool created)
{
if (created)
{
_cache?.AddMod(tempMod, tempMod.TotalManipulations > 0);
}
else
{
_cache?.ReloadMod(tempMod, tempMod.TotalManipulations > 0);
}
}
public void Remove(Mod.TemporaryMod tempMod)
{
@ -79,18 +77,14 @@ public partial class ModCollection
internal void ForceFile(Utf8GamePath path, FullPath fullPath)
{
if (CheckFullPath(path, fullPath))
{
_cache!.ResolvedFiles[path] = new ModPath(Mod.ForcedFiles, fullPath);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool CheckFullPath(Utf8GamePath path, FullPath fullPath)
{
if (fullPath.InternalName.Length < Utf8GamePath.MaxGamePathLength)
{
return true;
}
Penumbra.Log.Error($"The redirected path is too long to add the redirection\n\t{path}\n\t--> {fullPath}");
return false;
@ -104,6 +98,15 @@ public partial class ModCollection
internal MetaManager? MetaCache
=> _cache?.MetaManipulations;
public bool GetImcFile(Utf8GamePath path, [NotNullWhen(true)] out ImcFile? file)
{
if (_cache != null)
return _cache.MetaManipulations.GetImcFile(path, out file);
file = null;
return false;
}
internal IReadOnlyDictionary<Utf8GamePath, ModPath> ResolvedFiles
=> _cache?.ResolvedFiles ?? new Dictionary<Utf8GamePath, ModPath>();
@ -126,9 +129,7 @@ public partial class ModCollection
{
// Skip the empty collection.
if (Index == 0)
{
return;
}
Penumbra.Log.Debug($"[{Thread.CurrentThread.ManagedThreadId}] Recalculating effective file list for {AnonymizedName}");
_cache ??= new Cache(this);
@ -153,14 +154,10 @@ public partial class ModCollection
public void SetMetaFile(Interop.Structs.CharacterUtility.Index idx)
{
if (_cache == null)
{
Penumbra.CharacterUtility.ResetResource(idx);
}
else
{
_cache.MetaManipulations.SetFile(idx);
}
}
// Used for short periods of changed files.
public CharacterUtility.List.MetaReverter TemporarilySetEqdpFile(GenderRace genderRace, bool accessory)

1
Penumbra/Configuration.cs
View file

@ -60,6 +60,7 @@ public class Configuration : IPluginConfiguration
public bool EnableResourceLogging { get; set; } = false;
public string ResourceLoggingFilter { get; set; } = string.Empty;
public bool EnableResourceWatcher { get; set; } = false;
public bool OnlyAddMatchingResources { get; set; } = true;
public int MaxResourceWatcherRecords { get; set; } = ResourceWatcher.DefaultMaxEntries;
public ResourceTypeFlag ResourceWatcherResourceTypes { get; set; } = ResourceExtensions.AllResourceTypes;

11
Penumbra/Interop/CharacterUtility.DecalReverter.cs
View file

@ -2,6 +2,7 @@ using System;
using FFXIVClientStructs.FFXIV.Client.System.Resource;
using Penumbra.Collections;
using Penumbra.GameData.Enums;
using Penumbra.Interop.Loader;
using Penumbra.String.Classes;
namespace Penumbra.Interop;
@ -11,15 +12,15 @@ public unsafe partial class CharacterUtility
public sealed class DecalReverter : IDisposable
{
public static readonly Utf8GamePath DecalPath =
Utf8GamePath.FromString( "chara/common/texture/decal_equip/_stigma.tex", out var p ) ? p : Utf8GamePath.Empty;
Utf8GamePath.FromSpan("chara/common/texture/decal_equip/_stigma.tex"u8, out var p) ? p : Utf8GamePath.Empty;
public static readonly Utf8GamePath TransparentPath =
Utf8GamePath.FromString( "chara/common/texture/transparent.tex", out var p ) ? p : Utf8GamePath.Empty;
Utf8GamePath.FromSpan("chara/common/texture/transparent.tex"u8, out var p) ? p : Utf8GamePath.Empty;
private readonly Structs.TextureResourceHandle* _decal;
private readonly Structs.TextureResourceHandle* _transparent;
public DecalReverter( ModCollection? collection, bool doDecal )
public DecalReverter( ResourceService resources, ModCollection? collection, bool doDecal )
{
var ptr = Penumbra.CharacterUtility.Address;
_decal = null;
@ -27,7 +28,7 @@ public unsafe partial class CharacterUtility
if( doDecal )
{
var decalPath = collection?.ResolvePath( DecalPath )?.InternalName ?? DecalPath.Path;
var decalHandle = Penumbra.ResourceLoader.ResolvePathSync( ResourceCategory.Chara, ResourceType.Tex, decalPath );
var decalHandle = resources.GetResource( ResourceCategory.Chara, ResourceType.Tex, decalPath );
_decal = ( Structs.TextureResourceHandle* )decalHandle;
if( _decal != null )
{
@ -37,7 +38,7 @@ public unsafe partial class CharacterUtility
else
{
var transparentPath = collection?.ResolvePath( TransparentPath )?.InternalName ?? TransparentPath.Path;
var transparentHandle = Penumbra.ResourceLoader.ResolvePathSync( ResourceCategory.Chara, ResourceType.Tex, transparentPath );
var transparentHandle = resources.GetResource(ResourceCategory.Chara, ResourceType.Tex, transparentPath);
_transparent = ( Structs.TextureResourceHandle* )transparentHandle;
if( _transparent != null )
{

37
Penumbra/Interop/FontReloader.cs
View file

@ -6,52 +6,37 @@ namespace Penumbra.Interop;
// Handle font reloading via game functions.
// May cause a interface flicker while reloading.
public static unsafe class FontReloader
public unsafe class FontReloader
{
private static readonly AtkModule* AtkModule = null;
private static readonly delegate* unmanaged<AtkModule*, bool, bool, void> ReloadFontsFunc = null;
public bool Valid
=> _reloadFontsFunc != null;
public static bool Valid
=> ReloadFontsFunc != null;
public static void Reload()
public void Reload()
{
if (Valid)
{
ReloadFontsFunc( AtkModule, false, true );
}
_reloadFontsFunc(_atkModule, false, true);
else
{
Penumbra.Log.Error("Could not reload fonts, function could not be found.");
}
}
static FontReloader()
{
if( ReloadFontsFunc != null )
{
return;
}
private readonly AtkModule* _atkModule = null!;
private readonly delegate* unmanaged<AtkModule*, bool, bool, void> _reloadFontsFunc = null!;
public FontReloader()
{
var framework = Framework.Instance();
if (framework == null)
{
return;
}
var uiModule = framework->GetUiModule();
if (uiModule == null)
{
return;
}
var atkModule = uiModule->GetRaptureAtkModule();
if (atkModule == null)
{
return;
}
AtkModule = &atkModule->AtkModule;
ReloadFontsFunc = ( ( delegate* unmanaged< AtkModule*, bool, bool, void >* )AtkModule->vtbl )[ Offsets.ReloadFontsVfunc ];
_atkModule = &atkModule->AtkModule;
_reloadFontsFunc = ((delegate* unmanaged< AtkModule*, bool, bool, void >*)_atkModule->vtbl)[Offsets.ReloadFontsVfunc];
}
}

106
Penumbra/Interop/Loader/CharacterResolver.cs Normal file
View file

@ -0,0 +1,106 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
using FFXIVClientStructs.FFXIV.Client.System.Resource;
using Penumbra.Api;
using Penumbra.Collections;
using Penumbra.GameData.Enums;
using Penumbra.Interop.Resolver;
using Penumbra.Interop.Structs;
using Penumbra.String;
using Penumbra.String.Classes;
namespace Penumbra.Interop.Loader;
public class CharacterResolver : IDisposable
{
private readonly Configuration _config;
private readonly ModCollection.Manager _collectionManager;
private readonly TempCollectionManager _tempCollections;
private readonly ResourceLoader _loader;
private readonly PathResolver _pathResolver;
public unsafe CharacterResolver(Configuration config, ModCollection.Manager collectionManager, TempCollectionManager tempCollections,
ResourceLoader loader, PathResolver pathResolver)
{
_config = config;
_collectionManager = collectionManager;
_tempCollections = tempCollections;
_loader = loader;
_pathResolver = pathResolver;
_loader.ResolvePath = ResolvePath;
_loader.FileLoaded += ImcLoadResource;
}
/// <summary> Obtain a temporary or permanent collection by name. </summary>
public bool CollectionByName(string name, [NotNullWhen(true)] out ModCollection? collection)
=> _tempCollections.CollectionByName(name, out collection) || _collectionManager.ByName(name, out collection);
/// <summary> Try to resolve the given game path to the replaced path. </summary>
public (FullPath?, ResolveData) ResolvePath(Utf8GamePath path, ResourceCategory category, ResourceType resourceType)
{
// Check if mods are enabled or if we are in a inc-ref at 0 reference count situation.
if (!_config.EnableMods)
return (null, ResolveData.Invalid);
path = path.ToLower();
return category switch
{
// Only Interface collection.
ResourceCategory.Ui => (_collectionManager.Interface.ResolvePath(path),
_collectionManager.Interface.ToResolveData()),
// Never allow changing scripts.
ResourceCategory.UiScript => (null, ResolveData.Invalid),
ResourceCategory.GameScript => (null, ResolveData.Invalid),
// Use actual resolving.
ResourceCategory.Chara => _pathResolver.CharacterResolver(path, resourceType),
ResourceCategory.Shader => _pathResolver.CharacterResolver(path, resourceType),
ResourceCategory.Vfx => _pathResolver.CharacterResolver(path, resourceType),
ResourceCategory.Sound => _pathResolver.CharacterResolver(path, resourceType),
// None of these files are ever associated with specific characters,
// always use the default resolver for now.
ResourceCategory.Common => DefaultResolver(path),
ResourceCategory.BgCommon => DefaultResolver(path),
ResourceCategory.Bg => DefaultResolver(path),
ResourceCategory.Cut => DefaultResolver(path),
ResourceCategory.Exd => DefaultResolver(path),
ResourceCategory.Music => DefaultResolver(path),
_ => DefaultResolver(path),
};
}
// TODO
public unsafe void Dispose()
{
_loader.ResetResolvePath();
_loader.FileLoaded -= ImcLoadResource;
_pathResolver.Dispose();
}
// Use the default method of path replacement.
private (FullPath?, ResolveData) DefaultResolver(Utf8GamePath path)
{
var resolved = _collectionManager.Default.ResolvePath(path);
return (resolved, _collectionManager.Default.ToResolveData());
}
/// <summary> After loading an IMC file, replace its contents with the modded IMC file. </summary>
private unsafe void ImcLoadResource(ResourceHandle* resource, ByteString path, bool returnValue, bool custom, ByteString additionalData)
{
if (resource->FileType != ResourceType.Imc)
return;
var lastUnderscore = additionalData.LastIndexOf((byte)'_');
var name = lastUnderscore == -1 ? additionalData.ToString() : additionalData.Substring(0, lastUnderscore).ToString();
if (Utf8GamePath.FromByteString(path, out var gamePath)
&& CollectionByName(name, out var collection)
&& collection.HasCache
&& collection.GetImcFile(gamePath, out var file))
{
file.Replace(resource);
Penumbra.Log.Verbose(
$"[ResourceLoader] Loaded {gamePath} from file and replaced with IMC from collection {collection.AnonymizedName}.");
}
}
}

145
Penumbra/Interop/Loader/CreateFileWHook.cs
View file

@ -18,83 +18,18 @@ public unsafe class CreateFileWHook : IDisposable
{
public const int RequiredSize = 28;
// The prefix is not valid for any actual path, so should never run into false-positives.
private const char Prefix = ( char )( ( byte )'P' | ( ( '?' & 0x00FF ) << 8 ) );
private const int BufferSize = Utf8GamePath.MaxGamePathLength;
private delegate nint CreateFileWDelegate( char* fileName, uint access, uint shareMode, nint security, uint creation, uint flags, nint template );
private readonly Hook< CreateFileWDelegate > _createFileWHook;
/// <summary> Some storage to skip repeated allocations. </summary>
private readonly ThreadLocal< nint > _fileNameStorage = new(SetupStorage, true);
public CreateFileWHook()
=> _createFileWHook = Hook< CreateFileWDelegate >.FromImport( null, "KERNEL32.dll", "CreateFileW", 0, CreateFileWDetour );
/// <remarks> Long paths in windows need to start with "\\?\", so we keep this static in the pointers. </remarks>
private static nint SetupStorage()
{
var ptr = ( char* )Marshal.AllocHGlobal( 2 * BufferSize );
ptr[ 0 ] = '\\';
ptr[ 1 ] = '\\';
ptr[ 2 ] = '?';
ptr[ 3 ] = '\\';
ptr[ 4 ] = '\0';
return ( nint )ptr;
_createFileWHook = Hook<CreateFileWDelegate>.FromImport(null, "KERNEL32.dll", "CreateFileW", 0, CreateFileWDetour);
_createFileWHook.Enable();
}
public void Enable()
=> _createFileWHook.Enable();
public void Disable()
=> _createFileWHook.Disable();
public void Dispose()
{
Disable();
_createFileWHook.Dispose();
foreach( var ptr in _fileNameStorage.Values )
{
Marshal.FreeHGlobal( ptr );
}
}
private nint CreateFileWDetour( char* fileName, uint access, uint shareMode, nint security, uint creation, uint flags, nint template )
{
// Translate data if prefix fits.
if( CheckPtr( fileName, out var name ) )
{
// Use static storage.
var ptr = WriteFileName( name );
Penumbra.Log.Verbose( $"[ResourceHooks] Calling CreateFileWDetour with {ByteString.FromSpanUnsafe( name, false )}." );
return _createFileWHook.OriginalDisposeSafe( ptr, access, shareMode, security, creation, flags, template );
}
return _createFileWHook.OriginalDisposeSafe( fileName, access, shareMode, security, creation, flags, template );
}
/// <remarks>Write the UTF8-encoded byte string as UTF16 into the static buffers,
/// replacing any forward-slashes with back-slashes and adding a terminating null-wchar_t.</remarks>
private char* WriteFileName( ReadOnlySpan< byte > actualName )
{
var span = new Span< char >( ( char* )_fileNameStorage.Value + 4, BufferSize - 4 );
var written = Encoding.UTF8.GetChars( actualName, span );
for( var i = 0; i < written; ++i )
{
if( span[ i ] == '/' )
{
span[ i ] = '\\';
}
}
span[ written ] = '\0';
return ( char* )_fileNameStorage.Value;
}
/// <summary>
/// Write the data read specifically in the CreateFileW hook to a buffer array.
/// </summary>
/// <param name="buffer">The buffer the data is written to.</param>
/// <param name="address">The pointer to the UTF8 string containing the path.</param>
/// <param name="length">The length of the path in bytes.</param>
public static void WritePtr(char* buffer, byte* address, int length)
{
// Set the prefix, which is not valid for any actual path.
@ -129,6 +64,70 @@ public unsafe class CreateFileWHook : IDisposable
ptr[RequiredSize - 1] = 0;
}
public void Dispose()
{
_createFileWHook.Disable();
_createFileWHook.Dispose();
foreach (var ptr in _fileNameStorage.Values)
Marshal.FreeHGlobal(ptr);
}
/// <remarks> Long paths in windows need to start with "\\?\", so we keep this static in the pointers. </remarks>
private static nint SetupStorage()
{
var ptr = (char*)Marshal.AllocHGlobal(2 * BufferSize);
ptr[0] = '\\';
ptr[1] = '\\';
ptr[2] = '?';
ptr[3] = '\\';
ptr[4] = '\0';
return (nint)ptr;
}
// The prefix is not valid for any actual path, so should never run into false-positives.
private const char Prefix = (char)((byte)'P' | (('?' & 0x00FF) << 8));
private const int BufferSize = Utf8GamePath.MaxGamePathLength;
private delegate nint CreateFileWDelegate(char* fileName, uint access, uint shareMode, nint security, uint creation, uint flags,
nint template);
private readonly Hook<CreateFileWDelegate> _createFileWHook;
/// <summary> Some storage to skip repeated allocations. </summary>
private readonly ThreadLocal<nint> _fileNameStorage = new(SetupStorage, true);
private nint CreateFileWDetour(char* fileName, uint access, uint shareMode, nint security, uint creation, uint flags, nint template)
{
// Translate data if prefix fits.
if (CheckPtr(fileName, out var name))
{
// Use static storage.
var ptr = WriteFileName(name);
Penumbra.Log.Verbose($"[ResourceHooks] Calling CreateFileWDetour with {ByteString.FromSpanUnsafe(name, false)}.");
return _createFileWHook.OriginalDisposeSafe(ptr, access, shareMode, security, creation, flags, template);
}
return _createFileWHook.OriginalDisposeSafe(fileName, access, shareMode, security, creation, flags, template);
}
/// <remarks>Write the UTF8-encoded byte string as UTF16 into the static buffers,
/// replacing any forward-slashes with back-slashes and adding a terminating null-wchar_t.</remarks>
private char* WriteFileName(ReadOnlySpan<byte> actualName)
{
var span = new Span<char>((char*)_fileNameStorage.Value + 4, BufferSize - 4);
var written = Encoding.UTF8.GetChars(actualName, span);
for (var i = 0; i < written; ++i)
{
if (span[i] == '/')
span[i] = '\\';
}
span[written] = '\0';
return (char*)_fileNameStorage.Value;
}
private static bool CheckPtr(char* buffer, out ReadOnlySpan<byte> fileName)
{
if (buffer[0] is not Prefix)

50
Penumbra/Interop/Loader/FileReadHooks.cs
View file

@ -1,50 +0,0 @@
using System;
using Dalamud.Hooking;
using Dalamud.Utility.Signatures;
using FFXIVClientStructs.FFXIV.Client.System.Resource;
using Penumbra.GameData;
using Penumbra.Interop.Structs;
namespace Penumbra.Interop.Loader;
public unsafe class FileReadHooks : IDisposable
{
private delegate byte ReadSqPackPrototype(ResourceManager* resourceManager, SeFileDescriptor* pFileDesc, int priority, bool isSync);
[Signature(Sigs.ReadSqPack, DetourName = nameof(ReadSqPackDetour))]
private readonly Hook<ReadSqPackPrototype> _readSqPackHook = null!;
public FileReadHooks()
{
SignatureHelper.Initialise(this);
_readSqPackHook.Enable();
}
/// <summary> Invoked when a file is supposed to be read from SqPack. </summary>
/// <param name="fileDescriptor">The file descriptor containing what file to read.</param>
/// <param name="priority">The games priority. Should not generally be changed.</param>
/// <param name="isSync">Whether the file needs to be loaded synchronously. Should not generally be changed.</param>
/// <param name="callOriginal">Whether to call the original function after the event is finished.</param>
public delegate void ReadSqPackDelegate(ref SeFileDescriptor fileDescriptor, ref int priority, ref bool isSync, ref bool callOriginal);
/// <summary>
/// <inheritdoc cref="ReadSqPackDelegate"/> <para/>
/// Subscribers should be exception-safe.
/// </summary>
public event ReadSqPackDelegate? ReadSqPack;
private byte ReadSqPackDetour(ResourceManager* resourceManager, SeFileDescriptor* fileDescriptor, int priority, bool isSync)
{
var callOriginal = true;
ReadSqPack?.Invoke(ref *fileDescriptor, ref priority, ref isSync, ref callOriginal);
return callOriginal
? _readSqPackHook.Original(resourceManager, fileDescriptor, priority, isSync)
: (byte)1;
}
public void Dispose()
{
_readSqPackHook.Disable();
_readSqPackHook.Dispose();
}
}

90
Penumbra/Interop/Loader/FileReadService.cs Normal file
View file

@ -0,0 +1,90 @@
using System;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Dalamud.Hooking;
using Dalamud.Utility.Signatures;
using Penumbra.GameData;
using Penumbra.Interop.Structs;
using Penumbra.Util;
namespace Penumbra.Interop.Loader;
public unsafe class FileReadService : IDisposable
{
public FileReadService(PerformanceTracker performance, ResourceManagerService resourceManager)
{
_resourceManager = resourceManager;
_performance = performance;
SignatureHelper.Initialise(this);
_readSqPackHook.Enable();
}
/// <summary> Invoked when a file is supposed to be read from SqPack. </summary>
/// <param name="fileDescriptor">The file descriptor containing what file to read.</param>
/// <param name="priority">The games priority. Should not generally be changed.</param>
/// <param name="isSync">Whether the file needs to be loaded synchronously. Should not generally be changed.</param>
/// <param name="returnValue">The return value. If this is set, original will not be called.</param>
public delegate void ReadSqPackDelegate(SeFileDescriptor* fileDescriptor, ref int priority, ref bool isSync, ref byte? returnValue);
/// <summary>
/// <inheritdoc cref="ReadSqPackDelegate"/> <para/>
/// Subscribers should be exception-safe.
/// </summary>
public event ReadSqPackDelegate? ReadSqPack;
/// <summary>
/// Use the games ReadFile function to read a file from the hard drive instead of an SqPack.
/// </summary>
/// <param name="fileDescriptor">The file to load.</param>
/// <param name="priority">The games priority.</param>
/// <param name="isSync">Whether the file needs to be loaded synchronously.</param>
/// <returns>Unknown, not directly success/failure.</returns>
public byte ReadFile(SeFileDescriptor* fileDescriptor, int priority, bool isSync)
=> _readFile.Invoke(GetResourceManager(), fileDescriptor, priority, isSync);
public byte ReadDefaultSqPack(SeFileDescriptor* fileDescriptor, int priority, bool isSync)
=> _readSqPackHook.Original(GetResourceManager(), fileDescriptor, priority, isSync);
public void Dispose()
{
_readSqPackHook.Dispose();
}
private readonly PerformanceTracker _performance;
private readonly ResourceManagerService _resourceManager;
private delegate byte ReadSqPackPrototype(nint resourceManager, SeFileDescriptor* pFileDesc, int priority, bool isSync);
[Signature(Sigs.ReadSqPack, DetourName = nameof(ReadSqPackDetour))]
private readonly Hook<ReadSqPackPrototype> _readSqPackHook = null!;
private byte ReadSqPackDetour(nint resourceManager, SeFileDescriptor* fileDescriptor, int priority, bool isSync)
{
using var performance = _performance.Measure(PerformanceType.ReadSqPack);
byte? ret = null;
_lastFileThreadResourceManager.Value = resourceManager;
ReadSqPack?.Invoke(fileDescriptor, ref priority, ref isSync, ref ret);
_lastFileThreadResourceManager.Value = IntPtr.Zero;
return ret ?? _readSqPackHook.Original(resourceManager, fileDescriptor, priority, isSync);
}
private delegate byte ReadFileDelegate(nint resourceManager, SeFileDescriptor* fileDescriptor, int priority,
bool isSync);
/// We need to use the ReadFile function to load local, uncompressed files instead of loading them from the SqPacks.
[Signature(Sigs.ReadFile)]
private readonly ReadFileDelegate _readFile = null!;
private readonly ThreadLocal<nint> _lastFileThreadResourceManager = new(true);
/// <summary>
/// Usually files are loaded using the resource manager as a first pointer, but it seems some rare cases are using something else.
/// So we keep track of them per thread and use them.
/// </summary>
private nint GetResourceManager()
=> !_lastFileThreadResourceManager.IsValueCreated || _lastFileThreadResourceManager.Value == IntPtr.Zero
? (nint) _resourceManager.ResourceManager
: _lastFileThreadResourceManager.Value;
}

251
Penumbra/Interop/Loader/ResourceLoader.Debug.cs
View file

@ -1,251 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Dalamud.Hooking;
using Dalamud.Utility.Signatures;
using FFXIVClientStructs.FFXIV.Client.System.Resource;
using FFXIVClientStructs.FFXIV.Client.System.Resource.Handle;
using FFXIVClientStructs.Interop;
using FFXIVClientStructs.STD;
using Penumbra.Collections;
using Penumbra.GameData;
using Penumbra.GameData.Enums;
using Penumbra.String.Classes;
using Penumbra.Util;
namespace Penumbra.Interop.Loader;
public unsafe partial class ResourceLoader
{
// If in debug mode, this logs any resource at refcount 0 that gets decremented again, and skips the decrement instead.
private delegate byte ResourceHandleDecRef( ResourceHandle* handle );
private readonly Hook< ResourceHandleDecRef > _decRefHook;
public delegate IntPtr ResourceHandleDestructor( ResourceHandle* handle );
[Signature( Sigs.ResourceHandleDestructor, DetourName = nameof( ResourceHandleDestructorDetour ) )]
public static Hook< ResourceHandleDestructor >? ResourceHandleDestructorHook;
private IntPtr ResourceHandleDestructorDetour( ResourceHandle* handle )
{
if( handle != null )
{
Penumbra.Log.Information( $"[ResourceLoader] Destructing Resource Handle {handle->FileName} at 0x{( ulong )handle:X} (Refcount {handle->RefCount})." );
}
return ResourceHandleDestructorHook!.Original( handle );
}
// A static pointer to the SE Resource Manager
[Signature( Sigs.ResourceManager, ScanType = ScanType.StaticAddress)]
public static ResourceManager** ResourceManager;
// Gather some debugging data about penumbra-loaded objects.
public struct DebugData
{
public Structs.ResourceHandle* OriginalResource;
public Structs.ResourceHandle* ManipulatedResource;
public Utf8GamePath OriginalPath;
public FullPath ManipulatedPath;
public ResourceCategory Category;
public ResolveData ResolverInfo;
public ResourceType Extension;
}
private readonly SortedList< FullPath, DebugData > _debugList = new();
public IReadOnlyDictionary< FullPath, DebugData > DebugList
=> _debugList;
public void EnableDebug()
{
_decRefHook.Enable();
ResourceLoaded += AddModifiedDebugInfo;
}
public void DisableDebug()
{
_decRefHook.Disable();
ResourceLoaded -= AddModifiedDebugInfo;
}
private void AddModifiedDebugInfo( Structs.ResourceHandle* handle, Utf8GamePath originalPath, FullPath? manipulatedPath,
ResolveData resolverInfo )
{
using var performance = Penumbra.Performance.Measure( PerformanceType.DebugTimes );
if( manipulatedPath == null || manipulatedPath.Value.Crc64 == 0 )
{
return;
}
// Got some incomprehensible null-dereference exceptions here when hot-reloading penumbra.
try
{
var crc = ( uint )originalPath.Path.Crc32;
var originalResource = FindResource( handle->Category, handle->FileType, crc );
_debugList[ manipulatedPath.Value ] = new DebugData()
{
OriginalResource = ( Structs.ResourceHandle* )originalResource,
ManipulatedResource = handle,
Category = handle->Category,
Extension = handle->FileType,
OriginalPath = originalPath.Clone(),
ManipulatedPath = manipulatedPath.Value,
ResolverInfo = resolverInfo,
};
}
catch( Exception e )
{
Penumbra.Log.Error( e.ToString() );
}
}
// Find a key in a StdMap.
private static TValue* FindInMap< TKey, TValue >( StdMap< TKey, TValue >* map, in TKey key )
where TKey : unmanaged, IComparable< TKey >
where TValue : unmanaged
{
if( map == null || map->Count == 0 )
{
return null;
}
var node = map->Head->Parent;
while( !node->IsNil )
{
switch( key.CompareTo( node->KeyValuePair.Item1 ) )
{
case 0: return &node->KeyValuePair.Item2;
case < 0:
node = node->Left;
break;
default:
node = node->Right;
break;
}
}
return null;
}
// Iterate in tree-order through a map, applying action to each KeyValuePair.
private static void IterateMap< TKey, TValue >( StdMap< TKey, TValue >* map, Action< TKey, TValue > action )
where TKey : unmanaged
where TValue : unmanaged
{
if( map == null || map->Count == 0 )
{
return;
}
for( var node = map->SmallestValue; !node->IsNil; node = node->Next() )
{
action( node->KeyValuePair.Item1, node->KeyValuePair.Item2 );
}
}
// Find a resource in the resource manager by its category, extension and crc-hash
public static ResourceHandle* FindResource( ResourceCategory cat, ResourceType ext, uint crc32 )
{
ref var manager = ref *ResourceManager;
var catIdx = ( uint )cat >> 0x18;
cat = ( ResourceCategory )( ushort )cat;
ref var category = ref manager->ResourceGraph->ContainerArraySpan[(int) cat];
var extMap = FindInMap( category.CategoryMapsSpan[ (int) catIdx ].Value, ( uint )ext );
if( extMap == null )
{
return null;
}
var ret = FindInMap( extMap->Value, crc32 );
return ret == null ? null : ret->Value;
}
public delegate void ExtMapAction( ResourceCategory category, StdMap< uint, Pointer< StdMap< uint, Pointer< ResourceHandle > > > >* graph,
int idx );
public delegate void ResourceMapAction( uint ext, StdMap< uint, Pointer< ResourceHandle > >* graph );
public delegate void ResourceAction( uint crc32, ResourceHandle* graph );
// Iteration functions through the resource manager.
public static void IterateGraphs( ExtMapAction action )
{
ref var manager = ref *ResourceManager;
foreach( var resourceType in Enum.GetValues< ResourceCategory >().SkipLast( 1 ) )
{
ref var graph = ref manager->ResourceGraph->ContainerArraySpan[(int) resourceType];
for( var i = 0; i < 20; ++i )
{
var map = graph.CategoryMapsSpan[i];
if( map.Value != null )
{
action( resourceType, map, i );
}
}
}
}
public static void IterateExtMap( StdMap< uint, Pointer< StdMap< uint, Pointer< ResourceHandle > > > >* map, ResourceMapAction action )
=> IterateMap( map, ( ext, m ) => action( ext, m.Value ) );
public static void IterateResourceMap( StdMap< uint, Pointer< ResourceHandle > >* map, ResourceAction action )
=> IterateMap( map, ( crc, r ) => action( crc, r.Value ) );
public static void IterateResources( ResourceAction action )
{
IterateGraphs( ( _, extMap, _ )
=> IterateExtMap( extMap, ( _, resourceMap )
=> IterateResourceMap( resourceMap, action ) ) );
}
// Update the list of currently replaced resources.
// Only used when the Replaced Resources Tab in the Debug tab is open.
public void UpdateDebugInfo()
{
using var performance = Penumbra.Performance.Measure( PerformanceType.DebugTimes );
for( var i = 0; i < _debugList.Count; ++i )
{
var data = _debugList.Values[ i ];
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if( data.OriginalPath.Path == null )
{
_debugList.RemoveAt( i-- );
continue;
}
var regularResource = FindResource( data.Category, data.Extension, ( uint )data.OriginalPath.Path.Crc32 );
var modifiedResource = FindResource( data.Category, data.Extension, ( uint )data.ManipulatedPath.InternalName.Crc32 );
if( modifiedResource == null )
{
_debugList.RemoveAt( i-- );
}
else if( regularResource != data.OriginalResource || modifiedResource != data.ManipulatedResource )
{
_debugList[ _debugList.Keys[ i ] ] = data with
{
OriginalResource = ( Structs.ResourceHandle* )regularResource,
ManipulatedResource = ( Structs.ResourceHandle* )modifiedResource,
};
}
}
}
// Prevent resource management weirdness.
private byte ResourceHandleDecRefDetour( ResourceHandle* handle )
{
if( handle == null )
{
return 0;
}
if( handle->RefCount != 0 )
{
return _decRefHook.Original( handle );
}
Penumbra.Log.Error( $"Caught decrease of Reference Counter for {handle->FileName} at 0x{( ulong )handle:X} below 0." );
return 1;
}
}

301
Penumbra/Interop/Loader/ResourceLoader.Replacement.cs
View file

@ -1,301 +0,0 @@
using Dalamud.Hooking;
using Dalamud.Utility.Signatures;
using FFXIVClientStructs.FFXIV.Client.System.Resource;
using Penumbra.Collections;
using Penumbra.GameData;
using Penumbra.GameData.Enums;
using Penumbra.Interop.Structs;
using Penumbra.String;
using Penumbra.String.Classes;
using Penumbra.Util;
using System;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using static Penumbra.Interop.Loader.ResourceLoader;
using FileMode = Penumbra.Interop.Structs.FileMode;
using ResourceHandle = FFXIVClientStructs.FFXIV.Client.System.Resource.Handle.ResourceHandle;
namespace Penumbra.Interop.Loader;
public unsafe class FileReadHooks : IDisposable
{
private delegate byte ReadSqPackPrototype(ResourceManager* resourceManager, SeFileDescriptor* pFileDesc, int priority, bool isSync);
[Signature(Sigs.ReadSqPack, DetourName = nameof(ReadSqPackDetour))]
private readonly Hook<ReadSqPackPrototype> _readSqPackHook = null!;
public FileReadHooks()
{
SignatureHelper.Initialise(this);
_readSqPackHook.Enable();
}
public delegate void ReadSqPackDelegate(ref SeFileDescriptor fileDescriptor, ref int priority, ref bool isSync, ref bool callOriginal);
public event ReadSqPackDelegate? ReadSqPack;
private byte ReadSqPackDetour(ResourceManager* resourceManager, SeFileDescriptor* fileDescriptor, int priority, bool isSync)
{
var callOriginal = true;
ReadSqPack?.Invoke(ref *fileDescriptor, ref priority, ref isSync, ref callOriginal);
return callOriginal
? _readSqPackHook.Original(resourceManager, fileDescriptor, priority, isSync)
: (byte)1;
}
public void Dispose()
{
_readSqPackHook.Disable();
_readSqPackHook.Dispose();
}
}
public unsafe partial class ResourceLoader
{
[Conditional("DEBUG")]
private static void CompareHash(int local, int game, Utf8GamePath path)
{
if (local != game)
Penumbra.Log.Warning($"Hash function appears to have changed. Computed {local:X8} vs Game {game:X8} for {path}.");
}
private event Action<Utf8GamePath, ResourceType, FullPath?, object?>? PathResolved;
public ResourceHandle* ResolvePathSync(ResourceCategory category, ResourceType type, ByteString path)
{
var hash = path.Crc32;
return GetResourceHandler(true, *ResourceManager, &category, &type, &hash, path.Path, null, false);
}
private ResourceHandle* GetResourceHandler(bool isSync, ResourceManager* resourceManager, ResourceCategory* categoryId,
ResourceType* resourceType, int* resourceHash, byte* path, GetResourceParameters* pGetResParams, bool isUnk)
{
using var performance = Penumbra.Performance.Measure(PerformanceType.GetResourceHandler);
ResourceHandle* ret;
if (!Utf8GamePath.FromPointer(path, out var gamePath))
{
Penumbra.Log.Error("Could not create GamePath from resource path.");
return CallOriginalHandler(isSync, resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams, isUnk);
}
CompareHash(ComputeHash(gamePath.Path, pGetResParams), *resourceHash, gamePath);
ResourceRequested?.Invoke(gamePath, isSync);
// If no replacements are being made, we still want to be able to trigger the event.
var (resolvedPath, data) = ResolvePath(gamePath, *categoryId, *resourceType, *resourceHash);
PathResolved?.Invoke(gamePath, *resourceType, resolvedPath ?? (gamePath.IsRooted() ? new FullPath(gamePath) : null), data);
if (resolvedPath == null)
{
ret = CallOriginalHandler(isSync, resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams, isUnk);
ResourceLoaded?.Invoke((Structs.ResourceHandle*)ret, gamePath, null, data);
return ret;
}
// Replace the hash and path with the correct one for the replacement.
*resourceHash = ComputeHash(resolvedPath.Value.InternalName, pGetResParams);
path = resolvedPath.Value.InternalName.Path;
ret = CallOriginalHandler(isSync, resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams, isUnk);
ResourceLoaded?.Invoke((Structs.ResourceHandle*)ret, gamePath, resolvedPath.Value, data);
return ret;
}
// Use the default method of path replacement.
public static (FullPath?, ResolveData) DefaultResolver(Utf8GamePath path)
{
var resolved = Penumbra.CollectionManager.Default.ResolvePath(path);
return (resolved, Penumbra.CollectionManager.Default.ToResolveData());
}
// Try all resolve path subscribers or use the default replacer.
private (FullPath?, ResolveData) ResolvePath(Utf8GamePath path, ResourceCategory category, ResourceType resourceType, int resourceHash)
{
if (!DoReplacements || _incMode.Value)
return (null, ResolveData.Invalid);
path = path.ToLower();
switch (category)
{
// Only Interface collection.
case ResourceCategory.Ui:
{
var resolved = Penumbra.CollectionManager.Interface.ResolvePath(path);
return (resolved, Penumbra.CollectionManager.Interface.ToResolveData());
}
// Never allow changing scripts.
case ResourceCategory.UiScript:
case ResourceCategory.GameScript:
return (null, ResolveData.Invalid);
// Use actual resolving.
case ResourceCategory.Chara:
case ResourceCategory.Shader:
case ResourceCategory.Vfx:
case ResourceCategory.Sound:
if (ResolvePathCustomization != null)
foreach (var resolver in ResolvePathCustomization.GetInvocationList())
{
if (((ResolvePathDelegate)resolver).Invoke(path, category, resourceType, resourceHash, out var ret))
return ret;
}
break;
// None of these files are ever associated with specific characters,
// always use the default resolver for now.
case ResourceCategory.Common:
case ResourceCategory.BgCommon:
case ResourceCategory.Bg:
case ResourceCategory.Cut:
case ResourceCategory.Exd:
case ResourceCategory.Music:
default:
break;
}
return DefaultResolver(path);
}
// We need to use the ReadFile function to load local, uncompressed files instead of loading them from the SqPacks.
public delegate byte ReadFileDelegate(ResourceManager* resourceManager, SeFileDescriptor* fileDescriptor, int priority,
bool isSync);
[Signature(Sigs.ReadFile)]
public readonly ReadFileDelegate ReadFile = null!;
// We hook ReadSqPack to redirect rooted files to ReadFile.
public delegate byte ReadSqPackPrototype(ResourceManager* resourceManager, SeFileDescriptor* pFileDesc, int priority, bool isSync);
[Signature(Sigs.ReadSqPack, DetourName = nameof(ReadSqPackDetour))]
public readonly Hook<ReadSqPackPrototype> ReadSqPackHook = null!;
private byte ReadSqPackDetour(ResourceManager* resourceManager, SeFileDescriptor* fileDescriptor, int priority, bool isSync)
{
using var performance = Penumbra.Performance.Measure(PerformanceType.ReadSqPack);
if (!DoReplacements)
return ReadSqPackHook.Original(resourceManager, fileDescriptor, priority, isSync);
if (fileDescriptor == null || fileDescriptor->ResourceHandle == null)
{
Penumbra.Log.Error("Failure to load file from SqPack: invalid File Descriptor.");
return ReadSqPackHook.Original(resourceManager, fileDescriptor, priority, isSync);
}
if (!fileDescriptor->ResourceHandle->GamePath(out var gamePath) || gamePath.Length == 0)
return ReadSqPackHook.Original(resourceManager, fileDescriptor, priority, isSync);
// Paths starting with a '|' are handled separately to allow for special treatment.
// They are expected to also have a closing '|'.
if (ResourceLoadCustomization == null || gamePath.Path[0] != (byte)'|')
return DefaultLoadResource(gamePath.Path, resourceManager, fileDescriptor, priority, isSync);
// Split the path into the special-treatment part (between the first and second '|')
// and the actual path.
byte ret = 0;
var split = gamePath.Path.Split((byte)'|', 3, false);
fileDescriptor->ResourceHandle->FileNameData = split[2].Path;
fileDescriptor->ResourceHandle->FileNameLength = split[2].Length;
var funcFound = fileDescriptor->ResourceHandle->Category != ResourceCategory.Ui
&& ResourceLoadCustomization.GetInvocationList()
.Any(f => ((ResourceLoadCustomizationDelegate)f)
.Invoke(split[1], split[2], resourceManager, fileDescriptor, priority, isSync, out ret));
if (!funcFound)
ret = DefaultLoadResource(split[2], resourceManager, fileDescriptor, priority, isSync);
// Return original resource handle path so that they can be loaded separately.
fileDescriptor->ResourceHandle->FileNameData = gamePath.Path.Path;
fileDescriptor->ResourceHandle->FileNameLength = gamePath.Path.Length;
return ret;
}
// Load the resource from an SqPack and trigger the FileLoaded event.
private byte DefaultResourceLoad(ByteString path, ResourceManager* resourceManager,
SeFileDescriptor* fileDescriptor, int priority, bool isSync)
{
var ret = Penumbra.ResourceLoader.ReadSqPackHook.Original(resourceManager, fileDescriptor, priority, isSync);
FileLoaded?.Invoke(fileDescriptor->ResourceHandle, path, ret != 0, false);
return ret;
}
/// <summary> Load the resource from a path on the users hard drives. </summary>
/// <remarks> <see cref="CreateFileWHook" /> </remarks>
private byte DefaultRootedResourceLoad(ByteString gamePath, ResourceManager* resourceManager,
SeFileDescriptor* fileDescriptor, int priority, bool isSync)
{
// Specify that we are loading unpacked files from the drive.
// We need to copy the actual file path in UTF16 (Windows-Unicode) on two locations.
fileDescriptor->FileMode = FileMode.LoadUnpackedResource;
// Ensure that the file descriptor has its wchar_t array on aligned boundary even if it has to be odd.
var fd = stackalloc char[0x11 + 0x0B + 14];
fileDescriptor->FileDescriptor = (byte*)fd + 1;
CreateFileWHook.WritePtr(fd + 0x11, gamePath.Path, gamePath.Length);
CreateFileWHook.WritePtr(&fileDescriptor->Utf16FileName, gamePath.Path, gamePath.Length);
// Use the SE ReadFile function.
var ret = ReadFile(resourceManager, fileDescriptor, priority, isSync);
FileLoaded?.Invoke(fileDescriptor->ResourceHandle, gamePath, ret != 0, true);
return ret;
}
// Load a resource by its path. If it is rooted, it will be loaded from the drive, otherwise from the SqPack.
internal byte DefaultLoadResource(ByteString gamePath, ResourceManager* resourceManager, SeFileDescriptor* fileDescriptor, int priority,
bool isSync)
=> Utf8GamePath.IsRooted(gamePath)
? DefaultRootedResourceLoad(gamePath, resourceManager, fileDescriptor, priority, isSync)
: DefaultResourceLoad(gamePath, resourceManager, fileDescriptor, priority, isSync);
private void DisposeHooks()
{
DisableHooks();
_createFileWHook.Dispose();
ReadSqPackHook.Dispose();
GetResourceSyncHook.Dispose();
GetResourceAsyncHook.Dispose();
ResourceHandleDestructorHook?.Dispose();
_incRefHook.Dispose();
}
private static int ComputeHash(ByteString path, GetResourceParameters* pGetResParams)
{
if (pGetResParams == null || !pGetResParams->IsPartialRead)
return path.Crc32;
// When the game requests file only partially, crc32 includes that information, in format of:
// path/to/file.ext.hex_offset.hex_size
// ex) music/ex4/BGM_EX4_System_Title.scd.381adc.30000
return ByteString.Join(
(byte)'.',
path,
ByteString.FromStringUnsafe(pGetResParams->SegmentOffset.ToString("x"), true),
ByteString.FromStringUnsafe(pGetResParams->SegmentLength.ToString("x"), true)
).Crc32;
}
// A resource with ref count 0 that gets incremented goes through GetResourceAsync again.
// This means, that if the path determined from that is different than the resources path,
// a different resource gets loaded or incremented, while the IncRef'd resource stays at 0.
// This causes some problems and is hopefully prevented with this.
private readonly ThreadLocal<bool> _incMode = new();
private readonly Hook<ResourceHandleDestructor> _incRefHook;
private IntPtr ResourceHandleIncRefDetour(ResourceHandle* handle)
{
if (handle->RefCount > 0)
return _incRefHook.Original(handle);
_incMode.Value = true;
var ret = _incRefHook.Original(handle);
_incMode.Value = false;
return ret;
}
}

104
Penumbra/Interop/Loader/ResourceLoader.TexMdl.cs
View file

@ -1,104 +0,0 @@
using System;
using System.Collections.Generic;
using Dalamud.Hooking;
using Dalamud.Utility.Signatures;
using FFXIVClientStructs.FFXIV.Client.System.Resource.Handle;
using Penumbra.GameData;
using Penumbra.GameData.Enums;
using Penumbra.String.Classes;
namespace Penumbra.Interop.Loader;
// Since 6.0, Mdl and Tex Files require special treatment, probably due to datamining protection.
public unsafe partial class ResourceLoader
{
// Custom ulong flag to signal our files as opposed to SE files.
public static readonly IntPtr CustomFileFlag = new(0xDEADBEEF);
// We need to keep a list of all CRC64 hash values of our replaced Mdl and Tex files,
// i.e. CRC32 of filename in the lower bytes, CRC32 of parent path in the upper bytes.
private readonly HashSet< ulong > _customFileCrc = new();
public IReadOnlySet< ulong > CustomFileCrc
=> _customFileCrc;
// The function that checks a files CRC64 to determine whether it is 'protected'.
// We use it to check against our stored CRC64s and if it corresponds, we return the custom flag.
public delegate IntPtr CheckFileStatePrototype( IntPtr unk1, ulong crc64 );
[Signature( Sigs.CheckFileState, DetourName = nameof( CheckFileStateDetour ) )]
public readonly Hook< CheckFileStatePrototype > CheckFileStateHook = null!;
private IntPtr CheckFileStateDetour( IntPtr ptr, ulong crc64 )
=> _customFileCrc.Contains( crc64 ) ? CustomFileFlag : CheckFileStateHook.Original( ptr, crc64 );
// We use the local functions for our own files in the extern hook.
public delegate byte LoadTexFileLocalDelegate( ResourceHandle* handle, int unk1, IntPtr unk2, bool unk3 );
[Signature( Sigs.LoadTexFileLocal )]
public readonly LoadTexFileLocalDelegate LoadTexFileLocal = null!;
public delegate byte LoadMdlFileLocalPrototype( ResourceHandle* handle, IntPtr unk1, bool unk2 );
[Signature( Sigs.LoadMdlFileLocal )]
public readonly LoadMdlFileLocalPrototype LoadMdlFileLocal = null!;
// We hook the extern functions to just return the local one if given the custom flag as last argument.
public delegate byte LoadTexFileExternPrototype( ResourceHandle* handle, int unk1, IntPtr unk2, bool unk3, IntPtr unk4 );
[Signature( Sigs.LoadTexFileExtern, DetourName = nameof( LoadTexFileExternDetour ) )]
public readonly Hook< LoadTexFileExternPrototype > LoadTexFileExternHook = null!;
private byte LoadTexFileExternDetour( ResourceHandle* resourceHandle, int unk1, IntPtr unk2, bool unk3, IntPtr ptr )
=> ptr.Equals( CustomFileFlag )
? LoadTexFileLocal.Invoke( resourceHandle, unk1, unk2, unk3 )
: LoadTexFileExternHook.Original( resourceHandle, unk1, unk2, unk3, ptr );
public delegate byte LoadMdlFileExternPrototype( ResourceHandle* handle, IntPtr unk1, bool unk2, IntPtr unk3 );
[Signature( Sigs.LoadMdlFileExtern, DetourName = nameof( LoadMdlFileExternDetour ) )]
public readonly Hook< LoadMdlFileExternPrototype > LoadMdlFileExternHook = null!;
private byte LoadMdlFileExternDetour( ResourceHandle* resourceHandle, IntPtr unk1, bool unk2, IntPtr ptr )
=> ptr.Equals( CustomFileFlag )
? LoadMdlFileLocal.Invoke( resourceHandle, unk1, unk2 )
: LoadMdlFileExternHook.Original( resourceHandle, unk1, unk2, ptr );
private void AddCrc( Utf8GamePath _, ResourceType type, FullPath? path, object? _2 )
{
if( path.HasValue && type is ResourceType.Mdl or ResourceType.Tex )
{
_customFileCrc.Add( path.Value.Crc64 );
}
}
private void EnableTexMdlTreatment()
{
PathResolved += AddCrc;
CheckFileStateHook.Enable();
LoadTexFileExternHook.Enable();
LoadMdlFileExternHook.Enable();
}
private void DisableTexMdlTreatment()
{
PathResolved -= AddCrc;
_customFileCrc.Clear();
_customFileCrc.TrimExcess();
CheckFileStateHook.Disable();
LoadTexFileExternHook.Disable();
LoadMdlFileExternHook.Disable();
}
private void DisposeTexMdlTreatment()
{
CheckFileStateHook.Dispose();
LoadTexFileExternHook.Dispose();
LoadMdlFileExternHook.Dispose();
}
}

293
Penumbra/Interop/Loader/ResourceLoader.cs
View file

@ -1,6 +1,6 @@
using System;
using Dalamud.Hooking;
using Dalamud.Utility.Signatures;
using System.Diagnostics;
using System.Threading;
using FFXIVClientStructs.FFXIV.Client.System.Resource;
using Penumbra.Collections;
using Penumbra.GameData.Enums;
@ -10,114 +10,219 @@ using Penumbra.String.Classes;
namespace Penumbra.Interop.Loader;
public unsafe partial class ResourceLoader : IDisposable
public unsafe class ResourceLoader : IDisposable
{
// Toggle whether replacing paths is active, independently of hook and event state.
public bool DoReplacements { get; private set; }
private readonly ResourceService _resources;
private readonly FileReadService _fileReadService;
private readonly TexMdlService _texMdlService;
// Hooks are required for everything, even events firing.
public bool HooksEnabled { get; private set; }
public ResourceLoader(ResourceService resources, FileReadService fileReadService, TexMdlService texMdlService,
CreateFileWHook _)
{
_resources = resources;
_fileReadService = fileReadService;
_texMdlService = texMdlService;
ResetResolvePath();
public void EnableReplacements()
{
if( DoReplacements )
{
return;
_resources.ResourceRequested += ResourceHandler;
_resources.ResourceHandleIncRef += IncRefProtection;
_resources.ResourceHandleDecRef += DecRefProtection;
_fileReadService.ReadSqPack += ReadSqPackDetour;
}
DoReplacements = true;
EnableTexMdlTreatment();
EnableHooks();
}
/// <summary> The function to use to resolve a given path. </summary>
public Func<Utf8GamePath, ResourceCategory, ResourceType, (FullPath?, ResolveData)> ResolvePath = null!;
public void DisableReplacements()
{
if( !DoReplacements )
{
return;
}
/// <summary> Reset the ResolvePath function to always return null. </summary>
public void ResetResolvePath()
=> ResolvePath = (_1, _2, _3) => (null, ResolveData.Invalid);
DoReplacements = false;
DisableTexMdlTreatment();
}
public void EnableHooks()
{
if( HooksEnabled )
{
return;
}
HooksEnabled = true;
_createFileWHook.Enable();
ReadSqPackHook.Enable();
GetResourceSyncHook.Enable();
GetResourceAsyncHook.Enable();
_incRefHook.Enable();
}
public void DisableHooks()
{
if( !HooksEnabled )
{
return;
}
HooksEnabled = false;
_createFileWHook.Disable();
ReadSqPackHook.Disable();
GetResourceSyncHook.Disable();
GetResourceAsyncHook.Disable();
_incRefHook.Disable();
}
public ResourceLoader( Penumbra _ )
{
SignatureHelper.Initialise( this );
_decRefHook = Hook< ResourceHandleDecRef >.FromAddress(
( IntPtr )FFXIVClientStructs.FFXIV.Client.System.Resource.Handle.ResourceHandle.MemberFunctionPointers.DecRef,
ResourceHandleDecRefDetour );
_incRefHook = Hook< ResourceHandleDestructor >.FromAddress(
( IntPtr )FFXIVClientStructs.FFXIV.Client.System.Resource.Handle.ResourceHandle.MemberFunctionPointers.IncRef, ResourceHandleIncRefDetour );
}
// Event fired whenever a resource is requested.
public delegate void ResourceRequestedDelegate( Utf8GamePath path, bool synchronous );
public event ResourceRequestedDelegate? ResourceRequested;
// Event fired whenever a resource is returned.
// If the path was manipulated by penumbra, manipulatedPath will be the file path of the loaded resource.
// resolveData is additional data returned by the current ResolvePath function which can contain the collection and associated game object.
public delegate void ResourceLoadedDelegate(ResourceHandle* handle, Utf8GamePath originalPath, FullPath? manipulatedPath,
ResolveData resolveData);
/// <summary>
/// Event fired whenever a resource is returned.
/// If the path was manipulated by penumbra, manipulatedPath will be the file path of the loaded resource.
/// resolveData is additional data returned by the current ResolvePath function which can contain the collection and associated game object.
/// </summary>
public event ResourceLoadedDelegate? ResourceLoaded;
public delegate void FileLoadedDelegate(ResourceHandle* resource, ByteString path, bool returnValue, bool custom,
ByteString additionalData);
// Event fired whenever a resource is newly loaded.
// Success indicates the return value of the loading function (which does not imply that the resource was actually successfully loaded)
// custom is true if the file was loaded from local files instead of the default SqPacks.
public delegate void FileLoadedDelegate( ResourceHandle* resource, ByteString path, bool success, bool custom );
/// <summary>
/// Event fired whenever a resource is newly loaded.
/// ReturnValue indicates the return value of the loading function (which does not imply that the resource was actually successfully loaded)
/// custom is true if the file was loaded from local files instead of the default SqPacks.
/// AdditionalData is either empty or the part of the path inside the leading pipes.
/// </summary>
public event FileLoadedDelegate? FileLoaded;
// Customization point to control how path resolving is handled.
// Resolving goes through all subscribed functions in arbitrary order until one returns true,
// or uses default resolving if none return true.
public delegate bool ResolvePathDelegate( Utf8GamePath path, ResourceCategory category, ResourceType type, int hash,
out (FullPath?, ResolveData) ret );
public event ResolvePathDelegate? ResolvePathCustomization;
// Customize file loading for any GamePaths that start with "|".
// Same procedure as above.
public delegate bool ResourceLoadCustomizationDelegate( ByteString split, ByteString path, ResourceManager* resourceManager,
SeFileDescriptor* fileDescriptor, int priority, bool isSync, out byte retValue );
public event ResourceLoadCustomizationDelegate? ResourceLoadCustomization;
public void Dispose()
{
DisposeHooks();
DisposeTexMdlTreatment();
_resources.ResourceRequested -= ResourceHandler;
_resources.ResourceHandleIncRef -= IncRefProtection;
_resources.ResourceHandleDecRef -= DecRefProtection;
_fileReadService.ReadSqPack -= ReadSqPackDetour;
}
private void ResourceHandler(ref ResourceCategory category, ref ResourceType type, ref int hash, ref Utf8GamePath path,
GetResourceParameters* parameters, ref bool sync, ref ResourceHandle* returnValue)
{
if (returnValue != null)
return;
CompareHash(ComputeHash(path.Path, parameters), hash, path);
// If no replacements are being made, we still want to be able to trigger the event.
var (resolvedPath, data) = _incMode.Value ? (null, ResolveData.Invalid) : ResolvePath(path, category, type);
if (resolvedPath == null || !Utf8GamePath.FromByteString(resolvedPath.Value.InternalName, out var p))
{
returnValue = _resources.GetOriginalResource(sync, category, type, hash, path.Path, parameters);
ResourceLoaded?.Invoke(returnValue, path, resolvedPath, data);
return;
}
_texMdlService.AddCrc(type, resolvedPath);
// Replace the hash and path with the correct one for the replacement.
hash = ComputeHash(resolvedPath.Value.InternalName, parameters);
path = p;
returnValue = _resources.GetOriginalResource(sync, category, type, hash, path.Path, parameters);
ResourceLoaded?.Invoke(returnValue, p, resolvedPath.Value, data);
}
private void ReadSqPackDetour(SeFileDescriptor* fileDescriptor, ref int priority, ref bool isSync, ref byte? returnValue)
{
if (fileDescriptor->ResourceHandle == null)
{
Penumbra.Log.Error("[ResourceLoader] Failure to load file from SqPack: invalid File Descriptor.");
return;
}
if (!fileDescriptor->ResourceHandle->GamePath(out var gamePath) || gamePath.Length == 0)
{
Penumbra.Log.Error("[ResourceLoader] Failure to load file from SqPack: invalid path specified.");
return;
}
// Paths starting with a '|' are handled separately to allow for special treatment.
// They are expected to also have a closing '|'.
if (gamePath.Path[0] != (byte)'|')
{
returnValue = DefaultLoadResource(gamePath.Path, fileDescriptor, priority, isSync, ByteString.Empty);
return;
}
// Split the path into the special-treatment part (between the first and second '|')
// and the actual path.
var split = gamePath.Path.Split((byte)'|', 3, false);
fileDescriptor->ResourceHandle->FileNameData = split[2].Path;
fileDescriptor->ResourceHandle->FileNameLength = split[2].Length;
MtrlForceSync(fileDescriptor, ref isSync);
returnValue = DefaultLoadResource(split[2], fileDescriptor, priority, isSync, split[1]);
// Return original resource handle path so that they can be loaded separately.
fileDescriptor->ResourceHandle->FileNameData = gamePath.Path.Path;
fileDescriptor->ResourceHandle->FileNameLength = gamePath.Path.Length;
}
/// <summary> Load a resource by its path. If it is rooted, it will be loaded from the drive, otherwise from the SqPack. </summary>
private byte DefaultLoadResource(ByteString gamePath, SeFileDescriptor* fileDescriptor, int priority,
bool isSync, ByteString additionalData)
{
if (Utf8GamePath.IsRooted(gamePath))
{
// Specify that we are loading unpacked files from the drive.
// We need to obtain the actual file path in UTF16 (Windows-Unicode) on two locations,
// but we write a pointer to the given string instead and use the CreateFileW hook to handle it,
// because otherwise we are limited to 260 characters.
fileDescriptor->FileMode = FileMode.LoadUnpackedResource;
// Ensure that the file descriptor has its wchar_t array on aligned boundary even if it has to be odd.
var fd = stackalloc char[0x11 + 0x0B + 14];
fileDescriptor->FileDescriptor = (byte*)fd + 1;
CreateFileWHook.WritePtr(fd + 0x11, gamePath.Path, gamePath.Length);
CreateFileWHook.WritePtr(&fileDescriptor->Utf16FileName, gamePath.Path, gamePath.Length);
// Use the SE ReadFile function.
var ret = _fileReadService.ReadFile(fileDescriptor, priority, isSync);
FileLoaded?.Invoke(fileDescriptor->ResourceHandle, gamePath, ret != 0, true, additionalData);
return ret;
}
else
{
var ret = _fileReadService.ReadDefaultSqPack(fileDescriptor, priority, isSync);
FileLoaded?.Invoke(fileDescriptor->ResourceHandle, gamePath, ret != 0, false, additionalData);
return ret;
}
}
/// <summary> Special handling for materials. </summary>
private static void MtrlForceSync(SeFileDescriptor* fileDescriptor, ref bool isSync)
{
// Force isSync = true for Materials. I don't really understand why,
// or where the difference even comes from.
// Was called with True on my client and with false on other peoples clients,
// which caused problems.
isSync |= fileDescriptor->ResourceHandle->FileType is ResourceType.Mtrl;
}
/// <summary>
/// A resource with ref count 0 that gets incremented goes through GetResourceAsync again.
/// This means, that if the path determined from that is different than the resources path,
/// a different resource gets loaded or incremented, while the IncRef'd resource stays at 0.
/// This causes some problems and is hopefully prevented with this.
/// </summary>
private readonly ThreadLocal<bool> _incMode = new(() => false, true);
/// <inheritdoc cref="_incMode"/>
private void IncRefProtection(ResourceHandle* handle, ref nint? returnValue)
{
if (handle->RefCount != 0)
return;
_incMode.Value = true;
returnValue = _resources.IncRef(handle);
_incMode.Value = false;
}
/// <summary>
/// Catch weird errors with invalid decrements of the reference count.
/// </summary>
private void DecRefProtection(ResourceHandle* handle, ref byte? returnValue)
{
if (handle->RefCount != 0)
return;
Penumbra.Log.Error(
$"[ResourceLoader] Caught decrease of Reference Counter for {handle->FileName()} at 0x{(ulong)handle} below 0.");
returnValue = 1;
}
/// <summary> Compute the CRC32 hash for a given path together with potential resource parameters. </summary>
private static int ComputeHash(ByteString path, GetResourceParameters* pGetResParams)
{
if (pGetResParams == null || !pGetResParams->IsPartialRead)
return path.Crc32;
// When the game requests file only partially, crc32 includes that information, in format of:
// path/to/file.ext.hex_offset.hex_size
// ex) music/ex4/BGM_EX4_System_Title.scd.381adc.30000
return ByteString.Join(
(byte)'.',
path,
ByteString.FromStringUnsafe(pGetResParams->SegmentOffset.ToString("x"), true),
ByteString.FromStringUnsafe(pGetResParams->SegmentLength.ToString("x"), true)
).Crc32;
}
/// <summary>
/// In Debug build, compare the hashes the game computes with those Penumbra computes to notice potential changes in the CRC32 algorithm or resource parameters.
/// </summary>
[Conditional("DEBUG")]
private static void CompareHash(int local, int game, Utf8GamePath path)
{
if (local != game)
Penumbra.Log.Warning($"[ResourceLoader] Hash function appears to have changed. Computed {local:X8} vs Game {game:X8} for {path}.");
}
}

114
Penumbra/Interop/Loader/ResourceManagerService.cs Normal file
View file

@ -0,0 +1,114 @@
using System;
using System.Linq;
using Dalamud.Utility.Signatures;
using FFXIVClientStructs.FFXIV.Client.System.Resource;
using FFXIVClientStructs.FFXIV.Client.System.Resource.Handle;
using FFXIVClientStructs.Interop;
using FFXIVClientStructs.STD;
using Penumbra.GameData;
using Penumbra.GameData.Enums;
namespace Penumbra.Interop.Loader;
public unsafe class ResourceManagerService
{
public ResourceManagerService()
=> SignatureHelper.Initialise(this);
/// <summary> The SE Resource Manager as pointer. </summary>
public ResourceManager* ResourceManager
=> *ResourceManagerAddress;
/// <summary> Find a resource in the resource manager by its category, extension and crc-hash. </summary>
public ResourceHandle* FindResource(ResourceCategory cat, ResourceType ext, uint crc32)
{
ref var manager = ref *ResourceManager;
var catIdx = (uint)cat >> 0x18;
cat = (ResourceCategory)(ushort)cat;
ref var category = ref manager.ResourceGraph->ContainerArraySpan[(int)cat];
var extMap = FindInMap(category.CategoryMapsSpan[(int)catIdx].Value, (uint)ext);
if (extMap == null)
return null;
var ret = FindInMap(extMap->Value, crc32);
return ret == null ? null : ret->Value;
}
public delegate void ExtMapAction(ResourceCategory category, StdMap<uint, Pointer<StdMap<uint, Pointer<ResourceHandle>>>>* graph, int idx);
public delegate void ResourceMapAction(uint ext, StdMap<uint, Pointer<ResourceHandle>>* graph);
public delegate void ResourceAction(uint crc32, ResourceHandle* graph);
/// <summary> Iterate through the entire graph calling an action on every ExtMap. </summary>
public void IterateGraphs(ExtMapAction action)
{
ref var manager = ref *ResourceManager;
foreach (var resourceType in Enum.GetValues<ResourceCategory>().SkipLast(1))
{
ref var graph = ref manager.ResourceGraph->ContainerArraySpan[(int)resourceType];
for (var i = 0; i < 20; ++i)
{
var map = graph.CategoryMapsSpan[i];
if (map.Value != null)
action(resourceType, map, i);
}
}
}
/// <summary> Iterate through a specific ExtMap calling an action on every resource map. </summary>
public void IterateExtMap(StdMap<uint, Pointer<StdMap<uint, Pointer<ResourceHandle>>>>* map, ResourceMapAction action)
=> IterateMap(map, (ext, m) => action(ext, m.Value));
/// <summary> Iterate through a specific resource map calling an action on every resource. </summary>
public void IterateResourceMap(StdMap<uint, Pointer<ResourceHandle>>* map, ResourceAction action)
=> IterateMap(map, (crc, r) => action(crc, r.Value));
/// <summary> Iterate through the entire graph calling an action on every resource. </summary>
public void IterateResources(ResourceAction action)
{
IterateGraphs((_, extMap, _)
=> IterateExtMap(extMap, (_, resourceMap)
=> IterateResourceMap(resourceMap, action)));
}
/// <summary> A static pointer to the SE Resource Manager. </summary>
[Signature(Sigs.ResourceManager, ScanType = ScanType.StaticAddress)]
internal readonly ResourceManager** ResourceManagerAddress = null;
// Find a key in a StdMap.
private static TValue* FindInMap<TKey, TValue>(StdMap<TKey, TValue>* map, in TKey key)
where TKey : unmanaged, IComparable<TKey>
where TValue : unmanaged
{
if (map == null || map->Count == 0)
return null;
var node = map->Head->Parent;
while (!node->IsNil)
{
switch (key.CompareTo(node->KeyValuePair.Item1))
{
case 0: return &node->KeyValuePair.Item2;
case < 0:
node = node->Left;
break;
default:
node = node->Right;
break;
}
}
return null;
}
// Iterate in tree-order through a map, applying action to each KeyValuePair.
private static void IterateMap<TKey, TValue>(StdMap<TKey, TValue>* map, Action<TKey, TValue> action)
where TKey : unmanaged
where TValue : unmanaged
{
if (map == null || map->Count == 0)
return;
for (var node = map->SmallestValue; !node->IsNil; node = node->Next())
action(node->KeyValuePair.Item1, node->KeyValuePair.Item2);
}
}

140
Penumbra/Interop/Loader/ResourceHook.cs → Penumbra/Interop/Loader/ResourceService.cs
View file

@ -6,24 +6,48 @@ using Penumbra.GameData;
using Penumbra.GameData.Enums;
using Penumbra.Interop.Structs;
using Penumbra.String;
using ResourceHandle = FFXIVClientStructs.FFXIV.Client.System.Resource.Handle.ResourceHandle;
using Penumbra.String.Classes;
using Penumbra.Util;
namespace Penumbra.Interop.Loader;
public unsafe class ResourceHook : IDisposable
public unsafe class ResourceService : IDisposable
{
public ResourceHook()
private readonly PerformanceTracker _performance;
private readonly ResourceManagerService _resourceManager;
public ResourceService(PerformanceTracker performance, ResourceManagerService resourceManager)
{
_performance = performance;
_resourceManager = resourceManager;
SignatureHelper.Initialise(this);
_getResourceSyncHook.Enable();
_getResourceAsyncHook.Enable();
_resourceHandleDestructorHook.Enable();
_incRefHook = Hook<ResourceHandlePrototype>.FromAddress(
(nint)FFXIVClientStructs.FFXIV.Client.System.Resource.Handle.ResourceHandle.MemberFunctionPointers.IncRef,
ResourceHandleIncRefDetour);
_incRefHook.Enable();
_decRefHook = Hook<ResourceHandleDecRefPrototype>.FromAddress(
(nint)FFXIVClientStructs.FFXIV.Client.System.Resource.Handle.ResourceHandle.MemberFunctionPointers.DecRef,
ResourceHandleDecRefDetour);
_decRefHook.Enable();
}
public ResourceHandle* GetResource(ResourceCategory category, ResourceType type, ByteString path)
{
var hash = path.Crc32;
return GetResourceHandler(true, (ResourceManager*)_resourceManager.ResourceManagerAddress,
&category, &type, &hash, path.Path, null, false);
}
public void Dispose()
{
_getResourceSyncHook.Dispose();
_getResourceAsyncHook.Dispose();
_resourceHandleDestructorHook.Dispose();
_incRefHook.Dispose();
_decRefHook.Dispose();
}
#region GetResource
@ -33,24 +57,15 @@ public unsafe class ResourceHook : IDisposable
/// <param name="type">The resource type. Should not generally be changed.</param>
/// <param name="hash">The resource hash. Should generally fit to the path.</param>
/// <param name="path">The path of the requested resource.</param>
/// <param name="parameters">Mainly used for SCD streaming.</param>
/// <param name="parameters">Mainly used for SCD streaming, can be null.</param>
/// <param name="sync">Whether to request the resource synchronously or asynchronously.</param>
public delegate void GetResourcePreDelegate(ref ResourceCategory category, ref ResourceType type, ref int hash, ref ByteString path,
ref GetResourceParameters parameters, ref bool sync);
/// <param name="returnValue">The returned resource handle. If this is not null, calling original will be skipped. </param>
public delegate void GetResourcePreDelegate(ref ResourceCategory category, ref ResourceType type, ref int hash, ref Utf8GamePath path,
GetResourceParameters* parameters, ref bool sync, ref ResourceHandle* returnValue);
/// <summary> <inheritdoc cref="GetResourcePreDelegate"/> <para/>
/// Subscribers should be exception-safe.</summary>
public event GetResourcePreDelegate? GetResourcePre;
/// <summary>
/// The returned resource handle obtained from a resource request. Contains all the other information from the request.
/// </summary>
public delegate void GetResourcePostDelegate(ref ResourceHandle handle);
/// <summary> <inheritdoc cref="GetResourcePostDelegate"/> <para/>
/// Subscribers should be exception-safe.</summary>
public event GetResourcePostDelegate? GetResourcePost;
public event GetResourcePreDelegate? ResourceRequested;
private delegate ResourceHandle* GetResourceSyncPrototype(ResourceManager* resourceManager, ResourceCategory* pCategoryId,
ResourceType* pResourceType, int* pResourceHash, byte* pPath, GetResourceParameters* pGetResParams);
@ -79,15 +94,34 @@ public unsafe class ResourceHook : IDisposable
private ResourceHandle* GetResourceHandler(bool isSync, ResourceManager* resourceManager, ResourceCategory* categoryId,
ResourceType* resourceType, int* resourceHash, byte* path, GetResourceParameters* pGetResParams, bool isUnk)
{
var byteString = new ByteString(path);
GetResourcePre?.Invoke(ref *categoryId, ref *resourceType, ref *resourceHash, ref byteString, ref *pGetResParams, ref isSync);
var ret = isSync
? _getResourceSyncHook.Original(resourceManager, categoryId, resourceType, resourceHash, byteString.Path, pGetResParams)
: _getResourceAsyncHook.Original(resourceManager, categoryId, resourceType, resourceHash, byteString.Path, pGetResParams, isUnk);
GetResourcePost?.Invoke(ref *ret);
return ret;
using var performance = _performance.Measure(PerformanceType.GetResourceHandler);
if (!Utf8GamePath.FromPointer(path, out var gamePath))
{
Penumbra.Log.Error("[ResourceService] Could not create GamePath from resource path.");
return isSync
? _getResourceSyncHook.Original(resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams)
: _getResourceAsyncHook.Original(resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams, isUnk);
}
ResourceHandle* returnValue = null;
ResourceRequested?.Invoke(ref *categoryId, ref *resourceType, ref *resourceHash, ref gamePath, pGetResParams, ref isSync,
ref returnValue);
if (returnValue != null)
return returnValue;
return GetOriginalResource(isSync, *categoryId, *resourceType, *resourceHash, gamePath.Path, pGetResParams, isUnk);
}
/// <summary> Call the original GetResource function. </summary>
public ResourceHandle* GetOriginalResource(bool sync, ResourceCategory categoryId, ResourceType type, int hash, ByteString path,
GetResourceParameters* resourceParameters = null, bool unk = false)
=> sync
? _getResourceSyncHook.OriginalDisposeSafe(_resourceManager.ResourceManager, &categoryId, &type, &hash, path.Path,
resourceParameters)
: _getResourceAsyncHook.OriginalDisposeSafe(_resourceManager.ResourceManager, &categoryId, &type, &hash, path.Path,
resourceParameters,
unk);
#endregion
private delegate IntPtr ResourceHandlePrototype(ResourceHandle* handle);
@ -96,9 +130,8 @@ public unsafe class ResourceHook : IDisposable
/// <summary> Invoked before a resource handle reference count is incremented. </summary>
/// <param name="handle">The resource handle.</param>
/// <param name="callOriginal">Whether to call original after the event has run.</param>
/// <param name="returnValue">The return value to use if not calling original.</param>
public delegate void ResourceHandleIncRefDelegate(ref ResourceHandle handle, ref bool callOriginal, ref nint returnValue);
/// <param name="returnValue">The return value to use, setting this value will skip calling original.</param>
public delegate void ResourceHandleIncRefDelegate(ResourceHandle* handle, ref nint? returnValue);
/// <summary>
/// <inheritdoc cref="ResourceHandleIncRefDelegate"/> <para/>
@ -106,21 +139,19 @@ public unsafe class ResourceHook : IDisposable
/// </summary>
public event ResourceHandleIncRefDelegate? ResourceHandleIncRef;
public nint IncRef(ref ResourceHandle handle)
{
fixed (ResourceHandle* ptr = &handle)
{
return _incRefHook.Original(ptr);
}
}
/// <summary>
/// Call the game function that increases the reference counter of a resource handle.
/// </summary>
public nint IncRef(ResourceHandle* handle)
=> _incRefHook.OriginalDisposeSafe(handle);
private readonly Hook<ResourceHandlePrototype> _incRefHook;
private nint ResourceHandleIncRefDetour(ResourceHandle* handle)
{
var callOriginal = true;
var ret = IntPtr.Zero;
ResourceHandleIncRef?.Invoke(ref *handle, ref callOriginal, ref ret);
return callOriginal ? _incRefHook.Original(handle) : ret;
nint? ret = null;
ResourceHandleIncRef?.Invoke(handle, ref ret);
return ret ?? _incRefHook.OriginalDisposeSafe(handle);
}
#endregion
@ -129,9 +160,8 @@ public unsafe class ResourceHook : IDisposable
/// <summary> Invoked before a resource handle reference count is decremented. </summary>
/// <param name="handle">The resource handle.</param>
/// <param name="callOriginal">Whether to call original after the event has run.</param>
/// <param name="returnValue">The return value to use if not calling original.</param>
public delegate void ResourceHandleDecRefDelegate(ref ResourceHandle handle, ref bool callOriginal, ref byte returnValue);
/// <param name="returnValue">The return value to use, setting this value will skip calling original.</param>
public delegate void ResourceHandleDecRefDelegate(ResourceHandle* handle, ref byte? returnValue);
/// <summary>
/// <inheritdoc cref="ResourceHandleDecRefDelegate"/> <para/>
@ -139,29 +169,29 @@ public unsafe class ResourceHook : IDisposable
/// </summary>
public event ResourceHandleDecRefDelegate? ResourceHandleDecRef;
public byte DecRef(ref ResourceHandle handle)
{
fixed (ResourceHandle* ptr = &handle)
{
return _incRefHook.Original(ptr);
}
}
/// <summary>
/// Call the original game function that decreases the reference counter of a resource handle.
/// </summary>
public byte DecRef(ResourceHandle* handle)
=> _decRefHook.OriginalDisposeSafe(handle);
private delegate byte ResourceHandleDecRefPrototype(ResourceHandle* handle);
private readonly Hook<ResourceHandleDecRefPrototype> _decRefHook;
private byte ResourceHandleDecRefDetour(ResourceHandle* handle)
{
var callOriginal = true;
var ret = byte.MinValue;
ResourceHandleDecRef?.Invoke(ref *handle, ref callOriginal, ref ret);
return callOriginal ? _decRefHook!.Original(handle) : ret;
byte? ret = null;
ResourceHandleDecRef?.Invoke(handle, ref ret);
return ret ?? _decRefHook.OriginalDisposeSafe(handle);
}
#endregion
#region Destructor
/// <summary> Invoked before a resource handle is destructed. </summary>
/// <param name="handle">The resource handle.</param>
public delegate void ResourceHandleDtorDelegate(ref ResourceHandle handle);
public delegate void ResourceHandleDtorDelegate(ResourceHandle* handle);
/// <summary>
/// <inheritdoc cref="ResourceHandleDtorDelegate"/> <para/>
@ -174,8 +204,8 @@ public unsafe class ResourceHook : IDisposable
private nint ResourceHandleDestructorDetour(ResourceHandle* handle)
{
ResourceHandleDestructor?.Invoke(ref *handle);
return _resourceHandleDestructorHook!.Original(handle);
ResourceHandleDestructor?.Invoke(handle);
return _resourceHandleDestructorHook.OriginalDisposeSafe(handle);
}
#endregion

100
Penumbra/Interop/Loader/TexMdlService.cs Normal file
View file

@ -0,0 +1,100 @@
using System;
using System.Collections.Generic;
using Dalamud.Hooking;
using Dalamud.Utility.Signatures;
using FFXIVClientStructs.FFXIV.Client.System.Resource.Handle;
using Penumbra.GameData;
using Penumbra.GameData.Enums;
using Penumbra.String.Classes;
namespace Penumbra.Interop.Loader;
public unsafe class TexMdlService
{
/// <summary> Custom ulong flag to signal our files as opposed to SE files. </summary>
public static readonly IntPtr CustomFileFlag = new(0xDEADBEEF);
/// <summary>
/// We need to keep a list of all CRC64 hash values of our replaced Mdl and Tex files,
/// i.e. CRC32 of filename in the lower bytes, CRC32 of parent path in the upper bytes.
/// </summary>
public IReadOnlySet<ulong> CustomFileCrc
=> _customFileCrc;
public TexMdlService()
{
SignatureHelper.Initialise(this);
_checkFileStateHook.Enable();
_loadTexFileExternHook.Enable();
_loadMdlFileExternHook.Enable();
}
/// <summary> Add CRC64 if the given file is a model or texture file and has an associated path. </summary>
public void AddCrc(ResourceType type, FullPath? path)
{
if (path.HasValue && type is ResourceType.Mdl or ResourceType.Tex)
_customFileCrc.Add(path.Value.Crc64);
}
/// <summary> Add a fixed CRC64 value. </summary>
public void AddCrc(ulong crc64)
=> _customFileCrc.Add(crc64);
public void Dispose()
{
_checkFileStateHook.Dispose();
_loadTexFileExternHook.Dispose();
_loadMdlFileExternHook.Dispose();
}
private readonly HashSet<ulong> _customFileCrc = new();
private delegate IntPtr CheckFileStatePrototype(IntPtr unk1, ulong crc64);
[Signature(Sigs.CheckFileState, DetourName = nameof(CheckFileStateDetour))]
private readonly Hook<CheckFileStatePrototype> _checkFileStateHook = null!;
/// <summary>
/// The function that checks a files CRC64 to determine whether it is 'protected'.
/// We use it to check against our stored CRC64s and if it corresponds, we return the custom flag.
/// </summary>
private IntPtr CheckFileStateDetour(IntPtr ptr, ulong crc64)
=> _customFileCrc.Contains(crc64) ? CustomFileFlag : _checkFileStateHook.Original(ptr, crc64);
private delegate byte LoadTexFileLocalDelegate(ResourceHandle* handle, int unk1, IntPtr unk2, bool unk3);
/// <summary> We use the local functions for our own files in the extern hook. </summary>
[Signature(Sigs.LoadTexFileLocal)]
private readonly LoadTexFileLocalDelegate _loadTexFileLocal = null!;
private delegate byte LoadMdlFileLocalPrototype(ResourceHandle* handle, IntPtr unk1, bool unk2);
/// <summary> We use the local functions for our own files in the extern hook. </summary>
[Signature(Sigs.LoadMdlFileLocal)]
private readonly LoadMdlFileLocalPrototype _loadMdlFileLocal = null!;
private delegate byte LoadTexFileExternPrototype(ResourceHandle* handle, int unk1, IntPtr unk2, bool unk3, IntPtr unk4);
[Signature(Sigs.LoadTexFileExtern, DetourName = nameof(LoadTexFileExternDetour))]
private readonly Hook<LoadTexFileExternPrototype> _loadTexFileExternHook = null!;
/// <summary> We hook the extern functions to just return the local one if given the custom flag as last argument. </summary>
private byte LoadTexFileExternDetour(ResourceHandle* resourceHandle, int unk1, IntPtr unk2, bool unk3, IntPtr ptr)
=> ptr.Equals(CustomFileFlag)
? _loadTexFileLocal.Invoke(resourceHandle, unk1, unk2, unk3)
: _loadTexFileExternHook.Original(resourceHandle, unk1, unk2, unk3, ptr);
public delegate byte LoadMdlFileExternPrototype(ResourceHandle* handle, IntPtr unk1, bool unk2, IntPtr unk3);
[Signature(Sigs.LoadMdlFileExtern, DetourName = nameof(LoadMdlFileExternDetour))]
private readonly Hook<LoadMdlFileExternPrototype> _loadMdlFileExternHook = null!;
/// <summary> We hook the extern functions to just return the local one if given the custom flag as last argument. </summary>
private byte LoadMdlFileExternDetour(ResourceHandle* resourceHandle, IntPtr unk1, bool unk2, IntPtr ptr)
=> ptr.Equals(CustomFileFlag)
? _loadMdlFileLocal.Invoke(resourceHandle, unk1, unk2)
: _loadMdlFileExternHook.Original(resourceHandle, unk1, unk2, ptr);
}

2
Penumbra/Interop/ResidentResourceManager.cs
View file

@ -18,8 +18,6 @@ public unsafe class ResidentResourceManager
[Signature( Sigs.UnloadPlayerResources )]
public readonly ResidentResourceDelegate UnloadPlayerResources = null!;
public Structs.ResidentResourceManager* Address
=> *_residentResourceManagerAddress;

2
Penumbra/Interop/Resolver/PathResolver.DrawObjectState.cs
View file

@ -144,7 +144,7 @@ public unsafe partial class PathResolver
{
_lastCreatedCollection = IdentifyCollection(LastGameObject, false);
// Change the transparent or 1.0 Decal if necessary.
var decal = new CharacterUtility.DecalReverter(_lastCreatedCollection.ModCollection, UsesDecal(a, c));
var decal = new CharacterUtility.DecalReverter(Penumbra.ResourceService, _lastCreatedCollection.ModCollection, UsesDecal(a, c));
// Change the rsp parameters.
meta = new DisposableContainer(_lastCreatedCollection.ModCollection.TemporarilySetCmpFile(), decal);
try

2
Penumbra/Interop/Resolver/PathResolver.Meta.cs
View file

@ -194,7 +194,7 @@ public unsafe partial class PathResolver
_inChangeCustomize = true;
var resolveData = GetResolveData( human );
using var cmp = resolveData.ModCollection.TemporarilySetCmpFile();
using var decals = new CharacterUtility.DecalReverter( resolveData.ModCollection, DrawObjectState.UsesDecal( 0, data ) );
using var decals = new CharacterUtility.DecalReverter( Penumbra.ResourceService, resolveData.ModCollection, DrawObjectState.UsesDecal( 0, data ) );
var ret = _changeCustomize.Original( human, data, skipEquipment );
_inChangeCustomize = false;
return ret;

39
Penumbra/Interop/Resolver/PathResolver.Subfiles.cs
View file

@ -2,6 +2,7 @@ using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
using Dalamud.Hooking;
using Dalamud.Utility.Signatures;
@ -66,7 +67,6 @@ public unsafe partial class PathResolver
out (FullPath?, ResolveData) data)
{
if (nonDefault)
{
switch (type)
{
case ResourceType.Mtrl:
@ -75,7 +75,6 @@ public unsafe partial class PathResolver
data = (fullPath, resolveData);
return;
}
}
data = (resolved, resolveData);
}
@ -85,7 +84,6 @@ public unsafe partial class PathResolver
_loadMtrlShpkHook.Enable();
_loadMtrlTexHook.Enable();
_apricotResourceLoadHook.Enable();
_loader.ResourceLoadCustomization += SubfileLoadHandler;
_loader.ResourceLoaded += SubfileContainerRequested;
_events.ResourceHandleDestructor += ResourceDestroyed;
}
@ -95,7 +93,6 @@ public unsafe partial class PathResolver
_loadMtrlShpkHook.Disable();
_loadMtrlTexHook.Disable();
_apricotResourceLoadHook.Disable();
_loader.ResourceLoadCustomization -= SubfileLoadHandler;
_loader.ResourceLoaded -= SubfileContainerRequested;
_events.ResourceHandleDestructor -= ResourceDestroyed;
}
@ -108,16 +105,15 @@ public unsafe partial class PathResolver
_apricotResourceLoadHook.Dispose();
}
private void SubfileContainerRequested( ResourceHandle* handle, Utf8GamePath originalPath, FullPath? manipulatedPath, ResolveData resolveData )
private void SubfileContainerRequested(ResourceHandle* handle, Utf8GamePath originalPath, FullPath? manipulatedPath,
ResolveData resolveData)
{
switch (handle->FileType)
{
case ResourceType.Mtrl:
case ResourceType.Avfx:
if (handle->FileSize == 0)
{
_subFileCollection[ ( IntPtr )handle ] = resolveData;
}
_subFileCollection[(nint)handle] = resolveData;
break;
}
@ -126,31 +122,6 @@ public unsafe partial class PathResolver
private void ResourceDestroyed(ResourceHandle* handle)
=> _subFileCollection.TryRemove((IntPtr)handle, out _);
// We need to set the correct collection for the actual material path that is loaded
// before actually loading the file.
public static bool SubfileLoadHandler( ByteString split, ByteString path, ResourceManager* resourceManager,
SeFileDescriptor* fileDescriptor, int priority, bool isSync, out byte ret )
{
switch( fileDescriptor->ResourceHandle->FileType )
{
case ResourceType.Mtrl:
// Force isSync = true for this call. I don't really understand why,
// or where the difference even comes from.
// Was called with True on my client and with false on other peoples clients,
// which caused problems.
ret = Penumbra.ResourceLoader.DefaultLoadResource( path, resourceManager, fileDescriptor, priority, true );
return true;
case ResourceType.Avfx:
// Do nothing special right now.
ret = Penumbra.ResourceLoader.DefaultLoadResource( path, resourceManager, fileDescriptor, priority, isSync );
return true;
default:
ret = 0;
return false;
}
}
private delegate byte LoadMtrlFilesDelegate(IntPtr mtrlResourceHandle);
[Signature(Sigs.LoadMtrlTex, DetourName = nameof(LoadMtrlTexDetour))]
@ -182,9 +153,7 @@ public unsafe partial class PathResolver
private ResolveData LoadFileHelper(IntPtr resourceHandle)
{
if (resourceHandle == IntPtr.Zero)
{
return ResolveData.Invalid;
}
return _subFileCollection.TryGetValue(resourceHandle, out var c) ? c : ResolveData.Invalid;
}

9
Penumbra/Interop/Resolver/PathResolver.cs
View file

@ -9,6 +9,7 @@ using OtterGui.Classes;
using Penumbra.Collections;
using Penumbra.GameData.Enums;
using Penumbra.Interop.Loader;
using Penumbra.Interop.Structs;
using Penumbra.Services;
using Penumbra.String;
using Penumbra.String.Classes;
@ -54,7 +55,7 @@ public partial class PathResolver : IDisposable
}
// The modified resolver that handles game path resolving.
private bool CharacterResolver(Utf8GamePath gamePath, ResourceCategory _1, ResourceType type, int _2, out (FullPath?, ResolveData) data)
public (FullPath?, ResolveData) CharacterResolver(Utf8GamePath gamePath, ResourceType type)
{
using var performance = Penumbra.Performance.Measure(PerformanceType.CharacterResolver);
// Check if the path was marked for a specific collection,
@ -77,8 +78,8 @@ public partial class PathResolver : IDisposable
// so that the functions loading tex and shpk can find that path and use its collection.
// We also need to handle defaulted materials against a non-default collection.
var path = resolved == null ? gamePath.Path : resolved.Value.InternalName;
SubfileHelper.HandleCollection(resolveData, path, nonDefault, type, resolved, out data);
return true;
SubfileHelper.HandleCollection(resolveData, path, nonDefault, type, resolved, out var pair);
return pair;
}
public void Enable()
@ -95,7 +96,6 @@ public partial class PathResolver : IDisposable
_meta.Enable();
_subFiles.Enable();
_loader.ResolvePathCustomization += CharacterResolver;
Penumbra.Log.Debug("Character Path Resolver enabled.");
}
@ -113,7 +113,6 @@ public partial class PathResolver : IDisposable
_meta.Disable();
_subFiles.Disable();
_loader.ResolvePathCustomization -= CharacterResolver;
Penumbra.Log.Debug("Character Path Resolver disabled.");
}

51
Penumbra/Meta/Manager/MetaManager.Imc.cs
View file

@ -1,12 +1,9 @@
using System;
using System.Collections.Generic;
using FFXIVClientStructs.FFXIV.Client.System.Resource;
using System.Diagnostics.CodeAnalysis;
using OtterGui.Filesystem;
using Penumbra.GameData.Enums;
using Penumbra.Interop.Structs;
using Penumbra.Meta.Files;
using Penumbra.Meta.Manipulations;
using Penumbra.String;
using Penumbra.String.Classes;
namespace Penumbra.Meta.Manager;
@ -15,7 +12,6 @@ public partial class MetaManager
{
private readonly Dictionary< Utf8GamePath, ImcFile > _imcFiles = new();
private readonly List< ImcManipulation > _imcManipulations = new();
private static int _imcManagerCount;
public void SetImcFiles()
{
@ -132,52 +128,11 @@ public partial class MetaManager
_imcFiles.Clear();
_imcManipulations.Clear();
RestoreImcDelegate();
}
private static unsafe void SetupImcDelegate()
{
if( _imcManagerCount++ == 0 )
{
Penumbra.ResourceLoader.ResourceLoadCustomization += ImcLoadHandler;
}
}
private static unsafe void RestoreImcDelegate()
{
if( --_imcManagerCount == 0 )
{
Penumbra.ResourceLoader.ResourceLoadCustomization -= ImcLoadHandler;
}
}
private FullPath CreateImcPath( Utf8GamePath path )
=> new($"|{_collection.Name}_{_collection.ChangeCounter}|{path}");
private static unsafe bool ImcLoadHandler( ByteString split, ByteString path, ResourceManager* resourceManager,
SeFileDescriptor* fileDescriptor, int priority, bool isSync, out byte ret )
{
ret = 0;
if( fileDescriptor->ResourceHandle->FileType != ResourceType.Imc )
{
return false;
}
Penumbra.Log.Verbose( $"Using ImcLoadHandler for path {path}." );
ret = Penumbra.ResourceLoader.ReadSqPackHook.Original( resourceManager, fileDescriptor, priority, isSync );
var lastUnderscore = split.LastIndexOf( ( byte )'_' );
var name = lastUnderscore == -1 ? split.ToString() : split.Substring( 0, lastUnderscore ).ToString();
if( ( Penumbra.TempCollections.CollectionByName( name, out var collection )
|| Penumbra.CollectionManager.ByName( name, out collection ) )
&& collection.HasCache
&& collection.MetaCache!._imcFiles.TryGetValue( Utf8GamePath.FromSpan( path.Span, out var p ) ? p : Utf8GamePath.Empty, out var file ) )
{
Penumbra.Log.Debug( $"Loaded {path} from file and replaced with IMC from collection {collection.AnonymizedName}." );
file.Replace( fileDescriptor->ResourceHandle );
}
return true;
}
public bool GetImcFile(Utf8GamePath path, [NotNullWhen(true)] out ImcFile? file)
=> _imcFiles.TryGetValue(path, out file);
}

1
Penumbra/Meta/Manager/MetaManager.cs
View file

@ -35,7 +35,6 @@ public partial class MetaManager : IDisposable, IEnumerable< KeyValuePair< MetaM
public MetaManager( ModCollection collection )
{
_collection = collection;
SetupImcDelegate();
if( !Penumbra.CharacterUtility.Ready )
{
Penumbra.CharacterUtility.LoadingFinished += ApplyStoredManipulations;

32
Penumbra/Penumbra.cs
View file

@ -84,9 +84,16 @@ public class Penumbra : IDalamudPlugin
private readonly PenumbraNew _tmp;
public static ItemData ItemData { get; private set; } = null!;
// TODO
public static ResourceManagerService ResourceManagerService { get; private set; } = null!;
public static CharacterResolver CharacterResolver { get; private set; } = null!;
public static ResourceService ResourceService { get; private set; } = null!;
public Penumbra(DalamudPluginInterface pluginInterface)
{
Log = PenumbraNew.Log;
try
{
_tmp = new PenumbraNew(pluginInterface);
Performance = _tmp.Services.GetRequiredService<PerformanceTracker>();
ValidityChecker = _tmp.Services.GetRequiredService<ValidityChecker>();
@ -103,12 +110,10 @@ public class Penumbra : IDalamudPlugin
ItemData = _tmp.Services.GetRequiredService<ItemService>().AwaitedService;
Dalamud = _tmp.Services.GetRequiredService<DalamudServices>();
TempMods = _tmp.Services.GetRequiredService<TempModManager>();
try
{
ResourceLoader = new ResourceLoader(this);
ResourceLoader.EnableHooks();
_resourceWatcher = new ResourceWatcher(ResourceLoader);
ResidentResources = new ResidentResourceManager();
ResidentResources = _tmp.Services.GetRequiredService<ResidentResourceManager>();
ResourceManagerService = _tmp.Services.GetRequiredService<ResourceManagerService>();
_tmp.Services.GetRequiredService<StartTracker>().Measure(StartTimeType.Mods, () =>
{
ModManager = new Mod.Manager(Config.ModDirectory);
@ -126,19 +131,20 @@ public class Penumbra : IDalamudPlugin
ModFileSystem = ModFileSystem.Load();
ObjectReloader = new ObjectReloader();
ResourceService = _tmp.Services.GetRequiredService<ResourceService>();
ResourceLoader = new ResourceLoader(ResourceService, _tmp.Services.GetRequiredService<FileReadService>(), _tmp.Services.GetRequiredService<TexMdlService>(), _tmp.Services.GetRequiredService<CreateFileWHook>());
PathResolver = new PathResolver(_tmp.Services.GetRequiredService<StartTracker>(), _tmp.Services.GetRequiredService<CommunicatorService>(), _tmp.Services.GetRequiredService<GameEventManager>(), ResourceLoader);
CharacterResolver = new CharacterResolver(Config, CollectionManager, TempCollections, ResourceLoader, PathResolver);
_resourceWatcher = new ResourceWatcher(Config, ResourceService, ResourceLoader);
SetupInterface();
if (Config.EnableMods)
{
ResourceLoader.EnableReplacements();
PathResolver.Enable();
}
if (Config.DebugMode)
ResourceLoader.EnableDebug();
using (var tApi = _tmp.Services.GetRequiredService<StartTracker>().Measure(StartTimeType.Api))
{
Api = new PenumbraApi(_tmp.Services.GetRequiredService<CommunicatorService>(), this);
@ -171,7 +177,7 @@ public class Penumbra : IDalamudPlugin
{
using var tInterface = _tmp.Services.GetRequiredService<StartTracker>().Measure(StartTimeType.Interface);
var changelog = ConfigWindow.CreateChangelog();
var cfg = new ConfigWindow(_tmp.Services.GetRequiredService<CommunicatorService>(), _tmp.Services.GetRequiredService<StartTracker>(), this, _resourceWatcher)
var cfg = new ConfigWindow(_tmp.Services.GetRequiredService<CommunicatorService>(), _tmp.Services.GetRequiredService<StartTracker>(), _tmp.Services.GetRequiredService<FontReloader>(), this, _resourceWatcher)
{
IsOpen = Config.DebugMode,
};
@ -225,7 +231,6 @@ public class Penumbra : IDalamudPlugin
Config.EnableMods = enabled;
if (enabled)
{
ResourceLoader.EnableReplacements();
PathResolver.Enable();
if (CharacterUtility.Ready)
{
@ -236,7 +241,6 @@ public class Penumbra : IDalamudPlugin
}
else
{
ResourceLoader.DisableReplacements();
PathResolver.Disable();
if (CharacterUtility.Ready)
{
@ -293,7 +297,7 @@ public class Penumbra : IDalamudPlugin
ObjectReloader?.Dispose();
ModFileSystem?.Dispose();
CollectionManager?.Dispose();
PathResolver?.Dispose();
CharacterResolver?.Dispose(); // disposes PathResolver, TODO
_resourceWatcher?.Dispose();
ResourceLoader?.Dispose();
GameEvents?.Dispose();

12
Penumbra/PenumbraNew.cs
View file

@ -8,8 +8,10 @@ using Penumbra.Collections;
using Penumbra.GameData;
using Penumbra.GameData.Data;
using Penumbra.Interop;
using Penumbra.Interop.Loader;
using Penumbra.Interop.Resolver;
using Penumbra.Services;
using Penumbra.UI.Classes;
using Penumbra.Util;
namespace Penumbra;
@ -53,8 +55,14 @@ public class PenumbraNew
.AddSingleton<FrameworkManager>()
.AddSingleton<MetaFileManager>()
.AddSingleton<CutsceneCharacters>()
.AddSingleton<CharacterUtility>();
.AddSingleton<CharacterUtility>()
.AddSingleton<ResourceManagerService>()
.AddSingleton<ResourceService>()
.AddSingleton<FileReadService>()
.AddSingleton<TexMdlService>()
.AddSingleton<CreateFileWHook>()
.AddSingleton<ResidentResourceManager>()
.AddSingleton<FontReloader>();
// Add Configuration
services.AddTransient<ConfigMigrationService>()

51
Penumbra/UI/ConfigWindow.DebugTab.cs
View file

@ -61,8 +61,6 @@ public partial class ConfigWindow
DrawDebugTabGeneral();
DrawPerformanceTab();
ImGui.NewLine();
DrawDebugTabReplacedResources();
ImGui.NewLine();
DrawPathResolverDebug();
ImGui.NewLine();
DrawActorsDebug();
@ -134,53 +132,6 @@ public partial class ConfigWindow
Penumbra.Performance.Draw( "##performance", "Enable Runtime Performance Tracking", TimingExtensions.ToName );
}
// Draw all resources currently replaced by Penumbra and (if existing) the resources they replace.
// Resources are collected by iterating through the
private static unsafe void DrawDebugTabReplacedResources()
{
if( !ImGui.CollapsingHeader( "Replaced Resources" ) )
{
return;
}
Penumbra.ResourceLoader.UpdateDebugInfo();
if( Penumbra.ResourceLoader.DebugList.Count == 0 )
{
return;
}
using var table = Table( "##ReplacedResources", 6, ImGuiTableFlags.RowBg | ImGuiTableFlags.SizingFixedFit,
-Vector2.UnitX );
if( !table )
{
return;
}
foreach( var data in Penumbra.ResourceLoader.DebugList.Values.ToArray() )
{
if( data.ManipulatedPath.Crc64 == 0 )
{
continue;
}
var refCountManip = data.ManipulatedResource == null ? 0 : data.ManipulatedResource->RefCount;
var refCountOrig = data.OriginalResource == null ? 0 : data.OriginalResource->RefCount;
ImGui.TableNextColumn();
ImGui.TextUnformatted( data.ManipulatedPath.ToString() );
ImGui.TableNextColumn();
ImGui.TextUnformatted( ( ( ulong )data.ManipulatedResource ).ToString( "X" ) );
ImGui.TableNextColumn();
ImGui.TextUnformatted( refCountManip.ToString() );
ImGui.TableNextColumn();
ImGui.TextUnformatted( data.OriginalPath.ToString() );
ImGui.TableNextColumn();
ImGui.TextUnformatted( ( ( ulong )data.OriginalResource ).ToString( "X" ) );
ImGui.TableNextColumn();
ImGui.TextUnformatted( refCountOrig.ToString() );
}
}
private static unsafe void DrawActorsDebug()
{
if( !ImGui.CollapsingHeader( "Actors" ) )
@ -635,7 +586,7 @@ public partial class ConfigWindow
return;
}
ResourceLoader.IterateResources( ( _, r ) =>
Penumbra.ResourceManagerService.IterateResources( ( _, r ) =>
{
if( r->RefCount < 10000 )
{

11
Penumbra/UI/ConfigWindow.ResourceTab.cs
View file

@ -10,7 +10,6 @@ using ImGuiNET;
using OtterGui;
using OtterGui.Raii;
using OtterGui.Widgets;
using Penumbra.Interop.Loader;
using Penumbra.Services;
using Penumbra.String.Classes;
@ -46,13 +45,13 @@ public partial class ConfigWindow
unsafe
{
ResourceLoader.IterateGraphs( DrawCategoryContainer );
Penumbra.ResourceManagerService.IterateGraphs( DrawCategoryContainer );
}
ImGui.NewLine();
unsafe
{
ImGui.TextUnformatted( $"Static Address: 0x{( ulong )ResourceLoader.ResourceManager:X} (+0x{( ulong )ResourceLoader.ResourceManager - ( ulong )DalamudServices.SigScanner.Module.BaseAddress:X})" );
ImGui.TextUnformatted( $"Actual Address: 0x{( ulong )*ResourceLoader.ResourceManager:X}" );
ImGui.TextUnformatted( $"Static Address: 0x{( ulong )Penumbra.ResourceManagerService.ResourceManagerAddress:X} (+0x{( ulong )Penumbra.ResourceManagerService.ResourceManagerAddress - ( ulong )DalamudServices.SigScanner.Module.BaseAddress:X})" );
ImGui.TextUnformatted( $"Actual Address: 0x{( ulong )Penumbra.ResourceManagerService.ResourceManager:X}" );
}
}
@ -82,7 +81,7 @@ public partial class ConfigWindow
ImGui.TableSetupColumn( "Refs", ImGuiTableColumnFlags.WidthFixed, _refsColumnWidth );
ImGui.TableHeadersRow();
ResourceLoader.IterateResourceMap( map, ( hash, r ) =>
Penumbra.ResourceManagerService.IterateResourceMap( map, ( hash, r ) =>
{
// Filter unwanted names.
if( _resourceManagerFilter.Length != 0
@ -129,7 +128,7 @@ public partial class ConfigWindow
if( tree )
{
SetTableWidths();
ResourceLoader.IterateExtMap( map, ( ext, m ) => DrawResourceMap( category, ext, m ) );
Penumbra.ResourceManagerService.IterateExtMap( map, ( ext, m ) => DrawResourceMap( category, ext, m ) );
}
}

15
Penumbra/UI/ConfigWindow.SettingsTab.Advanced.cs
View file

@ -66,15 +66,6 @@ public partial class ConfigWindow
var tmp = Penumbra.Config.DebugMode;
if( ImGui.Checkbox( "##debugMode", ref tmp ) && tmp != Penumbra.Config.DebugMode )
{
if( tmp )
{
Penumbra.ResourceLoader.EnableDebug();
}
else
{
Penumbra.ResourceLoader.DisableDebug();
}
Penumbra.Config.DebugMode = tmp;
Penumbra.Config.Save();
}
@ -95,11 +86,11 @@ public partial class ConfigWindow
+ "You usually should not need to do this." );
}
private static void DrawReloadFontsButton()
private void DrawReloadFontsButton()
{
if( ImGuiUtil.DrawDisabledButton( "Reload Fonts", Vector2.Zero, "Force the game to reload its font files.", !FontReloader.Valid ) )
if( ImGuiUtil.DrawDisabledButton( "Reload Fonts", Vector2.Zero, "Force the game to reload its font files.", !_fontReloader.Valid ) )
{
FontReloader.Reload();
_fontReloader.Reload();
}
}

10
Penumbra/UI/ConfigWindow.SettingsTab.cs
View file

@ -11,6 +11,7 @@ using ImGuiNET;
using OtterGui;
using OtterGui.Raii;
using OtterGui.Widgets;
using Penumbra.Interop;
using Penumbra.Services;
using Penumbra.UI.Classes;
@ -22,11 +23,14 @@ public partial class ConfigWindow
{
public const int RootDirectoryMaxLength = 64;
private readonly ConfigWindow _window;
private readonly FontReloader _fontReloader;
public ReadOnlySpan<byte> Label
=> "Settings"u8;
public SettingsTab( ConfigWindow window )
=> _window = window;
public SettingsTab( ConfigWindow window, FontReloader fontReloader )
{
_window = window;
_fontReloader = fontReloader;
}
public void DrawHeader()
{

5
Penumbra/UI/ConfigWindow.cs
View file

@ -7,6 +7,7 @@ using OtterGui;
using OtterGui.Raii;
using OtterGui.Widgets;
using Penumbra.Api.Enums;
using Penumbra.Interop;
using Penumbra.Mods;
using Penumbra.Services;
using Penumbra.UI.Classes;
@ -35,14 +36,14 @@ public sealed partial class ConfigWindow : Window, IDisposable
public void SelectMod(Mod mod)
=> _selector.SelectByValue(mod);
public ConfigWindow(CommunicatorService communicator, StartTracker timer, Penumbra penumbra, ResourceWatcher watcher)
public ConfigWindow(CommunicatorService communicator, StartTracker timer, FontReloader fontReloader, Penumbra penumbra, ResourceWatcher watcher)
: base(GetLabel())
{
_penumbra = penumbra;
_resourceWatcher = watcher;
ModEditPopup = new ModEditWindow(communicator);
_settingsTab = new SettingsTab(this);
_settingsTab = new SettingsTab(this, fontReloader);
_selector = new ModFileSystemSelector(communicator, _penumbra.ModFileSystem);
_modPanel = new ModPanel(this);
_modsTab = new ModsTab(_selector, _modPanel, _penumbra);

120
Penumbra/UI/ResourceWatcher/ResourceWatcher.cs
View file

@ -1,12 +1,16 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text.RegularExpressions;
using Dalamud.Interface;
using FFXIVClientStructs.FFXIV.Client.System.Resource;
using ImGuiNET;
using OtterGui.Raii;
using OtterGui.Widgets;
using Penumbra.Collections;
using Penumbra.GameData.Enums;
using Penumbra.Interop.Loader;
using Penumbra.Interop.Structs;
using Penumbra.String;
@ -19,36 +23,36 @@ public partial class ResourceWatcher : IDisposable, ITab
{
public const int DefaultMaxEntries = 1024;
private readonly Configuration _config;
private readonly ResourceService _resources;
private readonly ResourceLoader _loader;
private readonly List<Record> _records = new();
private readonly ConcurrentQueue<Record> _newRecords = new();
private readonly Table _table;
private bool _writeToLog;
private bool _isEnabled;
private string _logFilter = string.Empty;
private Regex? _logRegex;
private int _maxEntries;
private int _newMaxEntries;
public unsafe ResourceWatcher( ResourceLoader loader )
public unsafe ResourceWatcher(Configuration config, ResourceService resources, ResourceLoader loader)
{
_config = config;
_resources = resources;
_loader = loader;
_table = new Table(_records);
_loader.ResourceRequested += OnResourceRequested;
_resources.ResourceRequested += OnResourceRequested;
_resources.ResourceHandleDestructor += OnResourceDestroyed;
_loader.ResourceLoaded += OnResourceLoaded;
_loader.FileLoaded += OnFileLoaded;
UpdateFilter( Penumbra.Config.ResourceLoggingFilter, false );
_writeToLog = Penumbra.Config.EnableResourceLogging;
_isEnabled = Penumbra.Config.EnableResourceWatcher;
_maxEntries = Penumbra.Config.MaxResourceWatcherRecords;
_newMaxEntries = _maxEntries;
UpdateFilter(_config.ResourceLoggingFilter, false);
_newMaxEntries = _config.MaxResourceWatcherRecords;
}
public unsafe void Dispose()
{
Clear();
_records.TrimExcess();
_loader.ResourceRequested -= OnResourceRequested;
_resources.ResourceRequested -= OnResourceRequested;
_resources.ResourceHandleDestructor -= OnResourceDestroyed;
_loader.ResourceLoaded -= OnResourceLoaded;
_loader.FileLoaded -= OnFileLoaded;
}
@ -68,9 +72,10 @@ public partial class ResourceWatcher : IDisposable, ITab
UpdateRecords();
ImGui.SetCursorPosY(ImGui.GetCursorPosY() + ImGui.GetTextLineHeightWithSpacing() / 2);
if( ImGui.Checkbox( "Enable", ref _isEnabled ) )
var isEnabled = _config.EnableResourceWatcher;
if (ImGui.Checkbox("Enable", ref isEnabled))
{
Penumbra.Config.EnableResourceWatcher = _isEnabled;
Penumbra.Config.EnableResourceWatcher = isEnabled;
Penumbra.Config.Save();
}
@ -78,14 +83,21 @@ public partial class ResourceWatcher : IDisposable, ITab
DrawMaxEntries();
ImGui.SameLine();
if (ImGui.Button("Clear"))
{
Clear();
ImGui.SameLine();
var onlyMatching = _config.OnlyAddMatchingResources;
if (ImGui.Checkbox("Store Only Matching", ref onlyMatching))
{
Penumbra.Config.OnlyAddMatchingResources = onlyMatching;
Penumbra.Config.Save();
}
ImGui.SameLine();
if( ImGui.Checkbox( "Write to Log", ref _writeToLog ) )
var writeToLog = _config.EnableResourceLogging;
if (ImGui.Checkbox("Write to Log", ref writeToLog))
{
Penumbra.Config.EnableResourceLogging = _writeToLog;
Penumbra.Config.EnableResourceLogging = writeToLog;
Penumbra.Config.Save();
}
@ -105,17 +117,13 @@ public partial class ResourceWatcher : IDisposable, ITab
using var color = ImRaii.PushColor(ImGuiCol.Border, Colors.RegexWarningBorder, invalidRegex);
using var style = ImRaii.PushStyle(ImGuiStyleVar.FrameBorderSize, ImGuiHelpers.GlobalScale, invalidRegex);
if (ImGui.InputTextWithHint("##logFilter", "If path matches this Regex...", ref tmp, 256))
{
UpdateFilter(tmp, true);
}
}
private void UpdateFilter(string newString, bool config)
{
if (newString == _logFilter)
{
return;
}
_logFilter = newString;
try
@ -152,23 +160,20 @@ public partial class ResourceWatcher : IDisposable, ITab
_newMaxEntries = DefaultMaxEntries;
}
if( _maxEntries != DefaultMaxEntries && ImGui.IsItemHovered() )
{
var maxEntries = _config.MaxResourceWatcherRecords;
if (maxEntries != DefaultMaxEntries && ImGui.IsItemHovered())
ImGui.SetTooltip($"CTRL + Right-Click to reset to default {DefaultMaxEntries}.");
}
if (!change)
{
return;
}
_newMaxEntries = Math.Max(16, _newMaxEntries);
if( _newMaxEntries != _maxEntries )
if (_newMaxEntries != maxEntries)
{
_maxEntries = _newMaxEntries;
Penumbra.Config.MaxResourceWatcherRecords = _maxEntries;
_config.MaxResourceWatcherRecords = _newMaxEntries;
Penumbra.Config.Save();
_records.RemoveRange( 0, _records.Count - _maxEntries );
if (_newMaxEntries > _records.Count)
_records.RemoveRange(0, _records.Count - _newMaxEntries);
}
}
@ -178,43 +183,38 @@ public partial class ResourceWatcher : IDisposable, ITab
if (count > 0)
{
while (_newRecords.TryDequeue(out var rec) && count-- > 0)
{
_records.Add(rec);
}
if( _records.Count > _maxEntries )
{
_records.RemoveRange( 0, _records.Count - _maxEntries );
}
if (_records.Count > _config.MaxResourceWatcherRecords)
_records.RemoveRange(0, _records.Count - _config.MaxResourceWatcherRecords);
_table.Reset();
}
}
private void OnResourceRequested( Utf8GamePath data, bool synchronous )
private unsafe void OnResourceRequested(ref ResourceCategory category, ref ResourceType type, ref int hash, ref Utf8GamePath path,
GetResourceParameters* parameters, ref bool sync, ref ResourceHandle* returnValue)
{
if( _writeToLog && FilterMatch( data.Path, out var match ) )
{
Penumbra.Log.Information( $"[ResourceLoader] [REQ] {match} was requested {( synchronous ? "synchronously." : "asynchronously." )}" );
}
if (_config.EnableResourceLogging && FilterMatch(path.Path, out var match))
Penumbra.Log.Information($"[ResourceLoader] [REQ] {match} was requested {(sync ? "synchronously." : "asynchronously.")}");
if( _isEnabled )
if (_config.EnableResourceWatcher)
{
_newRecords.Enqueue( Record.CreateRequest( data.Path, synchronous ) );
var record = Record.CreateRequest(path.Path, sync);
if (!_config.OnlyAddMatchingResources || _table.WouldBeVisible(record))
_newRecords.Enqueue(record);
}
}
private unsafe void OnResourceLoaded(ResourceHandle* handle, Utf8GamePath path, FullPath? manipulatedPath, ResolveData data)
{
if( _writeToLog )
if (_config.EnableResourceLogging)
{
var log = FilterMatch(path.Path, out var name);
var name2 = string.Empty;
if (manipulatedPath != null)
{
log |= FilterMatch(manipulatedPath.Value.InternalName, out name2);
}
if (log)
{
@ -224,26 +224,42 @@ public partial class ResourceWatcher : IDisposable, ITab
}
}
if( _isEnabled )
if (_config.EnableResourceWatcher)
{
var record = manipulatedPath == null
? Record.CreateDefaultLoad(path.Path, handle, data.ModCollection)
: Record.CreateLoad( path.Path, manipulatedPath.Value.InternalName, handle, data.ModCollection );
: Record.CreateLoad(path.Path, manipulatedPath.Value.InternalName, handle,
data.ModCollection);
if (!_config.OnlyAddMatchingResources || _table.WouldBeVisible(record))
_newRecords.Enqueue(record);
}
}
private unsafe void OnFileLoaded( ResourceHandle* resource, ByteString path, bool success, bool custom )
{
if( _writeToLog && FilterMatch( path, out var match ) )
private unsafe void OnFileLoaded(ResourceHandle* resource, ByteString path, bool success, bool custom, ByteString _)
{
if (_config.EnableResourceLogging && FilterMatch(path, out var match))
Penumbra.Log.Information(
$"[ResourceLoader] [FILE] [{resource->FileType}] Loading {match} from {(custom ? "local files" : "SqPack")} into 0x{(ulong)resource:X} returned {success}.");
if (_config.EnableResourceWatcher)
{
var record = Record.CreateFileLoad(path, resource, success, custom);
if (!_config.OnlyAddMatchingResources || _table.WouldBeVisible(record))
_newRecords.Enqueue(record);
}
}
if( _isEnabled )
private unsafe void OnResourceDestroyed(ResourceHandle* resource)
{
_newRecords.Enqueue( Record.CreateFileLoad( path, resource, success, custom ) );
if (_config.EnableResourceLogging && FilterMatch(resource->FileName(), out var match))
Penumbra.Log.Information(
$"[ResourceLoader] [DEST] [{resource->FileType}] Destroyed {match} at 0x{(ulong)resource:X}.");
if (_config.EnableResourceWatcher)
{
var record = Record.CreateDestruction(resource);
if (!_config.OnlyAddMatchingResources || _table.WouldBeVisible(record))
_newRecords.Enqueue(record);
}
}
}