ToxicMushroom/Penumbra
1
0
Fork 0
mirror of https://github.com/xivdev/Penumbra.git synced 2025-12-12 18:27:24 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'async-stuff'

Browse source 
# Conflicts:
#	Penumbra/Interop/Hooks/ResourceLoading/ResourceLoader.cs
This commit is contained in:
Ottermandias 2025-01-25 13:55:54 +01:00
commit e508e6158f
11 changed files with 307 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Penumbra/Interop/Hooks/HookSettings.cs
View file

@ -100,6 +100,7 @@ public class HookOverrides
public bool DecRef;
public bool GetResourceSync;
public bool GetResourceAsync;
public bool UpdateResourceState;
public bool CheckFileState;
public bool TexResourceHandleOnLoad;
public bool LoadMdlFileExtern;

130
Penumbra/Interop/Hooks/ResourceLoading/ResourceLoader.cs
View file

@ -2,6 +2,7 @@ using FFXIVClientStructs.FFXIV.Client.System.Resource;
using OtterGui.Services;
using Penumbra.Api.Enums;
using Penumbra.Collections;
using Penumbra.Interop.Hooks.Resources;
using Penumbra.Interop.PathResolving;
using Penumbra.Interop.SafeHandles;
using Penumbra.Interop.Structs;
@ -13,27 +14,38 @@ namespace Penumbra.Interop.Hooks.ResourceLoading;
public unsafe class ResourceLoader : IDisposable, IService
{
private readonly ResourceService _resources;
private readonly FileReadService _fileReadService;
private readonly RsfService _rsfService;
private readonly PapHandler _papHandler;
private readonly Configuration _config;
private readonly ResourceService _resources;
private readonly FileReadService _fileReadService;
private readonly RsfService _rsfService;
private readonly PapHandler _papHandler;
private readonly Configuration _config;
private readonly ResourceHandleDestructor _destructor;
private readonly ConcurrentDictionary<nint, Utf8GamePath> _ongoingLoads = [];
private ResolveData _resolvedData = ResolveData.Invalid;
public event Action<Utf8GamePath, FullPath?, ResolveData>? PapRequested;
public ResourceLoader(ResourceService resources, FileReadService fileReadService, RsfService rsfService, Configuration config, PeSigScanner sigScanner)
public IReadOnlyDictionary<nint, Utf8GamePath> OngoingLoads
=> _ongoingLoads;
public ResourceLoader(ResourceService resources, FileReadService fileReadService, RsfService rsfService, Configuration config, PeSigScanner sigScanner,
ResourceHandleDestructor destructor)
{
_resources = resources;
_fileReadService = fileReadService;
_rsfService = rsfService;
_rsfService = rsfService;
_config = config;
_destructor = destructor;
ResetResolvePath();
_resources.ResourceRequested += ResourceHandler;
_resources.ResourceHandleIncRef += IncRefProtection;
_resources.ResourceHandleDecRef += DecRefProtection;
_fileReadService.ReadSqPack += ReadSqPackDetour;
_resources.ResourceRequested += ResourceHandler;
_resources.ResourceStateUpdating += ResourceStateUpdatingHandler;
_resources.ResourceStateUpdated += ResourceStateUpdatedHandler;
_resources.ResourceHandleIncRef += IncRefProtection;
_resources.ResourceHandleDecRef += DecRefProtection;
_fileReadService.ReadSqPack += ReadSqPackDetour;
_destructor.Subscribe(ResourceDestructorHandler, ResourceHandleDestructor.Priority.ResourceLoader);
_papHandler = new PapHandler(sigScanner, PapResourceHandler);
_papHandler.Enable();
@ -109,12 +121,32 @@ public unsafe class ResourceLoader : IDisposable, IService
/// </summary>
public event FileLoadedDelegate? FileLoaded;
public delegate void ResourceCompleteDelegate(ResourceHandle* resource, CiByteString path, Utf8GamePath originalPath,
ReadOnlySpan<byte> additionalData, bool isAsync);
/// <summary>
/// Event fired just before a resource finishes loading.
/// <see cref="ResourceHandle.LoadState"/> must be checked to know whether the load was successful or not.
/// AdditionalData is either empty or the part of the path inside the leading pipes.
/// </summary>
public event ResourceCompleteDelegate? BeforeResourceComplete;
/// <summary>
/// Event fired when a resource has finished loading.
/// <see cref="ResourceHandle.LoadState"/> must be checked to know whether the load was successful or not.
/// AdditionalData is either empty or the part of the path inside the leading pipes.
/// </summary>
public event ResourceCompleteDelegate? ResourceComplete;
public void Dispose()
{
_resources.ResourceRequested -= ResourceHandler;
_resources.ResourceHandleIncRef -= IncRefProtection;
_resources.ResourceHandleDecRef -= DecRefProtection;
_fileReadService.ReadSqPack -= ReadSqPackDetour;
_resources.ResourceRequested -= ResourceHandler;
_resources.ResourceStateUpdating -= ResourceStateUpdatingHandler;
_resources.ResourceStateUpdated -= ResourceStateUpdatedHandler;
_resources.ResourceHandleIncRef -= IncRefProtection;
_resources.ResourceHandleDecRef -= DecRefProtection;
_fileReadService.ReadSqPack -= ReadSqPackDetour;
_destructor.Unsubscribe(ResourceDestructorHandler);
_papHandler.Dispose();
}
@ -135,7 +167,8 @@ public unsafe class ResourceLoader : IDisposable, IService
if (resolvedPath == null || !Utf8GamePath.FromByteString(resolvedPath.Value.InternalName, out var p))
{
returnValue = _resources.GetOriginalResource(sync, category, type, hash, path.Path, parameters);
returnValue = _resources.GetOriginalResource(sync, category, type, hash, path.Path, original, parameters);
TrackResourceLoad(returnValue, original);
ResourceLoaded?.Invoke(returnValue, path, resolvedPath, data);
return;
}
@ -145,10 +178,57 @@ public unsafe class ResourceLoader : IDisposable, IService
hash = ComputeHash(resolvedPath.Value.InternalName, parameters);
var oldPath = path;
path = p;
returnValue = _resources.GetOriginalResource(sync, category, type, hash, path.Path, parameters);
returnValue = _resources.GetOriginalResource(sync, category, type, hash, path.Path, original, parameters);
TrackResourceLoad(returnValue, original);
ResourceLoaded?.Invoke(returnValue, oldPath, resolvedPath.Value, data);
}
private void TrackResourceLoad(ResourceHandle* handle, Utf8GamePath original)
{
if (handle->UnkState == 2 && handle->LoadState >= LoadState.Success)
return;
_ongoingLoads.TryAdd((nint)handle, original.Clone());
}
private void ResourceStateUpdatedHandler(ResourceHandle* handle, Utf8GamePath syncOriginal, (byte, LoadState) previousState, ref uint returnValue)
{
if (handle->UnkState != 2 || handle->LoadState < LoadState.Success || previousState is { Item1: 2, Item2: >= LoadState.Success })
return;
if (!_ongoingLoads.TryRemove((nint)handle, out var asyncOriginal))
asyncOriginal = Utf8GamePath.Empty;
var path = handle->CsHandle.FileName;
if (!syncOriginal.IsEmpty && !asyncOriginal.IsEmpty && !syncOriginal.Equals(asyncOriginal))
Penumbra.Log.Warning($"[ResourceLoader] Resource original paths inconsistency: 0x{(nint)handle:X}, of path {path}, sync original {syncOriginal}, async original {asyncOriginal}.");
var original = !asyncOriginal.IsEmpty ? asyncOriginal : syncOriginal;
Penumbra.Log.Excessive($"[ResourceLoader] Resource is complete: 0x{(nint)handle:X}, of path {path}, original {original}, state {previousState.Item1}:{previousState.Item2} -> {handle->UnkState}:{handle->LoadState}, sync: {asyncOriginal.IsEmpty}");
if (PathDataHandler.Split(path.AsSpan(), out var actualPath, out var additionalData))
ResourceComplete?.Invoke(handle, new CiByteString(actualPath), original, additionalData, !asyncOriginal.IsEmpty);
else
ResourceComplete?.Invoke(handle, path.AsByteString(), original, [], !asyncOriginal.IsEmpty);
}
private void ResourceStateUpdatingHandler(ResourceHandle* handle, Utf8GamePath syncOriginal)
{
if (handle->UnkState != 1 || handle->LoadState != LoadState.Success)
return;
if (!_ongoingLoads.TryGetValue((nint)handle, out var asyncOriginal))
asyncOriginal = Utf8GamePath.Empty;
var path = handle->CsHandle.FileName;
var original = asyncOriginal.IsEmpty ? syncOriginal : asyncOriginal;
Penumbra.Log.Excessive($"[ResourceLoader] Resource is about to be complete: 0x{(nint)handle:X}, of path {path}, original {original}");
if (PathDataHandler.Split(path.AsSpan(), out var actualPath, out var additionalData))
BeforeResourceComplete?.Invoke(handle, new CiByteString(actualPath), original, additionalData, !asyncOriginal.IsEmpty);
else
BeforeResourceComplete?.Invoke(handle, path.AsByteString(), original, [], !asyncOriginal.IsEmpty);
}
private void ReadSqPackDetour(SeFileDescriptor* fileDescriptor, ref int priority, ref bool isSync, ref byte? returnValue)
{
if (fileDescriptor->ResourceHandle == null)
@ -176,7 +256,6 @@ public unsafe class ResourceLoader : IDisposable, IService
gamePath.Path.IsAscii);
fileDescriptor->ResourceHandle->FileNameData = path.Path;
fileDescriptor->ResourceHandle->FileNameLength = path.Length;
ForceSync(fileDescriptor, ref isSync);
returnValue = DefaultLoadResource(path, fileDescriptor, priority, isSync, data);
// Return original resource handle path so that they can be loaded separately.
fileDescriptor->ResourceHandle->FileNameData = gamePath.Path.Path;
@ -215,16 +294,6 @@ public unsafe class ResourceLoader : IDisposable, IService
}
}
/// <summary> Special handling for materials and IMCs. </summary>
private static void ForceSync(SeFileDescriptor* fileDescriptor, ref bool isSync)
{
// Force isSync = true for Materials. I don't really understand why,
// or where the difference even comes from.
// Was called with True on my client and with false on other peoples clients,
// which caused problems.
isSync |= fileDescriptor->ResourceHandle->FileType is ResourceType.Mtrl or ResourceType.Imc;
}
/// <summary>
/// A resource with ref count 0 that gets incremented goes through GetResourceAsync again.
/// This means, that if the path determined from that is different than the resources path,
@ -265,6 +334,11 @@ public unsafe class ResourceLoader : IDisposable, IService
returnValue = 1;
}
private void ResourceDestructorHandler(ResourceHandle* handle)
{
_ongoingLoads.TryRemove((nint)handle, out _);
}
/// <summary> Compute the CRC32 hash for a given path together with potential resource parameters. </summary>
private static int ComputeHash(CiByteString path, GetResourceParameters* pGetResParams)
{

76
Penumbra/Interop/Hooks/ResourceLoading/ResourceService.cs
View file

@ -19,6 +19,8 @@ public unsafe class ResourceService : IDisposable, IRequiredService
private readonly PerformanceTracker _performance;
private readonly ResourceManagerService _resourceManager;
private readonly ThreadLocal<Utf8GamePath> _currentGetResourcePath = new(() => Utf8GamePath.Empty);
public ResourceService(PerformanceTracker performance, ResourceManagerService resourceManager, IGameInteropProvider interop)
{
_performance = performance;
@ -34,6 +36,8 @@ public unsafe class ResourceService : IDisposable, IRequiredService
_getResourceSyncHook.Enable();
if (!HookOverrides.Instance.ResourceLoading.GetResourceAsync)
_getResourceAsyncHook.Enable();
if (!HookOverrides.Instance.ResourceLoading.UpdateResourceState)
_updateResourceStateHook.Enable();
if (!HookOverrides.Instance.ResourceLoading.IncRef)
_incRefHook.Enable();
if (!HookOverrides.Instance.ResourceLoading.DecRef)
@ -54,8 +58,10 @@ public unsafe class ResourceService : IDisposable, IRequiredService
{
_getResourceSyncHook.Dispose();
_getResourceAsyncHook.Dispose();
_updateResourceStateHook.Dispose();
_incRefHook.Dispose();
_decRefHook.Dispose();
_currentGetResourcePath.Dispose();
}
#region GetResource
@ -112,28 +118,82 @@ public unsafe class ResourceService : IDisposable, IRequiredService
unk9);
}
var original = gamePath;
ResourceHandle* returnValue = null;
ResourceRequested?.Invoke(ref *categoryId, ref *resourceType, ref *resourceHash, ref gamePath, gamePath, pGetResParams, ref isSync,
ResourceRequested?.Invoke(ref *categoryId, ref *resourceType, ref *resourceHash, ref gamePath, original, pGetResParams, ref isSync,
ref returnValue);
if (returnValue != null)
return returnValue;
return GetOriginalResource(isSync, *categoryId, *resourceType, *resourceHash, gamePath.Path, pGetResParams, isUnk, unk8, unk9);
return GetOriginalResource(isSync, *categoryId, *resourceType, *resourceHash, gamePath.Path, original, pGetResParams, isUnk, unk8, unk9);
}
/// <summary> Call the original GetResource function. </summary>
public ResourceHandle* GetOriginalResource(bool sync, ResourceCategory categoryId, ResourceType type, int hash, CiByteString path,
public ResourceHandle* GetOriginalResource(bool sync, ResourceCategory categoryId, ResourceType type, int hash, CiByteString path, Utf8GamePath original,
GetResourceParameters* resourceParameters = null, byte unk = 0, nint unk8 = 0, uint unk9 = 0)
=> sync
? _getResourceSyncHook.OriginalDisposeSafe(_resourceManager.ResourceManager, &categoryId, &type, &hash, path.Path,
resourceParameters, unk8, unk9)
: _getResourceAsyncHook.OriginalDisposeSafe(_resourceManager.ResourceManager, &categoryId, &type, &hash, path.Path,
resourceParameters, unk, unk8, unk9);
{
var previous = _currentGetResourcePath.Value;
try
{
_currentGetResourcePath.Value = original;
return sync
? _getResourceSyncHook.OriginalDisposeSafe(_resourceManager.ResourceManager, &categoryId, &type, &hash, path.Path,
resourceParameters, unk8, unk9)
: _getResourceAsyncHook.OriginalDisposeSafe(_resourceManager.ResourceManager, &categoryId, &type, &hash, path.Path,
resourceParameters, unk, unk8, unk9);
} finally
{
_currentGetResourcePath.Value = previous;
}
}
#endregion
private delegate nint ResourceHandlePrototype(ResourceHandle* handle);
#region UpdateResourceState
/// <summary> Invoked before a resource state is updated. </summary>
/// <param name="handle">The resource handle.</param>
/// <param name="syncOriginal">The original game path of the resource, if loaded synchronously.</param>
public delegate void ResourceStateUpdatingDelegate(ResourceHandle* handle, Utf8GamePath syncOriginal);
/// <summary> Invoked after a resource state is updated. </summary>
/// <param name="handle">The resource handle.</param>
/// <param name="syncOriginal">The original game path of the resource, if loaded synchronously.</param>
/// <param name="previousState">The previous state of the resource.</param>
/// <param name="returnValue">The return value to use.</param>
public delegate void ResourceStateUpdatedDelegate(ResourceHandle* handle, Utf8GamePath syncOriginal, (byte UnkState, LoadState LoadState) previousState, ref uint returnValue);
/// <summary>
/// <inheritdoc cref="ResourceStateUpdatingDelegate"/> <para/>
/// Subscribers should be exception-safe.
/// </summary>
public event ResourceStateUpdatingDelegate? ResourceStateUpdating;
/// <summary>
/// <inheritdoc cref="ResourceStateUpdatedDelegate"/> <para/>
/// Subscribers should be exception-safe.
/// </summary>
public event ResourceStateUpdatedDelegate? ResourceStateUpdated;
private delegate uint UpdateResourceStatePrototype(ResourceHandle* handle, byte offFileThread);
[Signature(Sigs.UpdateResourceState, DetourName = nameof(UpdateResourceStateDetour))]
private readonly Hook<UpdateResourceStatePrototype> _updateResourceStateHook = null!;
private uint UpdateResourceStateDetour(ResourceHandle* handle, byte offFileThread)
{
var previousState = (handle->UnkState, handle->LoadState);
var syncOriginal = _currentGetResourcePath.IsValueCreated ? _currentGetResourcePath.Value : Utf8GamePath.Empty;
ResourceStateUpdating?.Invoke(handle, syncOriginal);
var ret = _updateResourceStateHook.OriginalDisposeSafe(handle, offFileThread);
ResourceStateUpdated?.Invoke(handle, syncOriginal, previousState, ref ret);
return ret;
}
#endregion
#region IncRef
/// <summary> Invoked before a resource handle reference count is incremented. </summary>

5
Penumbra/Interop/Hooks/Resources/ResourceHandleDestructor.cs
View file

@ -14,9 +14,12 @@ public sealed unsafe class ResourceHandleDestructor : EventWrapperPtr<ResourceHa
/// <seealso cref="PathResolving.SubfileHelper"/>
SubfileHelper,
/// <seealso cref="ShaderReplacementFixer"/>
/// <seealso cref="PostProcessing.ShaderReplacementFixer"/>
ShaderReplacementFixer,
/// <seealso cref="ResourceLoading.ResourceLoader"/>
ResourceLoader,
/// <seealso cref="ResourceWatcher.OnResourceDestroyed"/>
ResourceWatcher,
}

15
Penumbra/Interop/Processing/FilePostProcessService.cs
View file

@ -4,6 +4,7 @@ using Penumbra.Api.Enums;
using Penumbra.Interop.Hooks.ResourceLoading;
using Penumbra.Interop.Structs;
using Penumbra.String;
using Penumbra.String.Classes;
namespace Penumbra.Interop.Processing;
@ -20,20 +21,20 @@ public unsafe class FilePostProcessService : IRequiredService, IDisposable
public FilePostProcessService(ResourceLoader resourceLoader, ServiceManager services)
{
_resourceLoader = resourceLoader;
_processors = services.GetServicesImplementing<IFilePostProcessor>().ToFrozenDictionary(s => s.Type, s => s);
_resourceLoader.FileLoaded += OnFileLoaded;
_resourceLoader = resourceLoader;
_processors = services.GetServicesImplementing<IFilePostProcessor>().ToFrozenDictionary(s => s.Type, s => s);
_resourceLoader.BeforeResourceComplete += OnBeforeResourceComplete;
}
public void Dispose()
{
_resourceLoader.FileLoaded -= OnFileLoaded;
_resourceLoader.BeforeResourceComplete -= OnBeforeResourceComplete;
}
private void OnFileLoaded(ResourceHandle* resource, CiByteString path, bool returnValue, bool custom,
ReadOnlySpan<byte> additionalData)
private void OnBeforeResourceComplete(ResourceHandle* resource, CiByteString path, Utf8GamePath original,
ReadOnlySpan<byte> additionalData, bool isAsync)
{
if (_processors.TryGetValue(resource->FileType, out var processor))
processor.PostProcess(resource, path, additionalData);
processor.PostProcess(resource, original.Path, additionalData);
}
}

15
Penumbra/Interop/Structs/ResourceHandle.cs
View file

@ -24,10 +24,20 @@ public unsafe struct TextureResourceHandle
public enum LoadState : byte
{
Constructing = 0x00,
Constructed = 0x01,
Async2 = 0x02,
AsyncRequested = 0x03,
Async4 = 0x04,
AsyncLoading = 0x05,
Async6 = 0x06,
Success = 0x07,
Async = 0x03,
Unknown8 = 0x08,
Failure = 0x09,
FailedSubResource = 0x0A,
FailureB = 0x0B,
FailureC = 0x0C,
FailureD = 0x0D,
None = 0xFF,
}
@ -74,6 +84,9 @@ public unsafe struct ResourceHandle
[FieldOffset(0x58)]
public int FileNameLength;
[FieldOffset(0xA8)]
public byte UnkState;
[FieldOffset(0xA9)]
public LoadState LoadState;