Merge pull request #220 from Soreepeong/fix/segmented-read

Fix music modding: calculate correct crc32 values for segmented read

Penumbra/Collections/ModCollection.Cache.cs

@@ -308,12 +308,6 @@ public partial class ModCollection
         {
             foreach( var (path, file) in subMod.Files.Concat( subMod.FileSwaps ) )
             {
-                // Skip all filtered files
-                if( Mod.FilterFile( path ) )
-                {
-                    continue;
-                }
-
                 AddFile( path, file, parentMod );
             }
 

Penumbra/Configuration.cs

@@ -52,7 +52,6 @@ public partial class Configuration : IPluginConfiguration
     public bool FixMainWindow { get; set; } = false;
     public bool ShowAdvanced { get; set; }
     public bool AutoDeduplicateOnImport { get; set; } = false;
-    public bool DisableSoundStreaming { get; set; } = true;
     public bool EnableHttpApi { get; set; }
 
     public string DefaultModImportPath { get; set; } = string.Empty;

Penumbra/Interop/Loader/ResourceLoader.Replacement.cs

@@ -1,12 +1,14 @@
 using System;
 using System.Diagnostics;
 using System.Linq;
+using System.Runtime.InteropServices;
 using Dalamud.Hooking;
 using Dalamud.Logging;
 using Dalamud.Utility.Signatures;
 using FFXIVClientStructs.FFXIV.Client.System.Resource;
 using Penumbra.GameData.ByteString;
 using Penumbra.GameData.Enums;
+using Penumbra.GameData.Util;
 using Penumbra.Interop.Structs;
 using FileMode = Penumbra.Interop.Structs.FileMode;
 using ResourceHandle = FFXIVClientStructs.FFXIV.Client.System.Resource.Handle.ResourceHandle;
@@ -17,31 +19,44 @@ public unsafe partial class ResourceLoader
 {
     // Resources can be obtained synchronously and asynchronously. We need to change behaviour in both cases.
     // Both work basically the same, so we can reduce the main work to one function used by both hooks.
+
+    [StructLayout( LayoutKind.Explicit )]
+    public struct GetResourceParameters
+    {
+        [FieldOffset( 16 )]
+        public uint SegmentOffset;
+
+        [FieldOffset( 20 )]
+        public uint SegmentLength;
+
+        public bool IsPartialRead => SegmentLength != 0;
+    }
+
     public delegate ResourceHandle* GetResourceSyncPrototype( ResourceManager* resourceManager, ResourceCategory* pCategoryId,
-        ResourceType* pResourceType, int* pResourceHash, byte* pPath, void* pUnknown );
+        ResourceType* pResourceType, int* pResourceHash, byte* pPath, GetResourceParameters* pGetResParams );
 
     [Signature( "E8 ?? ?? 00 00 48 8D 8F ?? ?? 00 00 48 89 87 ?? ?? 00 00", DetourName = "GetResourceSyncDetour" )]
     public Hook< GetResourceSyncPrototype > GetResourceSyncHook = null!;
 
     public delegate ResourceHandle* GetResourceAsyncPrototype( ResourceManager* resourceManager, ResourceCategory* pCategoryId,
-        ResourceType* pResourceType, int* pResourceHash, byte* pPath, void* pUnknown, bool isUnknown );
+        ResourceType* pResourceType, int* pResourceHash, byte* pPath, GetResourceParameters* pGetResParams, bool isUnknown );
 
     [Signature( "E8 ?? ?? ?? 00 48 8B D8 EB ?? F0 FF 83 ?? ?? 00 00", DetourName = "GetResourceAsyncDetour" )]
     public Hook< GetResourceAsyncPrototype > GetResourceAsyncHook = null!;
 
     private ResourceHandle* GetResourceSyncDetour( ResourceManager* resourceManager, ResourceCategory* categoryId, ResourceType* resourceType,
-        int* resourceHash, byte* path, void* unk )
+        int* resourceHash, byte* path, GetResourceParameters* pGetResParams )
-        => GetResourceHandler( true, resourceManager, categoryId, resourceType, resourceHash, path, unk, false );
+        => GetResourceHandler( true, resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams, false );
 
     private ResourceHandle* GetResourceAsyncDetour( ResourceManager* resourceManager, ResourceCategory* categoryId, ResourceType* resourceType,
-        int* resourceHash, byte* path, void* unk, bool isUnk )
+        int* resourceHash, byte* path, GetResourceParameters* pGetResParams, bool isUnk )
-        => GetResourceHandler( false, resourceManager, categoryId, resourceType, resourceHash, path, unk, isUnk );
+        => GetResourceHandler( false, resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams, isUnk );
 
     private ResourceHandle* CallOriginalHandler( bool isSync, ResourceManager* resourceManager, ResourceCategory* categoryId,
-        ResourceType* resourceType, int* resourceHash, byte* path, void* unk, bool isUnk )
+        ResourceType* resourceType, int* resourceHash, byte* path, GetResourceParameters* pGetResParams, bool isUnk )
         => isSync
-            ? GetResourceSyncHook.Original( resourceManager, categoryId, resourceType, resourceHash, path, unk )
+            ? GetResourceSyncHook.Original( resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams )
-            : GetResourceAsyncHook.Original( resourceManager, categoryId, resourceType, resourceHash, path, unk, isUnk );
+            : GetResourceAsyncHook.Original( resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams, isUnk );
 
 
     [Conditional( "DEBUG" )]
@@ -56,15 +71,15 @@ public unsafe partial class ResourceLoader
     private event Action< Utf8GamePath, FullPath?, object? >? PathResolved;
 
     private ResourceHandle* GetResourceHandler( bool isSync, ResourceManager* resourceManager, ResourceCategory* categoryId,
-        ResourceType* resourceType, int* resourceHash, byte* path, void* unk, bool isUnk )
+        ResourceType* resourceType, int* resourceHash, byte* path, GetResourceParameters* pGetResParams, bool isUnk )
     {
         if( !Utf8GamePath.FromPointer( path, out var gamePath ) )
         {
             PluginLog.Error( "Could not create GamePath from resource path." );
-            return CallOriginalHandler( isSync, resourceManager, categoryId, resourceType, resourceHash, path, unk, isUnk );
+            return CallOriginalHandler( isSync, resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams, isUnk );
         }
 
-        CompareHash( gamePath.Path.Crc32, *resourceHash, gamePath );
+        CompareHash( ComputeHash( gamePath.Path, pGetResParams ), *resourceHash, gamePath );
 
         ResourceRequested?.Invoke( gamePath, isSync );
 
@@ -73,15 +88,15 @@ public unsafe partial class ResourceLoader
         PathResolved?.Invoke( gamePath, resolvedPath, data );
         if( resolvedPath == null )
         {
-            var retUnmodified = CallOriginalHandler( isSync, resourceManager, categoryId, resourceType, resourceHash, path, unk, isUnk );
+            var retUnmodified = CallOriginalHandler( isSync, resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams, isUnk );
             ResourceLoaded?.Invoke( ( Structs.ResourceHandle* )retUnmodified, gamePath, null, data );
             return retUnmodified;
         }
 
         // Replace the hash and path with the correct one for the replacement.
-        *resourceHash = resolvedPath.Value.InternalName.Crc32;
+        *resourceHash = ComputeHash( resolvedPath.Value.InternalName, pGetResParams );
         path          = resolvedPath.Value.InternalName.Path;
-        var retModified = CallOriginalHandler( isSync, resourceManager, categoryId, resourceType, resourceHash, path, unk, isUnk );
+        var retModified = CallOriginalHandler( isSync, resourceManager, categoryId, resourceType, resourceHash, path, pGetResParams, isUnk );
         ResourceLoaded?.Invoke( ( Structs.ResourceHandle* )retModified, gamePath, resolvedPath.Value, data );
         return retModified;
     }
@@ -228,4 +243,20 @@ public unsafe partial class ResourceLoader
         GetResourceSyncHook.Dispose();
         GetResourceAsyncHook.Dispose();
     }
+
+    private int ComputeHash( Utf8String path, GetResourceParameters* pGetResParams )
+    {
+        if( pGetResParams == null || !pGetResParams->IsPartialRead )
+            return path.Crc32;
+
+        // When the game requests file only partially, crc32 includes that information, in format of:
+        // path/to/file.ext.hex_offset.hex_size
+        // ex) music/ex4/BGM_EX4_System_Title.scd.381adc.30000
+        return Utf8String.Join(
+            (byte)'.',
+            path,
+            Utf8String.FromStringUnsafe( pGetResParams->SegmentOffset.ToString( "x" ), true ),
+            Utf8String.FromStringUnsafe( pGetResParams->SegmentLength.ToString( "x" ), true )
+        ).Crc32;
+    }
 }

Penumbra/Interop/MusicManager.cs

@@ -1,41 +0,0 @@
-using System;
-using Dalamud.Logging;
-using Dalamud.Utility.Signatures;
-
-namespace Penumbra.Interop;
-
-// Use this to disable streaming of specific soundfiles,
-// which will allow replacement of .scd files.
-public unsafe class MusicManager
-{
-    // The wildcard is the offset in framework to the MusicManager in Framework.
-    [Signature( "48 8B 8E ?? ?? ?? ?? 39 78 20 0F 94 C2 45 33 C0", ScanType = ScanType.Text )]
-    private readonly IntPtr _musicInitCallLocation = IntPtr.Zero;
-
-    private readonly IntPtr _musicManager;
-
-    public MusicManager()
-    {
-        SignatureHelper.Initialise( this );
-        var framework          = Dalamud.Framework.Address.BaseAddress;
-        var musicManagerOffset = *( int* )( _musicInitCallLocation + 3 );
-        _musicManager = *( IntPtr* )( framework + musicManagerOffset );
-        PluginLog.Debug( "MusicManager found at 0x{Location:X16}", _musicManager.ToInt64() );
-    }
-
-    public bool StreamingEnabled
-    {
-        get => *( bool* )( _musicManager + 50 );
-        private set
-        {
-            PluginLog.Debug( value ? "Music streaming enabled." : "Music streaming disabled." );
-            *( bool* )( _musicManager + 50 ) = value;
-        }
-    }
-
-    public void EnableStreaming()
-        => StreamingEnabled = true;
-
-    public void DisableStreaming()
-        => StreamingEnabled = false;
-}

Penumbra/Mods/Mod.Files.cs

@@ -70,13 +70,6 @@ public partial class Mod
            .ToList();
     }
 
-    // Filter invalid files.
-    // If audio streaming is not disabled, replacing .scd files crashes the game,
-    // so only add those files if it is disabled.
-    public static bool FilterFile( Utf8GamePath gamePath )
-        => !Penumbra.Config.DisableSoundStreaming
-         && gamePath.Path.EndsWith( '.', 's', 'c', 'd' );
-
     private static IModGroup? LoadModGroup( FileInfo file, DirectoryInfo basePath )
     {
         if( !File.Exists( file.FullName ) )

Penumbra/Penumbra.cs

@@ -112,7 +112,6 @@ public class Penumbra : IDisposable
 
     public readonly  ResourceLogger ResourceLogger;
     public readonly  PathResolver   PathResolver;
-    public readonly  MusicManager   MusicManager;
     public readonly  ObjectReloader ObjectReloader;
     public readonly  ModFileSystem  ModFileSystem;
     public readonly  PenumbraApi    Api;
@@ -131,12 +130,6 @@ public class Penumbra : IDisposable
         Backup.CreateBackup( PenumbraBackupFiles() );
         Config = Configuration.Load();
 
-        MusicManager = new MusicManager();
-        if( Config.DisableSoundStreaming )
-        {
-            MusicManager.DisableStreaming();
-        }
-
         ResidentResources = new ResidentResourceManager();
         TempMods          = new TempModManager();
         MetaFileManager   = new MetaFileManager();
@@ -462,7 +455,6 @@ public class Penumbra : IDisposable
         sb.AppendFormat( "> **`Plugin Version:              `** {0}\n", Version );
         sb.AppendFormat( "> **`Commit Hash:                 `** {0}\n", CommitHash );
         sb.AppendFormat( "> **`Enable Mods:                 `** {0}\n", Config.EnableMods );
-        sb.AppendFormat( "> **`Enable Sound Modification:   `** {0}\n", Config.DisableSoundStreaming );
         sb.AppendFormat( "> **`Enable HTTP API:             `** {0}\n", Config.EnableHttpApi );
         sb.AppendFormat( "> **`Root Directory:              `** `{0}`, {1}\n", Config.ModDirectory, exists ? "Exists" : "Not Existing" );
         sb.AppendFormat( "> **`Free Drive Space:            `** {0}\n",

Penumbra/UI/ConfigWindow.DebugTab.cs

@@ -99,12 +99,11 @@ public partial class ConfigWindow
             PrintValue( "Mod Manager BasePath Exists", Directory.Exists( manager.BasePath.FullName ).ToString() );
             PrintValue( "Mod Manager Valid", manager.Valid.ToString() );
             PrintValue( "Path Resolver Enabled", _window._penumbra.PathResolver.Enabled.ToString() );
-            PrintValue( "Music Manager Streaming Disabled", ( !_window._penumbra.MusicManager.StreamingEnabled ).ToString() );
             PrintValue( "Web Server Enabled", ( _window._penumbra.WebServer != null ).ToString() );
         }
 
         // Draw all resources currently replaced by Penumbra and (if existing) the resources they replace.
-        // Resources are collected by iterating through the 
+        // Resources are collected by iterating through the
         private static unsafe void DrawDebugTabReplacedResources()
         {
             if( !ImGui.CollapsingHeader( "Replaced Resources" ) )

Penumbra/UI/ConfigWindow.SettingsTab.Advanced.cs

@@ -24,7 +24,6 @@ public partial class ConfigWindow
                 "Automatically deduplicate mod files on import. This will make mod file sizes smaller, but deletes (binary identical) files.",
                 Penumbra.Config.AutoDeduplicateOnImport, v => Penumbra.Config.AutoDeduplicateOnImport = v );
             DrawRequestedResourceLogging();
-            DrawDisableSoundStreamingBox();
             DrawEnableHttpApiBox();
             DrawEnableDebugModeBox();
             DrawEnableFullResourceLoggingBox();
@@ -62,36 +61,6 @@ public partial class ConfigWindow
             }
         }
 
-        // Toggling audio streaming will need to apply to the music manager
-        // and rediscover mods due to determining whether .scds will be loaded or not.
-        private void DrawDisableSoundStreamingBox()
-        {
-            var tmp = Penumbra.Config.DisableSoundStreaming;
-            if( ImGui.Checkbox( "##streaming", ref tmp ) && tmp != Penumbra.Config.DisableSoundStreaming )
-            {
-                Penumbra.Config.DisableSoundStreaming = tmp;
-                Penumbra.Config.Save();
-                if( tmp )
-                {
-                    _window._penumbra.MusicManager.DisableStreaming();
-                }
-                else
-                {
-                    _window._penumbra.MusicManager.EnableStreaming();
-                }
-
-                Penumbra.ModManager.DiscoverMods();
-            }
-
-            ImGui.SameLine();
-            ImGuiUtil.LabeledHelpMarker( "Enable Sound Modification",
-                "Disable streaming in the games audio engine. The game enables this by default, and Penumbra should disable it.\n"
-              + "If this is unchecked, you can not replace sound files in the game (*.scd files), they will be ignored by Penumbra.\n\n"
-              + "Only touch this if you experience sound problems like audio stuttering.\n"
-              + "If you toggle this, make sure no modified or to-be-modified sound file is currently playing or was recently playing, else you might crash.\n"
-              + "You might need to restart your game for this to fully take effect." );
-        }
-
         // Creates and destroys the web server when toggled.
         private void DrawEnableHttpApiBox()
         {