Make saving files and recalculating effective files threaded/once per frame.

Penumbra/Collections/CollectionManager.Active.cs

@@ -177,7 +177,10 @@ public partial class ModCollection
 
 
         public void SaveActiveCollections()
-            => SaveActiveCollections( Default.Name, Current.Name, Characters.Select( kvp => ( kvp.Key, kvp.Value.Name ) ) );
+        {
+            Penumbra.Framework.RegisterDelayed( nameof( SaveActiveCollections ),
+                () => SaveActiveCollections( Default.Name, Current.Name, Characters.Select( kvp => ( kvp.Key, kvp.Value.Name ) ) ) );
+        }
 
         internal static void SaveActiveCollections( string def, string current, IEnumerable< (string, string) > characters )
         {

Penumbra/Collections/ModCollection.Cache.Access.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Threading;
 using Dalamud.Logging;
 using OtterGui.Classes;
 using Penumbra.GameData.ByteString;
@@ -61,9 +62,6 @@ public partial class ModCollection
     internal IReadOnlyDictionary< Utf8GamePath, FullPath > ResolvedFiles
         => _cache?.ResolvedFiles ?? new Dictionary< Utf8GamePath, FullPath >();
 
-    internal IReadOnlySet< FullPath > MissingFiles
-        => _cache?.MissingFiles ?? new HashSet< FullPath >();
-
     internal IReadOnlyDictionary< string, object? > ChangedItems
         => _cache?.ChangedItems ?? new Dictionary< string, object? >();
 
@@ -76,6 +74,10 @@ public partial class ModCollection
     // Update the effective file list for the given cache.
     // Creates a cache if necessary.
     public void CalculateEffectiveFileList( bool withMetaManipulations, bool reloadDefault )
+        => Penumbra.Framework.RegisterImportant( nameof( CalculateEffectiveFileList ) + Name,
+            () => CalculateEffectiveFileListInternal( withMetaManipulations, reloadDefault ) );
+
+    private void CalculateEffectiveFileListInternal( bool withMetaManipulations, bool reloadDefault )
     {
         // Skip the empty collection.
         if( Index == 0 )
@@ -83,7 +85,7 @@ public partial class ModCollection
             return;
         }
 
-        PluginLog.Debug( "Recalculating effective file list for {CollectionName:l} [{WithMetaManipulations}] [{ReloadDefault}]", Name,
+        PluginLog.Debug( "[{Thread}] Recalculating effective file list for {CollectionName:l} [{WithMetaManipulations}] [{ReloadDefault}]", Thread.CurrentThread.ManagedThreadId, Name,
             withMetaManipulations, reloadDefault );
         _cache ??= new Cache( this );
         _cache.CalculateEffectiveFileList( withMetaManipulations );
@@ -92,6 +94,7 @@ public partial class ModCollection
             SetFiles();
             Penumbra.ResidentResources.Reload();
         }
+        PluginLog.Debug( "[{Thread}] Recalculation of effective file list for {CollectionName:l} finished.", Thread.CurrentThread.ManagedThreadId, Name);
     }
 
     // Set Metadata files.

Penumbra/Collections/ModCollection.Cache.cs

@@ -1,7 +1,10 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using Dalamud.Logging;
+using Dalamud.Utility;
 using Penumbra.GameData.ByteString;
 using Penumbra.Meta.Manager;
 using Penumbra.Meta.Manipulations;
@@ -15,15 +18,12 @@ public partial class ModCollection
     // It will only be setup if a collection gets activated in any way.
     private class Cache : IDisposable
     {
-        // Shared caches to avoid allocations.
+        private readonly Dictionary< Utf8GamePath, FileRegister >     _registeredFiles         = new();
-        private static readonly Dictionary< Utf8GamePath, FileRegister >     RegisteredFiles         = new(1024);
+        private readonly Dictionary< MetaManipulation, FileRegister > _registeredManipulations = new();
-        private static readonly Dictionary< MetaManipulation, FileRegister > RegisteredManipulations = new(1024);
-        private static readonly List< ModSettings? >                         ResolvedSettings        = new(128);
 
         private readonly ModCollection                        _collection;
         private readonly SortedList< string, object? >        _changedItems = new();
         public readonly  Dictionary< Utf8GamePath, FullPath > ResolvedFiles = new();
-        public readonly  HashSet< FullPath >                  MissingFiles  = new();
         public readonly  MetaManager                          MetaManipulations;
         public           ConflictCache                        Conflicts = new();
 
@@ -96,14 +96,10 @@ public partial class ModCollection
         // Clear all local and global caches to prepare for recomputation.
         private void ClearStorageAndPrepare()
         {
-            ResolvedFiles.Clear();
-            MissingFiles.Clear();
-            RegisteredFiles.Clear();
             _changedItems.Clear();
-            ResolvedSettings.Clear();
+            _registeredFiles.EnsureCapacity( 2 * ResolvedFiles.Count );
+            ResolvedFiles.Clear();
             Conflicts.ClearFileConflicts();
-            // Obtains actual settings for this collection with all inheritances.
-            ResolvedSettings.AddRange( _collection.ActualSettings );
         }
 
         // Recalculate all file changes from current settings. Include all fixed custom redirects.
@@ -113,7 +109,7 @@ public partial class ModCollection
             ClearStorageAndPrepare();
             if( withManipulations )
             {
-                RegisteredManipulations.Clear();
+                _registeredManipulations.EnsureCapacity( 2 * MetaManipulations.Count );
                 MetaManipulations.Reset();
             }
 
@@ -125,6 +121,10 @@ public partial class ModCollection
 
             AddMetaFiles();
             ++RecomputeCounter;
+            _registeredFiles.Clear();
+            _registeredFiles.TrimExcess();
+            _registeredManipulations.Clear();
+            _registeredManipulations.TrimExcess();
         }
 
         // Identify and record all manipulated objects for this entire collection.
@@ -158,14 +158,14 @@ public partial class ModCollection
         // Inside the same mod, conflicts are not recorded.
         private void AddFile( Utf8GamePath path, FullPath file, FileRegister priority )
         {
-            if( RegisteredFiles.TryGetValue( path, out var register ) )
+            if( _registeredFiles.TryGetValue( path, out var register ) )
             {
                 if( register.SameMod( priority, out var less ) )
                 {
                     Conflicts.AddConflict( register.ModIdx, priority.ModIdx, register.ModPriority, priority.ModPriority, path );
                     if( less )
                     {
-                        RegisteredFiles[ path ] = priority;
+                        _registeredFiles[ path ] = priority;
                         ResolvedFiles[ path ]    = file;
                     }
                 }
@@ -176,14 +176,14 @@ public partial class ModCollection
                     // Do not add conflicts.
                     if( less )
                     {
-                        RegisteredFiles[ path ] = priority;
+                        _registeredFiles[ path ] = priority;
                         ResolvedFiles[ path ]    = file;
                     }
                 }
             }
             else // File not seen before, just add it.
             {
-                RegisteredFiles.Add( path, priority );
+                _registeredFiles.Add( path, priority );
                 ResolvedFiles.Add( path, file );
             }
         }
@@ -194,14 +194,14 @@ public partial class ModCollection
         // Inside the same mod, conflicts are not recorded.
         private void AddManipulation( MetaManipulation manip, FileRegister priority )
         {
-            if( RegisteredManipulations.TryGetValue( manip, out var register ) )
+            if( _registeredManipulations.TryGetValue( manip, out var register ) )
             {
                 if( register.SameMod( priority, out var less ) )
                 {
                     Conflicts.AddConflict( register.ModIdx, priority.ModIdx, register.ModPriority, priority.ModPriority, manip );
                     if( less )
                     {
-                        RegisteredManipulations[ manip ] = priority;
+                        _registeredManipulations[ manip ] = priority;
                         MetaManipulations.ApplyMod( manip, priority.ModIdx );
                     }
                 }
@@ -212,14 +212,14 @@ public partial class ModCollection
                     // Do not add conflicts.
                     if( less )
                     {
-                        RegisteredManipulations[ manip ] = priority;
+                        _registeredManipulations[ manip ] = priority;
                         MetaManipulations.ApplyMod( manip, priority.ModIdx );
                     }
                 }
             }
             else // Manipulation not seen before, just add it.
             {
-                RegisteredManipulations[ manip ] = priority;
+                _registeredManipulations[ manip ] = priority;
                 MetaManipulations.ApplyMod( manip, priority.ModIdx );
             }
         }
@@ -250,7 +250,7 @@ public partial class ModCollection
         // Add all files and possibly manipulations of a given mod according to its settings in this collection.
         private void AddMod( int modIdx, bool withManipulations )
         {
-            var settings = ResolvedSettings[ modIdx ];
+            var settings = _collection[ modIdx ].Settings;
             if( settings is not { Enabled: true } )
             {
                 return;
@@ -300,7 +300,7 @@ public partial class ModCollection
             Penumbra.Redirects.Apply( ResolvedFiles );
             foreach( var gamePath in ResolvedFiles.Keys )
             {
-                RegisteredFiles.Add( gamePath, new FileRegister( -1, int.MaxValue, 0, 0 ) );
+                _registeredFiles.Add( gamePath, new FileRegister( -1, int.MaxValue, 0, 0 ) );
             }
         }
 

Penumbra/Collections/ModCollection.File.cs

@@ -22,7 +22,7 @@ public partial class ModCollection
         => new(Path.Combine( CollectionDirectory, $"{Name.RemoveInvalidPathSymbols()}.json" ));
 
     // Custom serialization due to shared mod information across managers.
-    public void Save()
+    private void SaveCollection()
     {
         try
         {
@@ -71,6 +71,9 @@ public partial class ModCollection
         }
     }
 
+    public void Save()
+        => Penumbra.Framework.RegisterDelayed( nameof( SaveCollection ) + Name, SaveCollection );
+
     public void Delete()
     {
         if( Index == 0 )

Penumbra/Configuration.cs

@@ -64,7 +64,7 @@ public partial class Configuration : IPluginConfiguration
     }
 
     // Save the current configuration.
-    public void Save()
+    private void SaveConfiguration()
     {
         try
         {
@@ -76,6 +76,9 @@ public partial class Configuration : IPluginConfiguration
         }
     }
 
+    public void Save()
+        => Penumbra.Framework.RegisterDelayed( nameof( SaveConfiguration ), SaveConfiguration );
+
     // Add missing colors to the dictionary if necessary.
     private void AddColors( bool forceSave )
     {

Penumbra/Mods/Manager/Mod.Manager.Options.cs

@@ -426,7 +426,7 @@ public sealed partial class Mod
                 }
                 else
                 {
-                    IModGroup.SaveModGroup( mod._groups[ groupIdx ], mod.ModPath, groupIdx );
+                    IModGroup.SaveDelayed( mod._groups[ groupIdx ], mod.ModPath, groupIdx );
                 }
             }
 

Penumbra/Mods/Mod.Creation.cs

@@ -72,7 +72,7 @@ public partial class Mod
                     Priority    = priority,
                 };
                 group.PrioritizedOptions.AddRange( subMods.OfType< SubMod >().Select( ( s, idx ) => ( s, idx ) ) );
-                IModGroup.SaveModGroup( group, baseFolder, index );
+                IModGroup.Save( group, baseFolder, index );
                 break;
             }
             case SelectType.Single:
@@ -84,7 +84,7 @@ public partial class Mod
                     Priority    = priority,
                 };
                 group.OptionData.AddRange( subMods.OfType< SubMod >() );
-                IModGroup.SaveModGroup( group, baseFolder, index );
+                IModGroup.Save( group, baseFolder, index );
                 break;
             }
         }

Penumbra/Mods/Mod.Files.cs

@@ -147,7 +147,7 @@ public partial class Mod
 
         foreach( var (group, index) in _groups.WithIndex() )
         {
-            IModGroup.SaveModGroup( group, ModPath, index );
+            IModGroup.SaveDelayed( group, ModPath, index );
         }
     }
 }

Penumbra/Mods/Mod.Meta.Migration.cs

@@ -83,7 +83,7 @@ public sealed partial class Mod
             mod._default.IncorporateMetaChanges( mod.ModPath, true );
             foreach( var (group, index) in mod.Groups.WithIndex() )
             {
-                IModGroup.SaveModGroup( group, mod.ModPath, index );
+                IModGroup.Save( group, mod.ModPath, index );
             }
 
             // Delete meta files.

Penumbra/Mods/Mod.Meta.cs

@@ -114,6 +114,9 @@ public sealed partial class Mod
     }
 
     private void SaveMeta()
+        => Penumbra.Framework.RegisterDelayed( nameof( SaveMetaFile ) + ModPath.Name, SaveMetaFile );
+
+    private void SaveMetaFile()
     {
         var metaFile = MetaFile;
         try

Penumbra/Mods/ModFileSystem.cs

@@ -15,12 +15,15 @@ public sealed class ModFileSystem : FileSystem< Mod >, IDisposable
     // Save the current sort order.
     // Does not save or copy the backup in the current mod directory,
     // as this is done on mod directory changes only.
-    private void Save()
+    private void SaveFilesystem()
     {
         SaveToFile( new FileInfo( ModFileSystemFile ), SaveMod, true );
         PluginLog.Verbose( "Saved mod filesystem." );
     }
 
+    private void Save()
+        => Penumbra.Framework.RegisterDelayed( nameof( SaveFilesystem ), SaveFilesystem );
+
     // Create a new ModFileSystem from the currently loaded mods and the current sort order file.
     public static ModFileSystem Load()
     {
@@ -50,6 +53,7 @@ public sealed class ModFileSystem : FileSystem< Mod >, IDisposable
         {
             Save();
         }
+
         PluginLog.Debug( "Reloaded mod filesystem." );
     }
 
@@ -98,6 +102,7 @@ public sealed class ModFileSystem : FileSystem< Mod >, IDisposable
                 {
                     Delete( leaf );
                 }
+
                 break;
             case ModPathChangeType.Moved:
                 Save();

Penumbra/Mods/Subclasses/IModGroup.cs

@@ -57,7 +57,15 @@ public interface IModGroup : IEnumerable< ISubMod >
         }
     }
 
-    public static void SaveModGroup( IModGroup group, DirectoryInfo basePath, int groupIdx )
+    public static void SaveDelayed( IModGroup group, DirectoryInfo basePath, int groupIdx )
+    {
+        Penumbra.Framework.RegisterDelayed( $"{nameof( SaveModGroup )}_{basePath.Name}_{group.Name}", () => SaveModGroup( group, basePath, groupIdx ) );
+    }
+
+    public static void Save( IModGroup group, DirectoryInfo basePath, int groupIdx )
+        => SaveModGroup( group, basePath, groupIdx );
+
+    private static void SaveModGroup( IModGroup group, DirectoryInfo basePath, int groupIdx )
     {
         var       file       = group.FileName( basePath, groupIdx );
         using var s          = File.Exists( file ) ? File.Open( file, FileMode.Truncate ) : File.Open( file, FileMode.CreateNew );

Penumbra/Penumbra.cs

@@ -44,6 +44,7 @@ public class Penumbra : IDalamudPlugin
     public static ModCollection.Manager CollectionManager { get; private set; } = null!;
     public static SimpleRedirectManager Redirects { get; private set; } = null!;
     public static ResourceLoader ResourceLoader { get; private set; } = null!;
+    public static FrameworkManager Framework { get; private set; } = null!;
 
 
     public readonly  ResourceLogger ResourceLogger;
@@ -62,6 +63,7 @@ public class Penumbra : IDalamudPlugin
     public Penumbra( DalamudPluginInterface pluginInterface )
     {
         Dalamud.Initialize( pluginInterface );
+        Framework = new FrameworkManager();
         GameData.GameData.GetIdentifier( Dalamud.GameData, Dalamud.ClientState.ClientLanguage );
         Backup.CreateBackup( PenumbraBackupFiles() );
         Config = Configuration.Load();

Penumbra/Util/FrameworkManager.cs

@@ -0,0 +1,75 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Dalamud.Game;
+
+namespace Penumbra.Util;
+
+// Manage certain actions to only occur on framework updates.
+public class FrameworkManager : IDisposable
+{
+    private readonly Dictionary< string, Action > _important = new();
+    private readonly Dictionary< string, Action > _delayed   = new();
+
+    public FrameworkManager()
+        => Dalamud.Framework.Update += OnUpdate;
+
+    // Register an action that is not time critical.
+    // One action per frame will be executed.
+    // On dispose, any remaining actions will be executed.
+    public void RegisterDelayed( string tag, Action action )
+        => _delayed[ tag ] = action;
+
+    // Register an action that should be executed on the next frame.
+    // All of those actions will be executed in the next frame.
+    // If there are more than one, they will be launched in separated tasks, but waited for.
+    public void RegisterImportant( string tag, Action action )
+        => _important[ tag ] = action;
+
+    public void Dispose()
+    {
+        Dalamud.Framework.Update -= OnUpdate;
+        HandleAll( _delayed );
+    }
+
+    private void OnUpdate( Framework _ )
+    {
+        HandleOne();
+        HandleAllTasks( _important );
+    }
+
+    private void HandleOne()
+    {
+        if( _delayed.Count > 0 )
+        {
+            var (key, action) = _delayed.First();
+            action();
+            _delayed.Remove( key );
+        }
+    }
+
+    private static void HandleAll( IDictionary< string, Action > dict )
+    {
+        foreach( var (_, action) in dict )
+        {
+            action();
+        }
+
+        dict.Clear();
+    }
+
+    private static void HandleAllTasks( IDictionary< string, Action > dict )
+    {
+        if( dict.Count < 2 )
+        {
+            HandleAll( dict );
+        }
+        else
+        {
+            var tasks = dict.Values.Select( Task.Run ).ToArray();
+            Task.WaitAll( tasks );
+            dict.Clear();
+        }
+    }
+}