Add Filesystem Compression as a toggle and button. Also some auto-formatting.

OtterGui

@@ -1 +1 @@
-Subproject commit 51ae60322c22c9d9b49365ad0b9fd60dc3d50296
+Subproject commit e3e3f42f093b53ad02694810398df5736174d711

Penumbra/Configuration.cs

@@ -99,6 +99,7 @@ public class Configuration : IPluginConfiguration, ISavable
     public bool PrintSuccessfulCommandsToChat { get; set; } = true;
     public bool FixMainWindow                 { get; set; } = false;
     public bool AutoDeduplicateOnImport       { get; set; } = true;
+    public bool UseFileSystemCompression      { get; set; } = true;
     public bool EnableHttpApi                 { get; set; } = true;
 
     public string DefaultModImportPath    { get; set; } = string.Empty;

Penumbra/Import/TexToolsImport.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -7,9 +6,10 @@ using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
-using Penumbra.Api;
+using OtterGui.Compression;
 using Penumbra.Import.Structs;
 using Penumbra.Mods;
+using Penumbra.Mods.Editor;
 using Penumbra.Mods.Manager;
 using FileMode = System.IO.FileMode;
 using ZipArchive = SharpCompress.Archives.Zip.ZipArchive;
@@ -25,41 +25,41 @@ public partial class TexToolsImporter : IDisposable
     private readonly DirectoryInfo _baseDirectory;
     private readonly string        _tmpFile;
 
-    private readonly IEnumerable< FileInfo > _modPackFiles;
+    private readonly IEnumerable<FileInfo>   _modPackFiles;
     private readonly int                     _modPackCount;
     private          FileStream?             _tmpFileStream;
     private          StreamDisposer?         _streamDisposer;
     private readonly CancellationTokenSource _cancellation = new();
     private readonly CancellationToken       _token;
 
-    public ImporterState State { get; private set; }
+    public          ImporterState                                               State { get; private set; }
-    public readonly List< (FileInfo File, DirectoryInfo? Mod, Exception? Error) > ExtractedMods;
+    public readonly List<(FileInfo File, DirectoryInfo? Mod, Exception? Error)> ExtractedMods;
 
     private readonly Configuration _config;
     private readonly ModEditor     _editor;
-    private readonly ModManager   _modManager;
+    private readonly ModManager    _modManager;
+    private readonly FileCompactor _compactor;
 
-    public TexToolsImporter( int count, IEnumerable< FileInfo > modPackFiles,
+    public TexToolsImporter(int count, IEnumerable<FileInfo> modPackFiles, Action<FileInfo, DirectoryInfo?, Exception?> handler,
-        Action< FileInfo, DirectoryInfo?, Exception? > handler, Configuration config, ModEditor editor, ModManager modManager)
+        Configuration config, ModEditor editor, ModManager modManager, FileCompactor compactor)
     {
         _baseDirectory = modManager.BasePath;
-        _tmpFile       = Path.Combine( _baseDirectory.FullName, TempFileName );
+        _tmpFile       = Path.Combine(_baseDirectory.FullName, TempFileName);
         _modPackFiles  = modPackFiles;
         _config        = config;
         _editor        = editor;
         _modManager    = modManager;
+        _compactor     = compactor;
         _modPackCount  = count;
-        ExtractedMods  = new List< (FileInfo, DirectoryInfo?, Exception?) >( count );
+        ExtractedMods  = new List<(FileInfo, DirectoryInfo?, Exception?)>(count);
         _token         = _cancellation.Token;
-        Task.Run( ImportFiles, _token )
+        Task.Run(ImportFiles, _token)
-           .ContinueWith( _ => CloseStreams() )
+            .ContinueWith(_ => CloseStreams())
-           .ContinueWith( _ =>
+            .ContinueWith(_ =>
             {
-                foreach( var (file, dir, error) in ExtractedMods )
+                foreach (var (file, dir, error) in ExtractedMods)
-                {
+                    handler(file, dir, error);
-                    handler( file, dir, error );
+            });
-                }
-            } );
     }
 
     private void CloseStreams()
@@ -71,45 +71,43 @@ public partial class TexToolsImporter : IDisposable
 
     public void Dispose()
     {
-        _cancellation.Cancel( true );
+        _cancellation.Cancel(true);
-        if( State != ImporterState.WritingPackToDisk )
+        if (State != ImporterState.WritingPackToDisk)
         {
             _tmpFileStream?.Dispose();
             _tmpFileStream = null;
         }
 
-        if( State != ImporterState.ExtractingModFiles )
+        if (State != ImporterState.ExtractingModFiles)
-        {
             ResetStreamDisposer();
-        }
     }
 
     private void ImportFiles()
     {
-        State                    = ImporterState.None;
+        State              = ImporterState.None;
-        _currentModPackIdx       = 0;
+        _currentModPackIdx = 0;
-        foreach( var file in _modPackFiles )
+        foreach (var file in _modPackFiles)
         {
             _currentModDirectory = null;
-            if( _token.IsCancellationRequested )
+            if (_token.IsCancellationRequested)
             {
-                ExtractedMods.Add( ( file, null, new TaskCanceledException( "Task canceled by user." ) ) );
+                ExtractedMods.Add((file, null, new TaskCanceledException("Task canceled by user.")));
                 continue;
             }
 
             try
             {
-                var directory = VerifyVersionAndImport( file );
+                var directory = VerifyVersionAndImport(file);
-                ExtractedMods.Add( ( file, directory, null ) );
+                ExtractedMods.Add((file, directory, null));
-                if( _config.AutoDeduplicateOnImport )
+                if (_config.AutoDeduplicateOnImport)
                 {
                     State = ImporterState.DeduplicatingFiles;
-                    _editor.Duplicates.DeduplicateMod( directory );
+                    _editor.Duplicates.DeduplicateMod(directory);
                 }
             }
-            catch( Exception e )
+            catch (Exception e)
             {
-                ExtractedMods.Add( ( file, _currentModDirectory, e ) );
+                ExtractedMods.Add((file, _currentModDirectory, e));
                 _currentNumOptions = 0;
                 _currentOptionIdx  = 0;
                 _currentFileIdx    = 0;
@@ -124,87 +122,75 @@ public partial class TexToolsImporter : IDisposable
 
     // Rudimentary analysis of a TTMP file by extension and version.
     // Puts out warnings if extension does not correspond to data.
-    private DirectoryInfo VerifyVersionAndImport( FileInfo modPackFile )
+    private DirectoryInfo VerifyVersionAndImport(FileInfo modPackFile)
     {
-        if( modPackFile.Extension.ToLowerInvariant() is ".pmp" or ".zip" or ".7z" or ".rar" )
+        if (modPackFile.Extension.ToLowerInvariant() is ".pmp" or ".zip" or ".7z" or ".rar")
-        {
+            return HandleRegularArchive(modPackFile);
-            return HandleRegularArchive( modPackFile );
-        }
 
         using var zfs              = modPackFile.OpenRead();
-        using var extractedModPack = ZipArchive.Open( zfs );
+        using var extractedModPack = ZipArchive.Open(zfs);
 
-        var mpl = FindZipEntry( extractedModPack, "TTMPL.mpl" );
+        var mpl = FindZipEntry(extractedModPack, "TTMPL.mpl");
-        if( mpl == null )
+        if (mpl == null)
-        {
+            throw new FileNotFoundException("ZIP does not contain a TTMPL.mpl file.");
-            throw new FileNotFoundException( "ZIP does not contain a TTMPL.mpl file." );
-        }
 
-        var modRaw = GetStringFromZipEntry( mpl, Encoding.UTF8 );
+        var modRaw = GetStringFromZipEntry(mpl, Encoding.UTF8);
 
         // At least a better validation than going by the extension.
-        if( modRaw.Contains( "\"TTMPVersion\":" ) )
+        if (modRaw.Contains("\"TTMPVersion\":"))
         {
-            if( modPackFile.Extension != ".ttmp2" )
+            if (modPackFile.Extension != ".ttmp2")
-            {
+                Penumbra.Log.Warning($"File {modPackFile.FullName} seems to be a V2 TTMP, but has the wrong extension.");
-                Penumbra.Log.Warning( $"File {modPackFile.FullName} seems to be a V2 TTMP, but has the wrong extension." );
-            }
 
-            return ImportV2ModPack( modPackFile, extractedModPack, modRaw );
+            return ImportV2ModPack(modPackFile, extractedModPack, modRaw);
         }
 
-        if( modPackFile.Extension != ".ttmp" )
+        if (modPackFile.Extension != ".ttmp")
-        {
+            Penumbra.Log.Warning($"File {modPackFile.FullName} seems to be a V1 TTMP, but has the wrong extension.");
-            Penumbra.Log.Warning( $"File {modPackFile.FullName} seems to be a V1 TTMP, but has the wrong extension." );
-        }
 
-        return ImportV1ModPack( modPackFile, extractedModPack, modRaw );
+        return ImportV1ModPack(modPackFile, extractedModPack, modRaw);
     }
 
     // You can in no way rely on any file paths in TTMPs so we need to just do this, sorry
-    private static ZipArchiveEntry? FindZipEntry( ZipArchive file, string fileName )
+    private static ZipArchiveEntry? FindZipEntry(ZipArchive file, string fileName)
-        => file.Entries.FirstOrDefault( e => !e.IsDirectory && e.Key.Contains( fileName ) );
+        => file.Entries.FirstOrDefault(e => !e.IsDirectory && e.Key.Contains(fileName));
 
-    private static string GetStringFromZipEntry( ZipArchiveEntry entry, Encoding encoding )
+    private static string GetStringFromZipEntry(ZipArchiveEntry entry, Encoding encoding)
     {
         using var ms = new MemoryStream();
         using var s  = entry.OpenEntryStream();
-        s.CopyTo( ms );
+        s.CopyTo(ms);
-        return encoding.GetString( ms.ToArray() );
+        return encoding.GetString(ms.ToArray());
     }
 
-    private void WriteZipEntryToTempFile( Stream s )
+    private void WriteZipEntryToTempFile(Stream s)
     {
         _tmpFileStream?.Dispose(); // should not happen
-        _tmpFileStream = new FileStream( _tmpFile, FileMode.Create );
+        _tmpFileStream = new FileStream(_tmpFile, FileMode.Create);
-        if( _token.IsCancellationRequested )
+        if (_token.IsCancellationRequested)
-        {
             return;
-        }
 
-        s.CopyTo( _tmpFileStream );
+        s.CopyTo(_tmpFileStream);
         _tmpFileStream.Dispose();
         _tmpFileStream = null;
     }
 
-    private StreamDisposer GetSqPackStreamStream( ZipArchive file, string entryName )
+    private StreamDisposer GetSqPackStreamStream(ZipArchive file, string entryName)
     {
         State = ImporterState.WritingPackToDisk;
 
         // write shitty zip garbage to disk
-        var entry = FindZipEntry( file, entryName );
+        var entry = FindZipEntry(file, entryName);
-        if( entry == null )
+        if (entry == null)
-        {
+            throw new FileNotFoundException($"ZIP does not contain a file named {entryName}.");
-            throw new FileNotFoundException( $"ZIP does not contain a file named {entryName}." );
-        }
 
         using var s = entry.OpenEntryStream();
 
-        WriteZipEntryToTempFile( s );
+        WriteZipEntryToTempFile(s);
 
         _streamDisposer?.Dispose(); // Should not happen.
-        var fs = new FileStream( _tmpFile, FileMode.Open );
+        var fs = new FileStream(_tmpFile, FileMode.Open);
-        return new StreamDisposer( fs );
+        return new StreamDisposer(fs);
     }
 
     private void ResetStreamDisposer()

Penumbra/Import/TexToolsImporter.Gui.cs

@@ -22,7 +22,6 @@ public partial class TexToolsImporter
     private string _currentOptionName = string.Empty;
     private string _currentFileName   = string.Empty;
 
-
     public void DrawProgressInfo( Vector2 size )
     {
         if( _modPackCount == 0 )

Penumbra/Import/TexToolsImporter.ModPack.cs

@@ -17,7 +17,7 @@ public partial class TexToolsImporter
     private DirectoryInfo? _currentModDirectory;
 
     // Version 1 mod packs are a simple collection of files without much information.
-    private DirectoryInfo ImportV1ModPack( FileInfo modPackFile, ZipArchive extractedModPack, string modRaw )
+    private DirectoryInfo ImportV1ModPack(FileInfo modPackFile, ZipArchive extractedModPack, string modRaw)
     {
         _currentOptionIdx  = 0;
         _currentNumOptions = 1;
@@ -25,174 +25,171 @@ public partial class TexToolsImporter
         _currentGroupName  = string.Empty;
         _currentOptionName = DefaultTexToolsData.DefaultOption;
 
-        Penumbra.Log.Information( "    -> Importing V1 ModPack" );
+        Penumbra.Log.Information("    -> Importing V1 ModPack");
 
         var modListRaw = modRaw.Split(
-            new[] { "\r\n", "\r", "\n" },
+            new[]
+            {
+                "\r\n",
+                "\r",
+                "\n",
+            },
             StringSplitOptions.RemoveEmptyEntries
         );
 
-        var modList = modListRaw.Select( m => JsonConvert.DeserializeObject< SimpleMod >( m, JsonSettings )! ).ToList();
+        var modList = modListRaw.Select(m => JsonConvert.DeserializeObject<SimpleMod>(m, JsonSettings)!).ToList();
 
-        _currentModDirectory = ModCreator.CreateModFolder( _baseDirectory, Path.GetFileNameWithoutExtension( modPackFile.Name ) );
+        _currentModDirectory = ModCreator.CreateModFolder(_baseDirectory, Path.GetFileNameWithoutExtension(modPackFile.Name));
         // Create a new ModMeta from the TTMP mod list info
-        _modManager.DataEditor.CreateMeta( _currentModDirectory, _currentModName, DefaultTexToolsData.Author, DefaultTexToolsData.Description, null, null );
+        _modManager.DataEditor.CreateMeta(_currentModDirectory, _currentModName, DefaultTexToolsData.Author, DefaultTexToolsData.Description,
+            null, null);
 
         // Open the mod data file from the mod pack as a SqPackStream
-        _streamDisposer = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" );
+        _streamDisposer = GetSqPackStreamStream(extractedModPack, "TTMPD.mpd");
-        ExtractSimpleModList( _currentModDirectory, modList );
+        ExtractSimpleModList(_currentModDirectory, modList);
-        _modManager.Creator.CreateDefaultFiles( _currentModDirectory );
+        _modManager.Creator.CreateDefaultFiles(_currentModDirectory);
         ResetStreamDisposer();
         return _currentModDirectory;
     }
 
     // Version 2 mod packs can either be simple or extended, import accordingly.
-    private DirectoryInfo ImportV2ModPack( FileInfo _, ZipArchive extractedModPack, string modRaw )
+    private DirectoryInfo ImportV2ModPack(FileInfo _, ZipArchive extractedModPack, string modRaw)
     {
-        var modList = JsonConvert.DeserializeObject< SimpleModPack >( modRaw, JsonSettings )!;
+        var modList = JsonConvert.DeserializeObject<SimpleModPack>(modRaw, JsonSettings)!;
 
-        if( modList.TtmpVersion.EndsWith( "s" ) )
+        if (modList.TtmpVersion.EndsWith("s"))
-        {
+            return ImportSimpleV2ModPack(extractedModPack, modList);
-            return ImportSimpleV2ModPack( extractedModPack, modList );
-        }
 
-        if( modList.TtmpVersion.EndsWith( "w" ) )
+        if (modList.TtmpVersion.EndsWith("w"))
-        {
+            return ImportExtendedV2ModPack(extractedModPack, modRaw);
-            return ImportExtendedV2ModPack( extractedModPack, modRaw );
-        }
 
         try
         {
-            Penumbra.Log.Warning( $"Unknown TTMPVersion <{modList.TtmpVersion}> given, trying to export as simple mod pack." );
+            Penumbra.Log.Warning($"Unknown TTMPVersion <{modList.TtmpVersion}> given, trying to export as simple mod pack.");
-            return ImportSimpleV2ModPack( extractedModPack, modList );
+            return ImportSimpleV2ModPack(extractedModPack, modList);
         }
-        catch( Exception e1 )
+        catch (Exception e1)
         {
-            Penumbra.Log.Warning( $"Exporting as simple mod pack failed with following error, retrying as extended mod pack:\n{e1}" );
+            Penumbra.Log.Warning($"Exporting as simple mod pack failed with following error, retrying as extended mod pack:\n{e1}");
             try
             {
-                return ImportExtendedV2ModPack( extractedModPack, modRaw );
+                return ImportExtendedV2ModPack(extractedModPack, modRaw);
             }
-            catch( Exception e2 )
+            catch (Exception e2)
             {
-                throw new IOException( "Exporting as extended mod pack failed, too. Version unsupported or file defect.", e2 );
+                throw new IOException("Exporting as extended mod pack failed, too. Version unsupported or file defect.", e2);
             }
         }
     }
 
     // Simple V2 mod packs are basically the same as V1 mod packs.
-    private DirectoryInfo ImportSimpleV2ModPack( ZipArchive extractedModPack, SimpleModPack modList )
+    private DirectoryInfo ImportSimpleV2ModPack(ZipArchive extractedModPack, SimpleModPack modList)
     {
         _currentOptionIdx  = 0;
         _currentNumOptions = 1;
         _currentModName    = modList.Name;
         _currentGroupName  = string.Empty;
         _currentOptionName = DefaultTexToolsData.DefaultOption;
-        Penumbra.Log.Information( "    -> Importing Simple V2 ModPack" );
+        Penumbra.Log.Information("    -> Importing Simple V2 ModPack");
 
-        _currentModDirectory = ModCreator.CreateModFolder( _baseDirectory, _currentModName );
+        _currentModDirectory = ModCreator.CreateModFolder(_baseDirectory, _currentModName);
-        _modManager.DataEditor.CreateMeta( _currentModDirectory, _currentModName, modList.Author, string.IsNullOrEmpty( modList.Description )
+        _modManager.DataEditor.CreateMeta(_currentModDirectory, _currentModName, modList.Author, string.IsNullOrEmpty(modList.Description)
             ? "Mod imported from TexTools mod pack"
-            : modList.Description, modList.Version, modList.Url );
+            : modList.Description, modList.Version, modList.Url);
 
         // Open the mod data file from the mod pack as a SqPackStream
-        _streamDisposer = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" );
+        _streamDisposer = GetSqPackStreamStream(extractedModPack, "TTMPD.mpd");
-        ExtractSimpleModList( _currentModDirectory, modList.SimpleModsList );
+        ExtractSimpleModList(_currentModDirectory, modList.SimpleModsList);
-        _modManager.Creator.CreateDefaultFiles( _currentModDirectory );
+        _modManager.Creator.CreateDefaultFiles(_currentModDirectory);
         ResetStreamDisposer();
         return _currentModDirectory;
     }
 
     // Obtain the number of relevant options to extract.
-    private static int GetOptionCount( ExtendedModPack pack )
+    private static int GetOptionCount(ExtendedModPack pack)
-        => ( pack.SimpleModsList.Length > 0 ? 1 : 0 )
+        => (pack.SimpleModsList.Length > 0 ? 1 : 0)
           + pack.ModPackPages
-               .Sum( page => page.ModGroups
+                .Sum(page => page.ModGroups
-                   .Where( g => g.GroupName.Length > 0 && g.OptionList.Length > 0 )
+                    .Where(g => g.GroupName.Length > 0 && g.OptionList.Length > 0)
-                   .Sum( group => group.OptionList
+                    .Sum(group => group.OptionList
-                           .Count( o => o.Name.Length > 0 && o.ModsJsons.Length > 0 )
+                            .Count(o => o.Name.Length > 0 && o.ModsJsons.Length > 0)
-                      + ( group.OptionList.Any( o => o.Name.Length > 0 && o.ModsJsons.Length == 0 ) ? 1 : 0 ) ) );
+                      + (group.OptionList.Any(o => o.Name.Length > 0 && o.ModsJsons.Length == 0) ? 1 : 0)));
 
-    private static string GetGroupName( string groupName, ISet< string > names )
+    private static string GetGroupName(string groupName, ISet<string> names)
     {
         var baseName = groupName;
         var i        = 2;
-        while( !names.Add( groupName ) )
+        while (!names.Add(groupName))
-        {
             groupName = $"{baseName} ({i++})";
-        }
 
         return groupName;
     }
 
     // Extended V2 mod packs contain multiple options that need to be handled separately.
-    private DirectoryInfo ImportExtendedV2ModPack( ZipArchive extractedModPack, string modRaw )
+    private DirectoryInfo ImportExtendedV2ModPack(ZipArchive extractedModPack, string modRaw)
     {
         _currentOptionIdx = 0;
-        Penumbra.Log.Information( "    -> Importing Extended V2 ModPack" );
+        Penumbra.Log.Information("    -> Importing Extended V2 ModPack");
 
-        var modList = JsonConvert.DeserializeObject< ExtendedModPack >( modRaw, JsonSettings )!;
+        var modList = JsonConvert.DeserializeObject<ExtendedModPack>(modRaw, JsonSettings)!;
-        _currentNumOptions = GetOptionCount( modList );
+        _currentNumOptions = GetOptionCount(modList);
         _currentModName    = modList.Name;
 
-        _currentModDirectory = ModCreator.CreateModFolder( _baseDirectory, _currentModName );
+        _currentModDirectory = ModCreator.CreateModFolder(_baseDirectory, _currentModName);
-        _modManager.DataEditor.CreateMeta( _currentModDirectory, _currentModName, modList.Author, modList.Description, modList.Version, modList.Url );
+        _modManager.DataEditor.CreateMeta(_currentModDirectory, _currentModName, modList.Author, modList.Description, modList.Version,
+            modList.Url);
 
-        if( _currentNumOptions == 0 )
+        if (_currentNumOptions == 0)
-        {
             return _currentModDirectory;
-        }
 
         // Open the mod data file from the mod pack as a SqPackStream
-        _streamDisposer = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" );
+        _streamDisposer = GetSqPackStreamStream(extractedModPack, "TTMPD.mpd");
 
         // It can contain a simple list, still.
-        if( modList.SimpleModsList.Length > 0 )
+        if (modList.SimpleModsList.Length > 0)
         {
             _currentGroupName  = string.Empty;
             _currentOptionName = "Default";
-            ExtractSimpleModList( _currentModDirectory, modList.SimpleModsList );
+            ExtractSimpleModList(_currentModDirectory, modList.SimpleModsList);
         }
 
         // Iterate through all pages
-        var options       = new List< ISubMod >();
+        var options       = new List<ISubMod>();
         var groupPriority = 0;
-        var groupNames    = new HashSet< string >();
+        var groupNames    = new HashSet<string>();
-        foreach( var page in modList.ModPackPages )
+        foreach (var page in modList.ModPackPages)
         {
-            foreach( var group in page.ModGroups.Where( group => group.GroupName.Length > 0 && group.OptionList.Length > 0 ) )
+            foreach (var group in page.ModGroups.Where(group => group.GroupName.Length > 0 && group.OptionList.Length > 0))
             {
-                var allOptions = group.OptionList.Where( option => option.Name.Length > 0 && option.ModsJsons.Length > 0 ).ToList();
+                var allOptions = group.OptionList.Where(option => option.Name.Length > 0 && option.ModsJsons.Length > 0).ToList();
                 var (numGroups, maxOptions) = group.SelectionType == GroupType.Single
-                    ? ( 1, allOptions.Count )
+                    ? (1, allOptions.Count)
-                    : ( 1 + allOptions.Count / IModGroup.MaxMultiOptions, IModGroup.MaxMultiOptions );
+                    : (1 + allOptions.Count / IModGroup.MaxMultiOptions, IModGroup.MaxMultiOptions);
-                _currentGroupName = GetGroupName( group.GroupName, groupNames );
+                _currentGroupName = GetGroupName(group.GroupName, groupNames);
 
-                var optionIdx       = 0;
+                var optionIdx = 0;
-                for( var groupId = 0; groupId < numGroups; ++groupId )
+                for (var groupId = 0; groupId < numGroups; ++groupId)
                 {
-                    var name           = numGroups == 1 ? _currentGroupName : $"{_currentGroupName}, Part {groupId + 1}";
+                    var name = numGroups == 1 ? _currentGroupName : $"{_currentGroupName}, Part {groupId + 1}";
                     options.Clear();
-                    var groupFolder = ModCreator.NewSubFolderName( _currentModDirectory, name )
+                    var groupFolder = ModCreator.NewSubFolderName(_currentModDirectory, name)
-                     ?? new DirectoryInfo( Path.Combine( _currentModDirectory.FullName,
+                     ?? new DirectoryInfo(Path.Combine(_currentModDirectory.FullName,
-                            numGroups == 1 ? $"Group {groupPriority + 1}" : $"Group {groupPriority + 1}, Part {groupId + 1}" ) );
+                            numGroups == 1 ? $"Group {groupPriority + 1}" : $"Group {groupPriority + 1}, Part {groupId + 1}"));
 
                     uint? defaultSettings = group.SelectionType == GroupType.Multi ? 0u : null;
-                    for( var i = 0; i + optionIdx < allOptions.Count && i < maxOptions; ++i )
+                    for (var i = 0; i + optionIdx < allOptions.Count && i < maxOptions; ++i)
                     {
-                        var option = allOptions[ i + optionIdx ];
+                        var option = allOptions[i + optionIdx];
                         _token.ThrowIfCancellationRequested();
                         _currentOptionName = option.Name;
-                        var optionFolder = ModCreator.NewSubFolderName( groupFolder, option.Name )
+                        var optionFolder = ModCreator.NewSubFolderName(groupFolder, option.Name)
-                         ?? new DirectoryInfo( Path.Combine( groupFolder.FullName, $"Option {i + optionIdx + 1}" ) );
+                         ?? new DirectoryInfo(Path.Combine(groupFolder.FullName, $"Option {i + optionIdx + 1}"));
-                        ExtractSimpleModList( optionFolder, option.ModsJsons );
+                        ExtractSimpleModList(optionFolder, option.ModsJsons);
-                        options.Add( _modManager.Creator.CreateSubMod( _currentModDirectory, optionFolder, option ) );
+                        options.Add(_modManager.Creator.CreateSubMod(_currentModDirectory, optionFolder, option));
-                        if( option.IsChecked )
+                        if (option.IsChecked)
-                        {
                             defaultSettings = group.SelectionType == GroupType.Multi
-                                ? ( defaultSettings!.Value | ( 1u << i ) )
+                                ? defaultSettings!.Value | (1u << i)
-                                : ( uint )i;
+                                : (uint)i;
-                        }
 
                         ++_currentOptionIdx;
                     }
@@ -201,30 +198,30 @@ public partial class TexToolsImporter
 
                     // Handle empty options for single select groups without creating a folder for them.
                     // We only want one of those at most, and it should usually be the first option.
-                    if( group.SelectionType == GroupType.Single )
+                    if (group.SelectionType == GroupType.Single)
                     {
-                        var empty = group.OptionList.FirstOrDefault( o => o.Name.Length > 0 && o.ModsJsons.Length == 0 );
+                        var empty = group.OptionList.FirstOrDefault(o => o.Name.Length > 0 && o.ModsJsons.Length == 0);
-                        if( empty != null )
+                        if (empty != null)
                         {
                             _currentOptionName = empty.Name;
-                            options.Insert( 0, ModCreator.CreateEmptySubMod( empty.Name ) );
+                            options.Insert(0, ModCreator.CreateEmptySubMod(empty.Name));
                             defaultSettings = defaultSettings == null ? 0 : defaultSettings.Value + 1;
                         }
                     }
 
-                    _modManager.Creator.CreateOptionGroup( _currentModDirectory, group.SelectionType, name, groupPriority, groupPriority,
+                    _modManager.Creator.CreateOptionGroup(_currentModDirectory, group.SelectionType, name, groupPriority, groupPriority,
-                        defaultSettings ?? 0, group.Description, options );
+                        defaultSettings ?? 0, group.Description, options);
                     ++groupPriority;
                 }
             }
         }
 
         ResetStreamDisposer();
-        _modManager.Creator.CreateDefaultFiles( _currentModDirectory );
+        _modManager.Creator.CreateDefaultFiles(_currentModDirectory);
         return _currentModDirectory;
     }
 
-    private void ExtractSimpleModList( DirectoryInfo outDirectory, ICollection< SimpleMod > mods )
+    private void ExtractSimpleModList(DirectoryInfo outDirectory, ICollection<SimpleMod> mods)
     {
         State = ImporterState.ExtractingModFiles;
 
@@ -232,51 +229,47 @@ public partial class TexToolsImporter
         _currentNumFiles = mods.Count(m => m.FullPath.Length > 0);
 
         // Extract each SimpleMod into the new mod folder
-        foreach( var simpleMod in mods.Where(m => m.FullPath.Length > 0 ) )
+        foreach (var simpleMod in mods.Where(m => m.FullPath.Length > 0))
         {
-            ExtractMod( outDirectory, simpleMod );
+            ExtractMod(outDirectory, simpleMod);
             ++_currentFileIdx;
         }
     }
 
-    private void ExtractMod( DirectoryInfo outDirectory, SimpleMod mod )
+    private void ExtractMod(DirectoryInfo outDirectory, SimpleMod mod)
     {
-        if( _streamDisposer is not PenumbraSqPackStream stream )
+        if (_streamDisposer is not PenumbraSqPackStream stream)
-        {
             return;
-        }
 
-        Penumbra.Log.Information( $"        -> Extracting {mod.FullPath} at {mod.ModOffset:X}" );
+        Penumbra.Log.Information($"        -> Extracting {mod.FullPath} at {mod.ModOffset:X}");
 
         _token.ThrowIfCancellationRequested();
-        var data = stream.ReadFile< PenumbraSqPackStream.PenumbraFileResource >( mod.ModOffset );
+        var data = stream.ReadFile<PenumbraSqPackStream.PenumbraFileResource>(mod.ModOffset);
 
         _currentFileName = mod.FullPath;
-        var extractedFile = new FileInfo( Path.Combine( outDirectory.FullName, mod.FullPath ) );
+        var extractedFile = new FileInfo(Path.Combine(outDirectory.FullName, mod.FullPath));
 
         extractedFile.Directory?.Create();
 
-        if( extractedFile.FullName.EndsWith( ".mdl" ) )
+        if (extractedFile.FullName.EndsWith(".mdl"))
-        {
+            ProcessMdl(data.Data);
-            ProcessMdl( data.Data );
-        }
 
-        File.WriteAllBytes( extractedFile.FullName, data.Data );
+        _compactor.WriteAllBytesAsync(extractedFile.FullName, data.Data, _token).Wait(_token);
     }
 
-    private static void ProcessMdl( byte[] mdl )
+    private static void ProcessMdl(byte[] mdl)
     {
         const int modelHeaderLodOffset = 22;
 
         // Model file header LOD num
-        mdl[ 64 ] = 1;
+        mdl[64] = 1;
 
         // Model header LOD num
-        var stackSize           = BitConverter.ToUInt32( mdl, 4 );
+        var stackSize           = BitConverter.ToUInt32(mdl, 4);
-        var runtimeBegin        = stackSize    + 0x44;
+        var runtimeBegin        = stackSize + 0x44;
         var stringsLengthOffset = runtimeBegin + 4;
-        var stringsLength       = BitConverter.ToUInt32( mdl, ( int )stringsLengthOffset );
+        var stringsLength       = BitConverter.ToUInt32(mdl, (int)stringsLengthOffset);
         var modelHeaderStart    = stringsLengthOffset + stringsLength + 4;
-        mdl[ modelHeaderStart + modelHeaderLodOffset ] = 1;
+        mdl[modelHeaderStart + modelHeaderLodOffset] = 1;
     }
 }

Penumbra/Import/TexToolsMeta.Export.cs

@@ -23,7 +23,7 @@ public partial class TexToolsMeta
             try
             {
                 Directory.CreateDirectory(Path.GetDirectoryName(path)!);
-                File.WriteAllBytes(path, data);
+                manager.Compactor.WriteAllBytes(path, data);
             }
             catch (Exception e)
             {
@@ -32,191 +32,187 @@ public partial class TexToolsMeta
         }
     }
 
-    public static Dictionary< string, byte[] > ConvertToTexTools( MetaFileManager manager, IEnumerable< MetaManipulation > manips )
+    public static Dictionary<string, byte[]> ConvertToTexTools(MetaFileManager manager, IEnumerable<MetaManipulation> manips)
     {
-        var ret = new Dictionary< string, byte[] >();
+        var ret = new Dictionary<string, byte[]>();
-        foreach( var group in manips.GroupBy( ManipToPath ) )
+        foreach (var group in manips.GroupBy(ManipToPath))
         {
-            if( group.Key.Length == 0 )
+            if (group.Key.Length == 0)
-            {
                 continue;
-            }
 
-            var bytes = group.Key.EndsWith( ".rgsp" )
+            var bytes = group.Key.EndsWith(".rgsp")
-                ? WriteRgspFile( manager, group.Key, group )
+                ? WriteRgspFile(manager, group.Key, group)
-                : WriteMetaFile( manager, group.Key, group );
+                : WriteMetaFile(manager, group.Key, group);
-            if( bytes.Length == 0 )
+            if (bytes.Length == 0)
-            {
                 continue;
-            }
 
-            ret.Add( group.Key, bytes );
+            ret.Add(group.Key, bytes);
         }
 
         return ret;
     }
 
-    private static byte[] WriteRgspFile( MetaFileManager manager, string path, IEnumerable< MetaManipulation > manips )
+    private static byte[] WriteRgspFile(MetaFileManager manager, string path, IEnumerable<MetaManipulation> manips)
     {
-        var       list = manips.GroupBy( m => m.Rsp.Attribute ).ToDictionary( m => m.Key, m => m.Last().Rsp );
+        var       list = manips.GroupBy(m => m.Rsp.Attribute).ToDictionary(m => m.Key, m => m.Last().Rsp);
-        using var m    = new MemoryStream( 45 );
+        using var m    = new MemoryStream(45);
-        using var b    = new BinaryWriter( m );
+        using var b    = new BinaryWriter(m);
         // Version
-        b.Write( byte.MaxValue );
+        b.Write(byte.MaxValue);
-        b.Write( ( ushort )2 );
+        b.Write((ushort)2);
 
         var race   = list.First().Value.SubRace;
         var gender = list.First().Value.Attribute.ToGender();
-        b.Write( ( byte )(race   - 1) ); // offset by one due to Unknown
+        b.Write((byte)(race - 1));   // offset by one due to Unknown
-        b.Write( ( byte )(gender - 1) ); // offset by one due to Unknown
+        b.Write((byte)(gender - 1)); // offset by one due to Unknown
 
-        void Add( params RspAttribute[] attributes )
+        void Add(params RspAttribute[] attributes)
         {
-            foreach( var attribute in attributes )
+            foreach (var attribute in attributes)
             {
-                var value = list.TryGetValue( attribute, out var tmp ) ? tmp.Entry : CmpFile.GetDefault( manager, race, attribute );
+                var value = list.TryGetValue(attribute, out var tmp) ? tmp.Entry : CmpFile.GetDefault(manager, race, attribute);
-                b.Write( value );
+                b.Write(value);
             }
         }
 
-        if( gender == Gender.Male )
+        if (gender == Gender.Male)
         {
-            Add( RspAttribute.MaleMinSize, RspAttribute.MaleMaxSize, RspAttribute.MaleMinTail, RspAttribute.MaleMaxTail );
+            Add(RspAttribute.MaleMinSize, RspAttribute.MaleMaxSize, RspAttribute.MaleMinTail, RspAttribute.MaleMaxTail);
         }
         else
         {
-            Add( RspAttribute.FemaleMinSize, RspAttribute.FemaleMaxSize, RspAttribute.FemaleMinTail, RspAttribute.FemaleMaxTail );
+            Add(RspAttribute.FemaleMinSize, RspAttribute.FemaleMaxSize, RspAttribute.FemaleMinTail, RspAttribute.FemaleMaxTail);
-            Add( RspAttribute.BustMinX, RspAttribute.BustMinY, RspAttribute.BustMinZ, RspAttribute.BustMaxX, RspAttribute.BustMaxY, RspAttribute.BustMaxZ );
+            Add(RspAttribute.BustMinX, RspAttribute.BustMinY, RspAttribute.BustMinZ, RspAttribute.BustMaxX, RspAttribute.BustMaxY,
+                RspAttribute.BustMaxZ);
         }
 
         return m.GetBuffer();
     }
 
-    private static byte[] WriteMetaFile( MetaFileManager manager, string path, IEnumerable< MetaManipulation > manips )
+    private static byte[] WriteMetaFile(MetaFileManager manager, string path, IEnumerable<MetaManipulation> manips)
     {
-        var filteredManips = manips.GroupBy( m => m.ManipulationType ).ToDictionary( p => p.Key, p => p.Select( x => x ) );
+        var filteredManips = manips.GroupBy(m => m.ManipulationType).ToDictionary(p => p.Key, p => p.Select(x => x));
 
         using var m = new MemoryStream();
-        using var b = new BinaryWriter( m );
+        using var b = new BinaryWriter(m);
 
         // Header
         // Current TT Metadata version.
-        b.Write( 2u );
+        b.Write(2u);
 
         // Null-terminated ASCII path.
-        var utf8Path = Encoding.ASCII.GetBytes( path );
+        var utf8Path = Encoding.ASCII.GetBytes(path);
-        b.Write( utf8Path );
+        b.Write(utf8Path);
-        b.Write( ( byte )0 );
+        b.Write((byte)0);
 
         // Number of Headers
-        b.Write( ( uint )filteredManips.Count );
+        b.Write((uint)filteredManips.Count);
         // Current TT Size of Headers
-        b.Write( ( uint )12 );
+        b.Write((uint)12);
 
         // Start of Header Entries for some reason, which is absolutely useless.
         var headerStart = b.BaseStream.Position + 4;
-        b.Write( ( uint )headerStart );
+        b.Write((uint)headerStart);
 
-        var offset = ( uint )( b.BaseStream.Position + 12 * filteredManips.Count );
+        var offset = (uint)(b.BaseStream.Position + 12 * filteredManips.Count);
-        foreach( var (header, data) in filteredManips )
+        foreach (var (header, data) in filteredManips)
         {
-            b.Write( ( uint )header );
+            b.Write((uint)header);
-            b.Write( offset );
+            b.Write(offset);
 
-            var size = WriteData( manager, b, offset, header, data );
+            var size = WriteData(manager, b, offset, header, data);
-            b.Write( size );
+            b.Write(size);
             offset += size;
         }
 
         return m.ToArray();
     }
 
-    private static uint WriteData( MetaFileManager manager, BinaryWriter b, uint offset, MetaManipulation.Type type, IEnumerable< MetaManipulation > manips )
+    private static uint WriteData(MetaFileManager manager, BinaryWriter b, uint offset, MetaManipulation.Type type,
+        IEnumerable<MetaManipulation> manips)
     {
         var oldPos = b.BaseStream.Position;
-        b.Seek( ( int )offset, SeekOrigin.Begin );
+        b.Seek((int)offset, SeekOrigin.Begin);
 
-        switch( type )
+        switch (type)
         {
             case MetaManipulation.Type.Imc:
                 var allManips = manips.ToList();
-                var baseFile  = new ImcFile( manager, allManips[ 0 ].Imc );
+                var baseFile  = new ImcFile(manager, allManips[0].Imc);
-                foreach( var manip in allManips )
+                foreach (var manip in allManips)
-                {
+                    manip.Imc.Apply(baseFile);
-                    manip.Imc.Apply( baseFile );
-                }
 
-                var partIdx = allManips[ 0 ].Imc.ObjectType is ObjectType.Equipment or ObjectType.Accessory
+                var partIdx = allManips[0].Imc.ObjectType is ObjectType.Equipment or ObjectType.Accessory
-                    ? ImcFile.PartIndex( allManips[ 0 ].Imc.EquipSlot )
+                    ? ImcFile.PartIndex(allManips[0].Imc.EquipSlot)
                     : 0;
 
-                for( var i = 0; i <= baseFile.Count; ++i )
+                for (var i = 0; i <= baseFile.Count; ++i)
                 {
-                    var entry = baseFile.GetEntry( partIdx, (Variant)i );
+                    var entry = baseFile.GetEntry(partIdx, (Variant)i);
-                    b.Write( entry.MaterialId );
+                    b.Write(entry.MaterialId);
-                    b.Write( entry.DecalId );
+                    b.Write(entry.DecalId);
-                    b.Write( entry.AttributeAndSound );
+                    b.Write(entry.AttributeAndSound);
-                    b.Write( entry.VfxId );
+                    b.Write(entry.VfxId);
-                    b.Write( entry.MaterialAnimationId );
+                    b.Write(entry.MaterialAnimationId);
                 }
 
                 break;
             case MetaManipulation.Type.Eqdp:
-                foreach( var manip in manips )
+                foreach (var manip in manips)
                 {
-                    b.Write( ( uint )Names.CombinedRace( manip.Eqdp.Gender, manip.Eqdp.Race ) );
+                    b.Write((uint)Names.CombinedRace(manip.Eqdp.Gender, manip.Eqdp.Race));
-                    var entry = ( byte )(( ( uint )manip.Eqdp.Entry >> Eqdp.Offset( manip.Eqdp.Slot ) ) & 0x03);
+                    var entry = (byte)(((uint)manip.Eqdp.Entry >> Eqdp.Offset(manip.Eqdp.Slot)) & 0x03);
-                    b.Write( entry );
+                    b.Write(entry);
                 }
 
                 break;
             case MetaManipulation.Type.Eqp:
-                foreach( var manip in manips )
+                foreach (var manip in manips)
                 {
-                    var bytes = BitConverter.GetBytes( (ulong) manip.Eqp.Entry );
+                    var bytes = BitConverter.GetBytes((ulong)manip.Eqp.Entry);
-                    var (numBytes, byteOffset) = Eqp.BytesAndOffset( manip.Eqp.Slot );
+                    var (numBytes, byteOffset) = Eqp.BytesAndOffset(manip.Eqp.Slot);
-                    for( var i = byteOffset; i < numBytes + byteOffset; ++i )
+                    for (var i = byteOffset; i < numBytes + byteOffset; ++i)
-                        b.Write( bytes[ i ] );
+                        b.Write(bytes[i]);
                 }
 
                 break;
             case MetaManipulation.Type.Est:
-                foreach( var manip in manips )
+                foreach (var manip in manips)
                 {
-                    b.Write( ( ushort )Names.CombinedRace( manip.Est.Gender, manip.Est.Race ) );
+                    b.Write((ushort)Names.CombinedRace(manip.Est.Gender, manip.Est.Race));
-                    b.Write( manip.Est.SetId.Id );
+                    b.Write(manip.Est.SetId.Id);
-                    b.Write( manip.Est.Entry );
+                    b.Write(manip.Est.Entry);
                 }
 
                 break;
             case MetaManipulation.Type.Gmp:
-                foreach( var manip in manips )
+                foreach (var manip in manips)
                 {
-                    b.Write( ( uint )manip.Gmp.Entry.Value );
+                    b.Write((uint)manip.Gmp.Entry.Value);
-                    b.Write( manip.Gmp.Entry.UnknownTotal );
+                    b.Write(manip.Gmp.Entry.UnknownTotal);
                 }
 
                 break;
         }
 
         var size = b.BaseStream.Position - offset;
-        b.Seek( ( int )oldPos, SeekOrigin.Begin );
+        b.Seek((int)oldPos, SeekOrigin.Begin);
-        return ( uint )size;
+        return (uint)size;
     }
 
-    private static string ManipToPath( MetaManipulation manip )
+    private static string ManipToPath(MetaManipulation manip)
         => manip.ManipulationType switch
         {
-            MetaManipulation.Type.Imc  => ManipToPath( manip.Imc ),
+            MetaManipulation.Type.Imc  => ManipToPath(manip.Imc),
-            MetaManipulation.Type.Eqdp => ManipToPath( manip.Eqdp ),
+            MetaManipulation.Type.Eqdp => ManipToPath(manip.Eqdp),
-            MetaManipulation.Type.Eqp  => ManipToPath( manip.Eqp ),
+            MetaManipulation.Type.Eqp  => ManipToPath(manip.Eqp),
-            MetaManipulation.Type.Est  => ManipToPath( manip.Est ),
+            MetaManipulation.Type.Est  => ManipToPath(manip.Est),
-            MetaManipulation.Type.Gmp  => ManipToPath( manip.Gmp ),
+            MetaManipulation.Type.Gmp  => ManipToPath(manip.Gmp),
-            MetaManipulation.Type.Rsp  => ManipToPath( manip.Rsp ),
+            MetaManipulation.Type.Rsp  => ManipToPath(manip.Rsp),
             _                          => string.Empty,
         };
 
-    private static string ManipToPath( ImcManipulation manip )
+    private static string ManipToPath(ImcManipulation manip)
     {
         var path = manip.GamePath().ToString();
         var replacement = manip.ObjectType switch
@@ -227,22 +223,22 @@ public partial class TexToolsMeta
             _                    => ".meta",
         };
 
-        return path.Replace( ".imc", replacement );
+        return path.Replace(".imc", replacement);
     }
 
-    private static string ManipToPath( EqdpManipulation manip )
+    private static string ManipToPath(EqdpManipulation manip)
         => manip.Slot.IsAccessory()
             ? $"chara/accessory/a{manip.SetId:D4}/a{manip.SetId:D4}_{manip.Slot.ToSuffix()}.meta"
             : $"chara/equipment/e{manip.SetId:D4}/e{manip.SetId:D4}_{manip.Slot.ToSuffix()}.meta";
 
-    private static string ManipToPath( EqpManipulation manip )
+    private static string ManipToPath(EqpManipulation manip)
         => manip.Slot.IsAccessory()
             ? $"chara/accessory/a{manip.SetId:D4}/a{manip.SetId:D4}_{manip.Slot.ToSuffix()}.meta"
             : $"chara/equipment/e{manip.SetId:D4}/e{manip.SetId:D4}_{manip.Slot.ToSuffix()}.meta";
 
-    private static string ManipToPath( EstManipulation manip )
+    private static string ManipToPath(EstManipulation manip)
     {
-        var raceCode = Names.CombinedRace( manip.Gender, manip.Race ).ToRaceCode();
+        var raceCode = Names.CombinedRace(manip.Gender, manip.Race).ToRaceCode();
         return manip.Slot switch
         {
             EstManipulation.EstType.Hair => $"chara/human/c{raceCode}/obj/hair/h{manip.SetId:D4}/c{raceCode}h{manip.SetId:D4}_hir.meta",
@@ -253,10 +249,10 @@ public partial class TexToolsMeta
         };
     }
 
-    private static string ManipToPath( GmpManipulation manip )
+    private static string ManipToPath(GmpManipulation manip)
         => $"chara/equipment/e{manip.SetId:D4}/e{manip.SetId:D4}_{EquipSlot.Head.ToSuffix()}.meta";
 
 
-    private static string ManipToPath( RspManipulation manip )
+    private static string ManipToPath(RspManipulation manip)
-        => $"chara/xls/charamake/rgsp/{( int )manip.SubRace - 1}-{( int )manip.Attribute.ToGender() - 1}.rgsp";
+        => $"chara/xls/charamake/rgsp/{(int)manip.SubRace - 1}-{(int)manip.Attribute.ToGender() - 1}.rgsp";
 }

Penumbra/Import/Textures/TextureDrawer.cs

@@ -10,7 +10,7 @@ using OtterGui;
 using OtterGui.Raii;
 using OtterGui.Widgets;
 using OtterTex;
-using Penumbra.Mods;
+using Penumbra.Mods.Editor;
 using Penumbra.String.Classes;
 using Penumbra.UI;
 using Penumbra.UI.Classes;

Penumbra/Meta/MetaFileManager.cs

@@ -4,6 +4,7 @@ using System.Runtime.CompilerServices;
 using Dalamud.Plugin.Services;
 using Dalamud.Utility.Signatures;
 using FFXIVClientStructs.FFXIV.Client.System.Memory;
+using OtterGui.Compression;
 using Penumbra.Collections;
 using Penumbra.Collections.Manager;
 using Penumbra.GameData;
@@ -26,9 +27,11 @@ public unsafe class MetaFileManager
     internal readonly ActiveCollectionData    ActiveCollections;
     internal readonly ValidityChecker         ValidityChecker;
     internal readonly IdentifierService       Identifier;
+    internal readonly FileCompactor           Compactor;
 
     public MetaFileManager(CharacterUtility characterUtility, ResidentResourceManager residentResources, IDataManager gameData,
-        ActiveCollectionData activeCollections, Configuration config, ValidityChecker validityChecker, IdentifierService identifier)
+        ActiveCollectionData activeCollections, Configuration config, ValidityChecker validityChecker, IdentifierService identifier,
+        FileCompactor compactor)
     {
         CharacterUtility  = characterUtility;
         ResidentResources = residentResources;
@@ -37,6 +40,7 @@ public unsafe class MetaFileManager
         Config            = config;
         ValidityChecker   = validityChecker;
         Identifier        = identifier;
+        Compactor         = compactor;
         SignatureHelper.Initialise(this);
     }
 

Penumbra/Mods/Editor/MdlMaterialEditor.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
 using OtterGui;
+using OtterGui.Compression;
 using Penumbra.GameData.Enums;
 using Penumbra.GameData.Files;
 using Penumbra.Mods.Editor;
@@ -26,10 +27,10 @@ public partial class MdlMaterialEditor
     public MdlMaterialEditor(ModFileCollection files)
         => _files = files;
 
-    public void SaveAllModels()
+    public void SaveAllModels(FileCompactor compactor)
     {
         foreach (var info in _modelFiles)
-            info.Save();
+            info.Save(compactor);
     }
 
     public void RestoreAllModels()

Penumbra/Mods/Editor/ModEditor.cs

@@ -1,10 +1,10 @@
 using System;
 using System.IO;
 using OtterGui;
-using Penumbra.Mods.Editor;
+using OtterGui.Compression;
 using Penumbra.Mods.Subclasses;
 
-namespace Penumbra.Mods;
+namespace Penumbra.Mods.Editor;
 
 public class ModEditor : IDisposable
 {
@@ -15,6 +15,7 @@ public class ModEditor : IDisposable
     public readonly ModFileCollection Files;
     public readonly ModSwapEditor     SwapEditor;
     public readonly MdlMaterialEditor MdlMaterialEditor;
+    public readonly FileCompactor     Compactor;
 
     public Mod? Mod       { get; private set; }
     public int  GroupIdx  { get; private set; }
@@ -24,7 +25,8 @@ public class ModEditor : IDisposable
     public ISubMod?   Option { get; private set; }
 
     public ModEditor(ModNormalizer modNormalizer, ModMetaEditor metaEditor, ModFileCollection files,
-        ModFileEditor fileEditor, DuplicateManager duplicates, ModSwapEditor swapEditor, MdlMaterialEditor mdlMaterialEditor)
+        ModFileEditor fileEditor, DuplicateManager duplicates, ModSwapEditor swapEditor, MdlMaterialEditor mdlMaterialEditor,
+        FileCompactor compactor)
     {
         ModNormalizer     = modNormalizer;
         MetaEditor        = metaEditor;
@@ -33,6 +35,7 @@ public class ModEditor : IDisposable
         Duplicates        = duplicates;
         SwapEditor        = swapEditor;
         MdlMaterialEditor = mdlMaterialEditor;
+        Compactor         = compactor;
     }
 
     public void LoadMod(Mod mod)

Penumbra/Mods/Editor/ModelMaterialInfo.cs

@@ -2,10 +2,11 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using OtterGui;
+using OtterGui.Compression;
 using Penumbra.GameData.Files;
 using Penumbra.String.Classes;
 
-namespace Penumbra.Mods;
+namespace Penumbra.Mods.Editor;
 
 /// <summary> A class that collects information about skin materials in a model file and handle changes on them. </summary>
 public class ModelMaterialInfo
@@ -40,7 +41,7 @@ public class ModelMaterialInfo
     }
 
     // Save a changed .mdl file.
-    public void Save()
+    public void Save(FileCompactor compactor)
     {
         if (!Changed)
             return;
@@ -50,7 +51,7 @@ public class ModelMaterialInfo
 
         try
         {
-            System.IO.File.WriteAllBytes(Path.FullName, File.Write());
+            compactor.WriteAllBytes(Path.FullName, File.Write());
             Changed = false;
         }
         catch (Exception e)

Penumbra/Mods/ItemSwap/ItemSwapContainer.cs

@@ -63,7 +63,6 @@ public class ItemSwapContainer
                             continue;
                         }
 
-
                         if( writeType == WriteType.UseSwaps && file.SwapToModdedExistsInGame && !file.DataWasChanged )
                         {
                             convertedSwaps.TryAdd( file.SwapFromRequestPath, file.SwapToModded );
@@ -73,7 +72,7 @@ public class ItemSwapContainer
                             var path  = file.GetNewPath( directory.FullName );
                             var bytes = file.FileData.Write();
                             Directory.CreateDirectory( Path.GetDirectoryName( path )! );
-                            File.WriteAllBytes( path, bytes );
+                            _manager.Compactor.WriteAllBytes( path, bytes );
                             convertedFiles.TryAdd( file.SwapFromRequestPath, new FullPath( path ) );
                         }
 

Penumbra/Mods/Manager/ModImportManager.cs

@@ -7,6 +7,7 @@ using System.IO;
 using System.Linq;
 using Dalamud.Interface.Internal.Notifications;
 using Penumbra.Import;
+using Penumbra.Mods.Editor;
 
 namespace Penumbra.Mods.Manager;
 
@@ -57,7 +58,7 @@ public class ModImportManager : IDisposable
         if (files.Length == 0)
             return;
 
-        _import = new TexToolsImporter(files.Length, files, AddNewMod, _config, _modEditor, _modManager);
+        _import = new TexToolsImporter(files.Length, files, AddNewMod, _config, _modEditor, _modManager, _modEditor.Compactor);
     }
 
     public bool Importing

Penumbra/Services/ServiceManager.cs

@@ -1,6 +1,7 @@
 using Dalamud.Plugin;
 using Microsoft.Extensions.DependencyInjection;
 using OtterGui.Classes;
+using OtterGui.Compression;
 using OtterGui.Log;
 using Penumbra.Api;
 using Penumbra.Collections.Cache;
@@ -62,7 +63,8 @@ public static class ServiceManager
             .AddSingleton<BackupService>()
             .AddSingleton<CommunicatorService>()
             .AddSingleton<ChatService>()
-            .AddSingleton<SaveService>();
+            .AddSingleton<SaveService>()
+            .AddSingleton<FileCompactor>();
 
 
     private static IServiceCollection AddGameData(this IServiceCollection services)

Penumbra/UI/AdvancedWindow/FileEditor.cs

@@ -9,6 +9,7 @@ using Dalamud.Plugin.Services;
 using ImGuiNET;
 using OtterGui;
 using OtterGui.Classes;
+using OtterGui.Compression;
 using OtterGui.Raii;
 using OtterGui.Widgets;
 using Penumbra.GameData.Files;
@@ -21,11 +22,12 @@ namespace Penumbra.UI.AdvancedWindow;
 public class FileEditor<T> : IDisposable where T : class, IWritable
 {
     private readonly FileDialogService _fileDialog;
-    private readonly IDataManager       _gameData;
+    private readonly IDataManager      _gameData;
     private readonly ModEditWindow     _owner;
+    private readonly FileCompactor     _compactor;
 
-    public FileEditor(ModEditWindow owner, IDataManager gameData, Configuration config, FileDialogService fileDialog, string tabName,
+    public FileEditor(ModEditWindow owner, IDataManager gameData, Configuration config, FileCompactor compactor, FileDialogService fileDialog,
-        string fileType, Func<IReadOnlyList<FileRegistry>> getFiles, Func<T, bool, bool> drawEdit, Func<string> getInitialPath,
+        string tabName, string fileType, Func<IReadOnlyList<FileRegistry>> getFiles, Func<T, bool, bool> drawEdit, Func<string> getInitialPath,
         Func<byte[], string, bool, T?> parseFile)
     {
         _owner          = owner;
@@ -36,6 +38,7 @@ public class FileEditor<T> : IDisposable where T : class, IWritable
         _drawEdit       = drawEdit;
         _getInitialPath = getInitialPath;
         _parseFile      = parseFile;
+        _compactor      = compactor;
         _combo          = new Combo(config, getFiles);
     }
 
@@ -108,8 +111,8 @@ public class FileEditor<T> : IDisposable where T : class, IWritable
                 {
                     _defaultException = null;
                     (_defaultFile as IDisposable)?.Dispose();
-                    _defaultFile      = null; // Avoid double disposal if an exception occurs during the parsing of the new file.
+                    _defaultFile = null; // Avoid double disposal if an exception occurs during the parsing of the new file.
-                    _defaultFile      = _parseFile(file.Data, _defaultPath, false);
+                    _defaultFile = _parseFile(file.Data, _defaultPath, false);
                 }
                 else
                 {
@@ -135,7 +138,7 @@ public class FileEditor<T> : IDisposable where T : class, IWritable
 
                     try
                     {
-                        File.WriteAllBytes(name, _defaultFile?.Write() ?? throw new Exception("File invalid."));
+                        _compactor.WriteAllBytes(name, _defaultFile?.Write() ?? throw new Exception("File invalid."));
                     }
                     catch (Exception e)
                     {
@@ -169,8 +172,8 @@ public class FileEditor<T> : IDisposable where T : class, IWritable
         _currentException = null;
         _currentPath      = null;
         (_currentFile as IDisposable)?.Dispose();
-        _currentFile      = null;
+        _currentFile = null;
-        _changed          = false;
+        _changed     = false;
     }
 
     private void DrawFileSelectCombo()
@@ -209,7 +212,7 @@ public class FileEditor<T> : IDisposable where T : class, IWritable
         if (ImGuiUtil.DrawDisabledButton("Save to File", Vector2.Zero,
                 $"Save the selected {_fileType} file with all changes applied. This is not revertible.", !_changed))
         {
-            File.WriteAllBytes(_currentPath!.File.FullName, _currentFile!.Write());
+            _compactor.WriteAllBytes(_currentPath!.File.FullName, _currentFile!.Write());
             _changed = false;
         }
     }

Penumbra/UI/AdvancedWindow/ModEditWindow.Materials.cs

@@ -195,7 +195,7 @@ public partial class ModEditWindow
             ImGui.TableNextColumn();
             if (ImGuiUtil.DrawDisabledButton(FontAwesomeIcon.Save.ToIconString(), iconSize,
                     "Save the changed mdl file.\nUse at own risk!", !info.Changed, true))
-                info.Save();
+                info.Save(_editor.Compactor);
 
             ImGui.TableNextColumn();
             if (ImGuiUtil.DrawDisabledButton(FontAwesomeIcon.Recycle.ToIconString(), iconSize,

Penumbra/UI/AdvancedWindow/ModEditWindow.Meta.cs

@@ -13,6 +13,7 @@ using Penumbra.Meta;
 using Penumbra.Meta.Files;
 using Penumbra.Meta.Manipulations;
 using Penumbra.Mods;
+using Penumbra.Mods.Editor;
 using Penumbra.UI.Classes;
 
 namespace Penumbra.UI.AdvancedWindow;

Penumbra/UI/AdvancedWindow/ModEditWindow.QuickImport.cs

@@ -11,6 +11,7 @@ using OtterGui.Raii;
 using Penumbra.GameData.Files;
 using Penumbra.Interop.ResourceTree;
 using Penumbra.Mods;
+using Penumbra.Mods.Editor;
 using Penumbra.String.Classes;
 
 namespace Penumbra.UI.AdvancedWindow;
@@ -72,7 +73,7 @@ public partial class ModEditWindow
 
                     try
                     {
-                        File.WriteAllBytes(name, writable!.Write());
+                        _editor.Compactor.WriteAllBytes(name, writable!.Write());
                     }
                     catch (Exception e)
                     {
@@ -194,7 +195,7 @@ public partial class ModEditWindow
             var directory = Path.GetDirectoryName(_targetPath);
             if (directory != null)
                 Directory.CreateDirectory(directory);
-            File.WriteAllBytes(_targetPath!, _file!.Write());
+            _editor.Compactor.WriteAllBytes(_targetPath!, _file!.Write());
             _editor.FileEditor.Revert(_editor.Mod!, _editor.Option!);
             var fileRegistry = _editor.Files.Available.First(file => file.File.FullName == _targetPath);
             _editor.FileEditor.AddPathsToSelected(_editor.Option!, new[]

Penumbra/UI/AdvancedWindow/ModEditWindow.cs

@@ -12,6 +12,7 @@ using Dalamud.Interface.Windowing;
 using Dalamud.Plugin.Services;
 using ImGuiNET;
 using OtterGui;
+using OtterGui.Compression;
 using OtterGui.Raii;
 using Penumbra.Collections.Manager;
 using Penumbra.Communication;
@@ -22,6 +23,7 @@ using Penumbra.Interop.ResourceTree;
 using Penumbra.Interop.Services;
 using Penumbra.Meta;
 using Penumbra.Mods;
+using Penumbra.Mods.Editor;
 using Penumbra.Mods.Manager;
 using Penumbra.Services;
 using Penumbra.String;
@@ -236,7 +238,7 @@ public partial class ModEditWindow : Window, IDisposable
             var anyChanges = editor.MdlMaterialEditor.ModelFiles.Any(m => m.Changed);
             if (ImGuiUtil.DrawDisabledButton("Save All Changes", buttonSize,
                     anyChanges ? "Irreversibly rewrites all currently applied changes to model files." : "No changes made yet.", !anyChanges))
-                editor.MdlMaterialEditor.SaveAllModels();
+                editor.MdlMaterialEditor.SaveAllModels(editor.Compactor);
 
             ImGui.SameLine();
             if (ImGuiUtil.DrawDisabledButton("Revert All Changes", buttonSize,
@@ -572,17 +574,18 @@ public partial class ModEditWindow : Window, IDisposable
         _textures          = textures;
         _fileDialog        = fileDialog;
         _gameEvents        = gameEvents;
-        _materialTab = new FileEditor<MtrlTab>(this, gameData, config, _fileDialog, "Materials", ".mtrl",
+        _materialTab = new FileEditor<MtrlTab>(this, gameData, config, _editor.Compactor, _fileDialog, "Materials", ".mtrl",
             () => _editor.Files.Mtrl, DrawMaterialPanel, () => _mod?.ModPath.FullName ?? string.Empty,
             (bytes, path, writable) => new MtrlTab(this, new MtrlFile(bytes), path, writable));
-        _modelTab = new FileEditor<MdlFile>(this, gameData, config, _fileDialog, "Models", ".mdl",
+        _modelTab = new FileEditor<MdlFile>(this, gameData, config, _editor.Compactor, _fileDialog, "Models", ".mdl",
             () => _editor.Files.Mdl, DrawModelPanel, () => _mod?.ModPath.FullName ?? string.Empty, (bytes, _, _) => new MdlFile(bytes));
-        _shaderPackageTab = new FileEditor<ShpkTab>(this, gameData, config, _fileDialog, "Shaders", ".shpk",
+        _shaderPackageTab = new FileEditor<ShpkTab>(this, gameData, config, _editor.Compactor, _fileDialog, "Shaders", ".shpk",
             () => _editor.Files.Shpk, DrawShaderPackagePanel, () => _mod?.ModPath.FullName ?? string.Empty,
             (bytes, _, _) => new ShpkTab(_fileDialog, bytes));
         _center             = new CombinedTexture(_left, _right);
         _textureSelectCombo = new TextureDrawer.PathSelectCombo(textures, editor);
-        _quickImportViewer  = new ResourceTreeViewer(_config, resourceTreeFactory, changedItemDrawer, 2, OnQuickImportRefresh, DrawQuickImportActions);
+        _quickImportViewer =
+            new ResourceTreeViewer(_config, resourceTreeFactory, changedItemDrawer, 2, OnQuickImportRefresh, DrawQuickImportActions);
         _communicator.ModPathChanged.Subscribe(OnModPathChanged, ModPathChanged.Priority.ModEditWindow);
     }
 

Penumbra/UI/ModsTab/ModPanelEditTab.cs

@@ -138,7 +138,7 @@ public class ModPanelEditTab : ITab
             _editor.LoadMod(_mod);
             _editor.MdlMaterialEditor.ReplaceAllMaterials("bibo",     "b");
             _editor.MdlMaterialEditor.ReplaceAllMaterials("bibopube", "c");
-            _editor.MdlMaterialEditor.SaveAllModels();
+            _editor.MdlMaterialEditor.SaveAllModels(_editor.Compactor);
             _editWindow.UpdateModels();
         }
 

Penumbra/UI/Tabs/SettingsTab.cs

@@ -7,6 +7,7 @@ using Dalamud.Interface.Components;
 using Dalamud.Utility;
 using ImGuiNET;
 using OtterGui;
+using OtterGui.Compression;
 using OtterGui.Custom;
 using OtterGui.Raii;
 using OtterGui.Widgets;
@@ -39,6 +40,7 @@ public class SettingsTab : ITab
     private readonly DalamudServices             _dalamud;
     private readonly HttpApi                     _httpApi;
     private readonly DalamudSubstitutionProvider _dalamudSubstitutionProvider;
+    private readonly FileCompactor               _compactor;
 
     private int _minimumX = int.MaxValue;
     private int _minimumY = int.MaxValue;
@@ -46,7 +48,7 @@ public class SettingsTab : ITab
     public SettingsTab(Configuration config, FontReloader fontReloader, TutorialService tutorial, Penumbra penumbra,
         FileDialogService fileDialog, ModManager modManager, ModFileSystemSelector selector, CharacterUtility characterUtility,
         ResidentResourceManager residentResources, DalamudServices dalamud, ModExportManager modExportManager, HttpApi httpApi,
-        DalamudSubstitutionProvider dalamudSubstitutionProvider)
+        DalamudSubstitutionProvider dalamudSubstitutionProvider, FileCompactor compactor)
     {
         _config                      = config;
         _fontReloader                = fontReloader;
@@ -61,6 +63,9 @@ public class SettingsTab : ITab
         _modExportManager            = modExportManager;
         _httpApi                     = httpApi;
         _dalamudSubstitutionProvider = dalamudSubstitutionProvider;
+        _compactor                   = compactor;
+        if (_compactor.CanCompact)
+            _compactor.Enabled = _config.UseFileSystemCompression;
     }
 
     public void DrawHeader()
@@ -661,6 +666,7 @@ public class SettingsTab : ITab
         Checkbox("Auto Deduplicate on Import",
             "Automatically deduplicate mod files on import. This will make mod file sizes smaller, but deletes (binary identical) files.",
             _config.AutoDeduplicateOnImport, v => _config.AutoDeduplicateOnImport = v);
+        DrawCompressionBox();
         Checkbox("Keep Default Metadata Changes on Import",
             "Normally, metadata changes that equal their default values, which are sometimes exported by TexTools, are discarded. "
           + "Toggle this to keep them, for example if an option in a mod is supposed to disable a metadata change from a prior option.",
@@ -673,6 +679,47 @@ public class SettingsTab : ITab
         ImGui.NewLine();
     }
 
+    private void DrawCompressionBox()
+    {
+        if (!_compactor.CanCompact)
+            return;
+
+        Checkbox("Use Filesystem Compression",
+            "Use Windows functionality to transparently reduce storage size of mod files on your computer. This might cost performance, but seems to generally be beneficial to performance by shifting more responsibility to the underused CPU and away from the overused hard drives.",
+            _config.UseFileSystemCompression,
+            v =>
+            {
+                _config.UseFileSystemCompression = v;
+                _compactor.Enabled               = v;
+            });
+        ImGui.SameLine();
+        if (ImGuiUtil.DrawDisabledButton("Compress Existing Files", Vector2.Zero,
+                "Try to compress all files in your root directory. This will take a while.",
+                _compactor.MassCompactRunning || !_modManager.Valid))
+            _compactor.StartMassCompact(_modManager.BasePath.EnumerateFiles("*.*", SearchOption.AllDirectories), CompressionAlgorithm.Xpress8K);
+
+        ImGui.SameLine();
+        if (ImGuiUtil.DrawDisabledButton("Decompress Existing Files", Vector2.Zero,
+                "Try to decompress all files in your root directory. This will take a while.",
+                _compactor.MassCompactRunning || !_modManager.Valid))
+            _compactor.StartMassCompact(_modManager.BasePath.EnumerateFiles("*.*", SearchOption.AllDirectories), CompressionAlgorithm.None);
+
+        if (_compactor.MassCompactRunning)
+        {
+            ImGui.ProgressBar((float)_compactor.CurrentIndex / _compactor.TotalFiles,
+                new Vector2(ImGui.GetContentRegionAvail().X - ImGui.GetStyle().ItemSpacing.X - UiHelpers.IconButtonSize.X, ImGui.GetFrameHeight()),
+                _compactor.CurrentFile?.FullName[(_modManager.BasePath.FullName.Length + 1)..] ?? "Gathering Files...");
+            ImGui.SameLine();
+            if (ImGuiUtil.DrawDisabledButton(FontAwesomeIcon.Ban.ToIconString(), UiHelpers.IconButtonSize, "Cancel the mass action.",
+                    !_compactor.MassCompactRunning, true))
+            _compactor.CancelMassCompact();
+        }
+        else
+        {
+            ImGui.Dummy(UiHelpers.IconButtonSize);
+        }
+    }
+
     /// <summary> Draw two integral inputs for minimum dimensions of this window. </summary>
     private void DrawMinimumDimensionConfig()
     {