From 4e704770cb2a4d6a15ad50445a9ac5972ca9af62 Mon Sep 17 00:00:00 2001 From: Ottermandias Date: Thu, 14 Sep 2023 17:23:54 +0200 Subject: [PATCH] Add Filesystem Compression as a toggle and button. Also some auto-formatting. --- OtterGui | 2 +- Penumbra/Configuration.cs | 1 + Penumbra/Import/TexToolsImport.cs | 146 ++++++------ Penumbra/Import/TexToolsImporter.Gui.cs | 1 - Penumbra/Import/TexToolsImporter.ModPack.cs | 215 +++++++++--------- Penumbra/Import/TexToolsMeta.Export.cs | 194 ++++++++-------- Penumbra/Import/Textures/TextureDrawer.cs | 2 +- Penumbra/Meta/MetaFileManager.cs | 8 +- Penumbra/Mods/Editor/MdlMaterialEditor.cs | 5 +- Penumbra/Mods/Editor/ModEditor.cs | 9 +- Penumbra/Mods/Editor/ModelMaterialInfo.cs | 7 +- Penumbra/Mods/ItemSwap/ItemSwapContainer.cs | 3 +- Penumbra/Mods/Manager/ModImportManager.cs | 3 +- Penumbra/Services/ServiceManager.cs | 4 +- Penumbra/UI/AdvancedWindow/FileEditor.cs | 57 ++--- .../AdvancedWindow/ModEditWindow.Materials.cs | 2 +- .../UI/AdvancedWindow/ModEditWindow.Meta.cs | 1 + .../ModEditWindow.QuickImport.cs | 5 +- Penumbra/UI/AdvancedWindow/ModEditWindow.cs | 13 +- Penumbra/UI/ModsTab/ModPanelEditTab.cs | 2 +- Penumbra/UI/Tabs/SettingsTab.cs | 49 +++- 21 files changed, 385 insertions(+), 344 deletions(-) diff --git a/OtterGui b/OtterGui index 51ae6032..e3e3f42f 160000 --- a/OtterGui +++ b/OtterGui @@ -1 +1 @@ -Subproject commit 51ae60322c22c9d9b49365ad0b9fd60dc3d50296 +Subproject commit e3e3f42f093b53ad02694810398df5736174d711 diff --git a/Penumbra/Configuration.cs b/Penumbra/Configuration.cs index cc7cc026..a80563c9 100644 --- a/Penumbra/Configuration.cs +++ b/Penumbra/Configuration.cs @@ -99,6 +99,7 @@ public class Configuration : IPluginConfiguration, ISavable public bool PrintSuccessfulCommandsToChat { get; set; } = true; public bool FixMainWindow { get; set; } = false; public bool AutoDeduplicateOnImport { get; set; } = true; + public bool UseFileSystemCompression { get; set; } = true; public bool EnableHttpApi { get; set; } = true; public string DefaultModImportPath { get; set; } = string.Empty; diff --git a/Penumbra/Import/TexToolsImport.cs b/Penumbra/Import/TexToolsImport.cs index dff1c921..49b344da 100644 --- a/Penumbra/Import/TexToolsImport.cs +++ b/Penumbra/Import/TexToolsImport.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -7,9 +6,10 @@ using System.Text; using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; -using Penumbra.Api; +using OtterGui.Compression; using Penumbra.Import.Structs; using Penumbra.Mods; +using Penumbra.Mods.Editor; using Penumbra.Mods.Manager; using FileMode = System.IO.FileMode; using ZipArchive = SharpCompress.Archives.Zip.ZipArchive; @@ -25,41 +25,41 @@ public partial class TexToolsImporter : IDisposable private readonly DirectoryInfo _baseDirectory; private readonly string _tmpFile; - private readonly IEnumerable< FileInfo > _modPackFiles; + private readonly IEnumerable _modPackFiles; private readonly int _modPackCount; private FileStream? _tmpFileStream; private StreamDisposer? _streamDisposer; private readonly CancellationTokenSource _cancellation = new(); private readonly CancellationToken _token; - public ImporterState State { get; private set; } - public readonly List< (FileInfo File, DirectoryInfo? Mod, Exception? Error) > ExtractedMods; + public ImporterState State { get; private set; } + public readonly List<(FileInfo File, DirectoryInfo? Mod, Exception? Error)> ExtractedMods; private readonly Configuration _config; private readonly ModEditor _editor; - private readonly ModManager _modManager; - - public TexToolsImporter( int count, IEnumerable< FileInfo > modPackFiles, - Action< FileInfo, DirectoryInfo?, Exception? > handler, Configuration config, ModEditor editor, ModManager modManager) + private readonly ModManager _modManager; + private readonly FileCompactor _compactor; + + public TexToolsImporter(int count, IEnumerable modPackFiles, Action handler, + Configuration config, ModEditor editor, ModManager modManager, FileCompactor compactor) { _baseDirectory = modManager.BasePath; - _tmpFile = Path.Combine( _baseDirectory.FullName, TempFileName ); + _tmpFile = Path.Combine(_baseDirectory.FullName, TempFileName); _modPackFiles = modPackFiles; _config = config; _editor = editor; - _modManager = modManager; + _modManager = modManager; + _compactor = compactor; _modPackCount = count; - ExtractedMods = new List< (FileInfo, DirectoryInfo?, Exception?) >( count ); + ExtractedMods = new List<(FileInfo, DirectoryInfo?, Exception?)>(count); _token = _cancellation.Token; - Task.Run( ImportFiles, _token ) - .ContinueWith( _ => CloseStreams() ) - .ContinueWith( _ => + Task.Run(ImportFiles, _token) + .ContinueWith(_ => CloseStreams()) + .ContinueWith(_ => { - foreach( var (file, dir, error) in ExtractedMods ) - { - handler( file, dir, error ); - } - } ); + foreach (var (file, dir, error) in ExtractedMods) + handler(file, dir, error); + }); } private void CloseStreams() @@ -71,45 +71,43 @@ public partial class TexToolsImporter : IDisposable public void Dispose() { - _cancellation.Cancel( true ); - if( State != ImporterState.WritingPackToDisk ) + _cancellation.Cancel(true); + if (State != ImporterState.WritingPackToDisk) { _tmpFileStream?.Dispose(); _tmpFileStream = null; } - if( State != ImporterState.ExtractingModFiles ) - { + if (State != ImporterState.ExtractingModFiles) ResetStreamDisposer(); - } } private void ImportFiles() { - State = ImporterState.None; - _currentModPackIdx = 0; - foreach( var file in _modPackFiles ) + State = ImporterState.None; + _currentModPackIdx = 0; + foreach (var file in _modPackFiles) { _currentModDirectory = null; - if( _token.IsCancellationRequested ) + if (_token.IsCancellationRequested) { - ExtractedMods.Add( ( file, null, new TaskCanceledException( "Task canceled by user." ) ) ); + ExtractedMods.Add((file, null, new TaskCanceledException("Task canceled by user."))); continue; } try { - var directory = VerifyVersionAndImport( file ); - ExtractedMods.Add( ( file, directory, null ) ); - if( _config.AutoDeduplicateOnImport ) + var directory = VerifyVersionAndImport(file); + ExtractedMods.Add((file, directory, null)); + if (_config.AutoDeduplicateOnImport) { State = ImporterState.DeduplicatingFiles; - _editor.Duplicates.DeduplicateMod( directory ); + _editor.Duplicates.DeduplicateMod(directory); } } - catch( Exception e ) + catch (Exception e) { - ExtractedMods.Add( ( file, _currentModDirectory, e ) ); + ExtractedMods.Add((file, _currentModDirectory, e)); _currentNumOptions = 0; _currentOptionIdx = 0; _currentFileIdx = 0; @@ -124,87 +122,75 @@ public partial class TexToolsImporter : IDisposable // Rudimentary analysis of a TTMP file by extension and version. // Puts out warnings if extension does not correspond to data. - private DirectoryInfo VerifyVersionAndImport( FileInfo modPackFile ) + private DirectoryInfo VerifyVersionAndImport(FileInfo modPackFile) { - if( modPackFile.Extension.ToLowerInvariant() is ".pmp" or ".zip" or ".7z" or ".rar" ) - { - return HandleRegularArchive( modPackFile ); - } + if (modPackFile.Extension.ToLowerInvariant() is ".pmp" or ".zip" or ".7z" or ".rar") + return HandleRegularArchive(modPackFile); using var zfs = modPackFile.OpenRead(); - using var extractedModPack = ZipArchive.Open( zfs ); + using var extractedModPack = ZipArchive.Open(zfs); - var mpl = FindZipEntry( extractedModPack, "TTMPL.mpl" ); - if( mpl == null ) - { - throw new FileNotFoundException( "ZIP does not contain a TTMPL.mpl file." ); - } + var mpl = FindZipEntry(extractedModPack, "TTMPL.mpl"); + if (mpl == null) + throw new FileNotFoundException("ZIP does not contain a TTMPL.mpl file."); - var modRaw = GetStringFromZipEntry( mpl, Encoding.UTF8 ); + var modRaw = GetStringFromZipEntry(mpl, Encoding.UTF8); // At least a better validation than going by the extension. - if( modRaw.Contains( "\"TTMPVersion\":" ) ) + if (modRaw.Contains("\"TTMPVersion\":")) { - if( modPackFile.Extension != ".ttmp2" ) - { - Penumbra.Log.Warning( $"File {modPackFile.FullName} seems to be a V2 TTMP, but has the wrong extension." ); - } + if (modPackFile.Extension != ".ttmp2") + Penumbra.Log.Warning($"File {modPackFile.FullName} seems to be a V2 TTMP, but has the wrong extension."); - return ImportV2ModPack( modPackFile, extractedModPack, modRaw ); + return ImportV2ModPack(modPackFile, extractedModPack, modRaw); } - if( modPackFile.Extension != ".ttmp" ) - { - Penumbra.Log.Warning( $"File {modPackFile.FullName} seems to be a V1 TTMP, but has the wrong extension." ); - } + if (modPackFile.Extension != ".ttmp") + Penumbra.Log.Warning($"File {modPackFile.FullName} seems to be a V1 TTMP, but has the wrong extension."); - return ImportV1ModPack( modPackFile, extractedModPack, modRaw ); + return ImportV1ModPack(modPackFile, extractedModPack, modRaw); } // You can in no way rely on any file paths in TTMPs so we need to just do this, sorry - private static ZipArchiveEntry? FindZipEntry( ZipArchive file, string fileName ) - => file.Entries.FirstOrDefault( e => !e.IsDirectory && e.Key.Contains( fileName ) ); + private static ZipArchiveEntry? FindZipEntry(ZipArchive file, string fileName) + => file.Entries.FirstOrDefault(e => !e.IsDirectory && e.Key.Contains(fileName)); - private static string GetStringFromZipEntry( ZipArchiveEntry entry, Encoding encoding ) + private static string GetStringFromZipEntry(ZipArchiveEntry entry, Encoding encoding) { using var ms = new MemoryStream(); using var s = entry.OpenEntryStream(); - s.CopyTo( ms ); - return encoding.GetString( ms.ToArray() ); + s.CopyTo(ms); + return encoding.GetString(ms.ToArray()); } - private void WriteZipEntryToTempFile( Stream s ) + private void WriteZipEntryToTempFile(Stream s) { _tmpFileStream?.Dispose(); // should not happen - _tmpFileStream = new FileStream( _tmpFile, FileMode.Create ); - if( _token.IsCancellationRequested ) - { + _tmpFileStream = new FileStream(_tmpFile, FileMode.Create); + if (_token.IsCancellationRequested) return; - } - s.CopyTo( _tmpFileStream ); + s.CopyTo(_tmpFileStream); _tmpFileStream.Dispose(); _tmpFileStream = null; } - private StreamDisposer GetSqPackStreamStream( ZipArchive file, string entryName ) + private StreamDisposer GetSqPackStreamStream(ZipArchive file, string entryName) { State = ImporterState.WritingPackToDisk; // write shitty zip garbage to disk - var entry = FindZipEntry( file, entryName ); - if( entry == null ) - { - throw new FileNotFoundException( $"ZIP does not contain a file named {entryName}." ); - } + var entry = FindZipEntry(file, entryName); + if (entry == null) + throw new FileNotFoundException($"ZIP does not contain a file named {entryName}."); using var s = entry.OpenEntryStream(); - WriteZipEntryToTempFile( s ); + WriteZipEntryToTempFile(s); _streamDisposer?.Dispose(); // Should not happen. - var fs = new FileStream( _tmpFile, FileMode.Open ); - return new StreamDisposer( fs ); + var fs = new FileStream(_tmpFile, FileMode.Open); + return new StreamDisposer(fs); } private void ResetStreamDisposer() @@ -212,4 +198,4 @@ public partial class TexToolsImporter : IDisposable _streamDisposer?.Dispose(); _streamDisposer = null; } -} \ No newline at end of file +} diff --git a/Penumbra/Import/TexToolsImporter.Gui.cs b/Penumbra/Import/TexToolsImporter.Gui.cs index 79cd728b..db818341 100644 --- a/Penumbra/Import/TexToolsImporter.Gui.cs +++ b/Penumbra/Import/TexToolsImporter.Gui.cs @@ -22,7 +22,6 @@ public partial class TexToolsImporter private string _currentOptionName = string.Empty; private string _currentFileName = string.Empty; - public void DrawProgressInfo( Vector2 size ) { if( _modPackCount == 0 ) diff --git a/Penumbra/Import/TexToolsImporter.ModPack.cs b/Penumbra/Import/TexToolsImporter.ModPack.cs index 0776faea..32c9c0e1 100644 --- a/Penumbra/Import/TexToolsImporter.ModPack.cs +++ b/Penumbra/Import/TexToolsImporter.ModPack.cs @@ -17,7 +17,7 @@ public partial class TexToolsImporter private DirectoryInfo? _currentModDirectory; // Version 1 mod packs are a simple collection of files without much information. - private DirectoryInfo ImportV1ModPack( FileInfo modPackFile, ZipArchive extractedModPack, string modRaw ) + private DirectoryInfo ImportV1ModPack(FileInfo modPackFile, ZipArchive extractedModPack, string modRaw) { _currentOptionIdx = 0; _currentNumOptions = 1; @@ -25,174 +25,171 @@ public partial class TexToolsImporter _currentGroupName = string.Empty; _currentOptionName = DefaultTexToolsData.DefaultOption; - Penumbra.Log.Information( " -> Importing V1 ModPack" ); + Penumbra.Log.Information(" -> Importing V1 ModPack"); var modListRaw = modRaw.Split( - new[] { "\r\n", "\r", "\n" }, + new[] + { + "\r\n", + "\r", + "\n", + }, StringSplitOptions.RemoveEmptyEntries ); - var modList = modListRaw.Select( m => JsonConvert.DeserializeObject< SimpleMod >( m, JsonSettings )! ).ToList(); + var modList = modListRaw.Select(m => JsonConvert.DeserializeObject(m, JsonSettings)!).ToList(); - _currentModDirectory = ModCreator.CreateModFolder( _baseDirectory, Path.GetFileNameWithoutExtension( modPackFile.Name ) ); + _currentModDirectory = ModCreator.CreateModFolder(_baseDirectory, Path.GetFileNameWithoutExtension(modPackFile.Name)); // Create a new ModMeta from the TTMP mod list info - _modManager.DataEditor.CreateMeta( _currentModDirectory, _currentModName, DefaultTexToolsData.Author, DefaultTexToolsData.Description, null, null ); + _modManager.DataEditor.CreateMeta(_currentModDirectory, _currentModName, DefaultTexToolsData.Author, DefaultTexToolsData.Description, + null, null); // Open the mod data file from the mod pack as a SqPackStream - _streamDisposer = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" ); - ExtractSimpleModList( _currentModDirectory, modList ); - _modManager.Creator.CreateDefaultFiles( _currentModDirectory ); + _streamDisposer = GetSqPackStreamStream(extractedModPack, "TTMPD.mpd"); + ExtractSimpleModList(_currentModDirectory, modList); + _modManager.Creator.CreateDefaultFiles(_currentModDirectory); ResetStreamDisposer(); return _currentModDirectory; } // Version 2 mod packs can either be simple or extended, import accordingly. - private DirectoryInfo ImportV2ModPack( FileInfo _, ZipArchive extractedModPack, string modRaw ) + private DirectoryInfo ImportV2ModPack(FileInfo _, ZipArchive extractedModPack, string modRaw) { - var modList = JsonConvert.DeserializeObject< SimpleModPack >( modRaw, JsonSettings )!; + var modList = JsonConvert.DeserializeObject(modRaw, JsonSettings)!; - if( modList.TtmpVersion.EndsWith( "s" ) ) - { - return ImportSimpleV2ModPack( extractedModPack, modList ); - } + if (modList.TtmpVersion.EndsWith("s")) + return ImportSimpleV2ModPack(extractedModPack, modList); - if( modList.TtmpVersion.EndsWith( "w" ) ) - { - return ImportExtendedV2ModPack( extractedModPack, modRaw ); - } + if (modList.TtmpVersion.EndsWith("w")) + return ImportExtendedV2ModPack(extractedModPack, modRaw); try { - Penumbra.Log.Warning( $"Unknown TTMPVersion <{modList.TtmpVersion}> given, trying to export as simple mod pack." ); - return ImportSimpleV2ModPack( extractedModPack, modList ); + Penumbra.Log.Warning($"Unknown TTMPVersion <{modList.TtmpVersion}> given, trying to export as simple mod pack."); + return ImportSimpleV2ModPack(extractedModPack, modList); } - catch( Exception e1 ) + catch (Exception e1) { - Penumbra.Log.Warning( $"Exporting as simple mod pack failed with following error, retrying as extended mod pack:\n{e1}" ); + Penumbra.Log.Warning($"Exporting as simple mod pack failed with following error, retrying as extended mod pack:\n{e1}"); try { - return ImportExtendedV2ModPack( extractedModPack, modRaw ); + return ImportExtendedV2ModPack(extractedModPack, modRaw); } - catch( Exception e2 ) + catch (Exception e2) { - throw new IOException( "Exporting as extended mod pack failed, too. Version unsupported or file defect.", e2 ); + throw new IOException("Exporting as extended mod pack failed, too. Version unsupported or file defect.", e2); } } } // Simple V2 mod packs are basically the same as V1 mod packs. - private DirectoryInfo ImportSimpleV2ModPack( ZipArchive extractedModPack, SimpleModPack modList ) + private DirectoryInfo ImportSimpleV2ModPack(ZipArchive extractedModPack, SimpleModPack modList) { _currentOptionIdx = 0; _currentNumOptions = 1; _currentModName = modList.Name; _currentGroupName = string.Empty; _currentOptionName = DefaultTexToolsData.DefaultOption; - Penumbra.Log.Information( " -> Importing Simple V2 ModPack" ); + Penumbra.Log.Information(" -> Importing Simple V2 ModPack"); - _currentModDirectory = ModCreator.CreateModFolder( _baseDirectory, _currentModName ); - _modManager.DataEditor.CreateMeta( _currentModDirectory, _currentModName, modList.Author, string.IsNullOrEmpty( modList.Description ) + _currentModDirectory = ModCreator.CreateModFolder(_baseDirectory, _currentModName); + _modManager.DataEditor.CreateMeta(_currentModDirectory, _currentModName, modList.Author, string.IsNullOrEmpty(modList.Description) ? "Mod imported from TexTools mod pack" - : modList.Description, modList.Version, modList.Url ); + : modList.Description, modList.Version, modList.Url); // Open the mod data file from the mod pack as a SqPackStream - _streamDisposer = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" ); - ExtractSimpleModList( _currentModDirectory, modList.SimpleModsList ); - _modManager.Creator.CreateDefaultFiles( _currentModDirectory ); + _streamDisposer = GetSqPackStreamStream(extractedModPack, "TTMPD.mpd"); + ExtractSimpleModList(_currentModDirectory, modList.SimpleModsList); + _modManager.Creator.CreateDefaultFiles(_currentModDirectory); ResetStreamDisposer(); return _currentModDirectory; } // Obtain the number of relevant options to extract. - private static int GetOptionCount( ExtendedModPack pack ) - => ( pack.SimpleModsList.Length > 0 ? 1 : 0 ) + private static int GetOptionCount(ExtendedModPack pack) + => (pack.SimpleModsList.Length > 0 ? 1 : 0) + pack.ModPackPages - .Sum( page => page.ModGroups - .Where( g => g.GroupName.Length > 0 && g.OptionList.Length > 0 ) - .Sum( group => group.OptionList - .Count( o => o.Name.Length > 0 && o.ModsJsons.Length > 0 ) - + ( group.OptionList.Any( o => o.Name.Length > 0 && o.ModsJsons.Length == 0 ) ? 1 : 0 ) ) ); + .Sum(page => page.ModGroups + .Where(g => g.GroupName.Length > 0 && g.OptionList.Length > 0) + .Sum(group => group.OptionList + .Count(o => o.Name.Length > 0 && o.ModsJsons.Length > 0) + + (group.OptionList.Any(o => o.Name.Length > 0 && o.ModsJsons.Length == 0) ? 1 : 0))); - private static string GetGroupName( string groupName, ISet< string > names ) + private static string GetGroupName(string groupName, ISet names) { var baseName = groupName; var i = 2; - while( !names.Add( groupName ) ) - { + while (!names.Add(groupName)) groupName = $"{baseName} ({i++})"; - } return groupName; } // Extended V2 mod packs contain multiple options that need to be handled separately. - private DirectoryInfo ImportExtendedV2ModPack( ZipArchive extractedModPack, string modRaw ) + private DirectoryInfo ImportExtendedV2ModPack(ZipArchive extractedModPack, string modRaw) { _currentOptionIdx = 0; - Penumbra.Log.Information( " -> Importing Extended V2 ModPack" ); + Penumbra.Log.Information(" -> Importing Extended V2 ModPack"); - var modList = JsonConvert.DeserializeObject< ExtendedModPack >( modRaw, JsonSettings )!; - _currentNumOptions = GetOptionCount( modList ); + var modList = JsonConvert.DeserializeObject(modRaw, JsonSettings)!; + _currentNumOptions = GetOptionCount(modList); _currentModName = modList.Name; - _currentModDirectory = ModCreator.CreateModFolder( _baseDirectory, _currentModName ); - _modManager.DataEditor.CreateMeta( _currentModDirectory, _currentModName, modList.Author, modList.Description, modList.Version, modList.Url ); + _currentModDirectory = ModCreator.CreateModFolder(_baseDirectory, _currentModName); + _modManager.DataEditor.CreateMeta(_currentModDirectory, _currentModName, modList.Author, modList.Description, modList.Version, + modList.Url); - if( _currentNumOptions == 0 ) - { + if (_currentNumOptions == 0) return _currentModDirectory; - } // Open the mod data file from the mod pack as a SqPackStream - _streamDisposer = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" ); + _streamDisposer = GetSqPackStreamStream(extractedModPack, "TTMPD.mpd"); // It can contain a simple list, still. - if( modList.SimpleModsList.Length > 0 ) + if (modList.SimpleModsList.Length > 0) { _currentGroupName = string.Empty; _currentOptionName = "Default"; - ExtractSimpleModList( _currentModDirectory, modList.SimpleModsList ); + ExtractSimpleModList(_currentModDirectory, modList.SimpleModsList); } // Iterate through all pages - var options = new List< ISubMod >(); + var options = new List(); var groupPriority = 0; - var groupNames = new HashSet< string >(); - foreach( var page in modList.ModPackPages ) + var groupNames = new HashSet(); + foreach (var page in modList.ModPackPages) { - foreach( var group in page.ModGroups.Where( group => group.GroupName.Length > 0 && group.OptionList.Length > 0 ) ) + foreach (var group in page.ModGroups.Where(group => group.GroupName.Length > 0 && group.OptionList.Length > 0)) { - var allOptions = group.OptionList.Where( option => option.Name.Length > 0 && option.ModsJsons.Length > 0 ).ToList(); + var allOptions = group.OptionList.Where(option => option.Name.Length > 0 && option.ModsJsons.Length > 0).ToList(); var (numGroups, maxOptions) = group.SelectionType == GroupType.Single - ? ( 1, allOptions.Count ) - : ( 1 + allOptions.Count / IModGroup.MaxMultiOptions, IModGroup.MaxMultiOptions ); - _currentGroupName = GetGroupName( group.GroupName, groupNames ); + ? (1, allOptions.Count) + : (1 + allOptions.Count / IModGroup.MaxMultiOptions, IModGroup.MaxMultiOptions); + _currentGroupName = GetGroupName(group.GroupName, groupNames); - var optionIdx = 0; - for( var groupId = 0; groupId < numGroups; ++groupId ) + var optionIdx = 0; + for (var groupId = 0; groupId < numGroups; ++groupId) { - var name = numGroups == 1 ? _currentGroupName : $"{_currentGroupName}, Part {groupId + 1}"; + var name = numGroups == 1 ? _currentGroupName : $"{_currentGroupName}, Part {groupId + 1}"; options.Clear(); - var groupFolder = ModCreator.NewSubFolderName( _currentModDirectory, name ) - ?? new DirectoryInfo( Path.Combine( _currentModDirectory.FullName, - numGroups == 1 ? $"Group {groupPriority + 1}" : $"Group {groupPriority + 1}, Part {groupId + 1}" ) ); + var groupFolder = ModCreator.NewSubFolderName(_currentModDirectory, name) + ?? new DirectoryInfo(Path.Combine(_currentModDirectory.FullName, + numGroups == 1 ? $"Group {groupPriority + 1}" : $"Group {groupPriority + 1}, Part {groupId + 1}")); uint? defaultSettings = group.SelectionType == GroupType.Multi ? 0u : null; - for( var i = 0; i + optionIdx < allOptions.Count && i < maxOptions; ++i ) + for (var i = 0; i + optionIdx < allOptions.Count && i < maxOptions; ++i) { - var option = allOptions[ i + optionIdx ]; + var option = allOptions[i + optionIdx]; _token.ThrowIfCancellationRequested(); _currentOptionName = option.Name; - var optionFolder = ModCreator.NewSubFolderName( groupFolder, option.Name ) - ?? new DirectoryInfo( Path.Combine( groupFolder.FullName, $"Option {i + optionIdx + 1}" ) ); - ExtractSimpleModList( optionFolder, option.ModsJsons ); - options.Add( _modManager.Creator.CreateSubMod( _currentModDirectory, optionFolder, option ) ); - if( option.IsChecked ) - { + var optionFolder = ModCreator.NewSubFolderName(groupFolder, option.Name) + ?? new DirectoryInfo(Path.Combine(groupFolder.FullName, $"Option {i + optionIdx + 1}")); + ExtractSimpleModList(optionFolder, option.ModsJsons); + options.Add(_modManager.Creator.CreateSubMod(_currentModDirectory, optionFolder, option)); + if (option.IsChecked) defaultSettings = group.SelectionType == GroupType.Multi - ? ( defaultSettings!.Value | ( 1u << i ) ) - : ( uint )i; - } + ? defaultSettings!.Value | (1u << i) + : (uint)i; ++_currentOptionIdx; } @@ -201,30 +198,30 @@ public partial class TexToolsImporter // Handle empty options for single select groups without creating a folder for them. // We only want one of those at most, and it should usually be the first option. - if( group.SelectionType == GroupType.Single ) + if (group.SelectionType == GroupType.Single) { - var empty = group.OptionList.FirstOrDefault( o => o.Name.Length > 0 && o.ModsJsons.Length == 0 ); - if( empty != null ) + var empty = group.OptionList.FirstOrDefault(o => o.Name.Length > 0 && o.ModsJsons.Length == 0); + if (empty != null) { _currentOptionName = empty.Name; - options.Insert( 0, ModCreator.CreateEmptySubMod( empty.Name ) ); + options.Insert(0, ModCreator.CreateEmptySubMod(empty.Name)); defaultSettings = defaultSettings == null ? 0 : defaultSettings.Value + 1; } } - _modManager.Creator.CreateOptionGroup( _currentModDirectory, group.SelectionType, name, groupPriority, groupPriority, - defaultSettings ?? 0, group.Description, options ); + _modManager.Creator.CreateOptionGroup(_currentModDirectory, group.SelectionType, name, groupPriority, groupPriority, + defaultSettings ?? 0, group.Description, options); ++groupPriority; } } } ResetStreamDisposer(); - _modManager.Creator.CreateDefaultFiles( _currentModDirectory ); + _modManager.Creator.CreateDefaultFiles(_currentModDirectory); return _currentModDirectory; } - private void ExtractSimpleModList( DirectoryInfo outDirectory, ICollection< SimpleMod > mods ) + private void ExtractSimpleModList(DirectoryInfo outDirectory, ICollection mods) { State = ImporterState.ExtractingModFiles; @@ -232,51 +229,47 @@ public partial class TexToolsImporter _currentNumFiles = mods.Count(m => m.FullPath.Length > 0); // Extract each SimpleMod into the new mod folder - foreach( var simpleMod in mods.Where(m => m.FullPath.Length > 0 ) ) + foreach (var simpleMod in mods.Where(m => m.FullPath.Length > 0)) { - ExtractMod( outDirectory, simpleMod ); + ExtractMod(outDirectory, simpleMod); ++_currentFileIdx; } } - private void ExtractMod( DirectoryInfo outDirectory, SimpleMod mod ) + private void ExtractMod(DirectoryInfo outDirectory, SimpleMod mod) { - if( _streamDisposer is not PenumbraSqPackStream stream ) - { + if (_streamDisposer is not PenumbraSqPackStream stream) return; - } - Penumbra.Log.Information( $" -> Extracting {mod.FullPath} at {mod.ModOffset:X}" ); + Penumbra.Log.Information($" -> Extracting {mod.FullPath} at {mod.ModOffset:X}"); _token.ThrowIfCancellationRequested(); - var data = stream.ReadFile< PenumbraSqPackStream.PenumbraFileResource >( mod.ModOffset ); + var data = stream.ReadFile(mod.ModOffset); _currentFileName = mod.FullPath; - var extractedFile = new FileInfo( Path.Combine( outDirectory.FullName, mod.FullPath ) ); + var extractedFile = new FileInfo(Path.Combine(outDirectory.FullName, mod.FullPath)); extractedFile.Directory?.Create(); - if( extractedFile.FullName.EndsWith( ".mdl" ) ) - { - ProcessMdl( data.Data ); - } + if (extractedFile.FullName.EndsWith(".mdl")) + ProcessMdl(data.Data); - File.WriteAllBytes( extractedFile.FullName, data.Data ); + _compactor.WriteAllBytesAsync(extractedFile.FullName, data.Data, _token).Wait(_token); } - private static void ProcessMdl( byte[] mdl ) + private static void ProcessMdl(byte[] mdl) { const int modelHeaderLodOffset = 22; // Model file header LOD num - mdl[ 64 ] = 1; + mdl[64] = 1; // Model header LOD num - var stackSize = BitConverter.ToUInt32( mdl, 4 ); - var runtimeBegin = stackSize + 0x44; + var stackSize = BitConverter.ToUInt32(mdl, 4); + var runtimeBegin = stackSize + 0x44; var stringsLengthOffset = runtimeBegin + 4; - var stringsLength = BitConverter.ToUInt32( mdl, ( int )stringsLengthOffset ); + var stringsLength = BitConverter.ToUInt32(mdl, (int)stringsLengthOffset); var modelHeaderStart = stringsLengthOffset + stringsLength + 4; - mdl[ modelHeaderStart + modelHeaderLodOffset ] = 1; + mdl[modelHeaderStart + modelHeaderLodOffset] = 1; } -} \ No newline at end of file +} diff --git a/Penumbra/Import/TexToolsMeta.Export.cs b/Penumbra/Import/TexToolsMeta.Export.cs index 7f455ab0..2eac8f59 100644 --- a/Penumbra/Import/TexToolsMeta.Export.cs +++ b/Penumbra/Import/TexToolsMeta.Export.cs @@ -12,7 +12,7 @@ using Penumbra.Meta.Manipulations; namespace Penumbra.Import; public partial class TexToolsMeta -{ +{ public static void WriteTexToolsMeta(MetaFileManager manager, IEnumerable manipulations, DirectoryInfo basePath) { var files = ConvertToTexTools(manager, manipulations); @@ -23,7 +23,7 @@ public partial class TexToolsMeta try { Directory.CreateDirectory(Path.GetDirectoryName(path)!); - File.WriteAllBytes(path, data); + manager.Compactor.WriteAllBytes(path, data); } catch (Exception e) { @@ -32,191 +32,187 @@ public partial class TexToolsMeta } } - public static Dictionary< string, byte[] > ConvertToTexTools( MetaFileManager manager, IEnumerable< MetaManipulation > manips ) + public static Dictionary ConvertToTexTools(MetaFileManager manager, IEnumerable manips) { - var ret = new Dictionary< string, byte[] >(); - foreach( var group in manips.GroupBy( ManipToPath ) ) + var ret = new Dictionary(); + foreach (var group in manips.GroupBy(ManipToPath)) { - if( group.Key.Length == 0 ) - { + if (group.Key.Length == 0) continue; - } - var bytes = group.Key.EndsWith( ".rgsp" ) - ? WriteRgspFile( manager, group.Key, group ) - : WriteMetaFile( manager, group.Key, group ); - if( bytes.Length == 0 ) - { + var bytes = group.Key.EndsWith(".rgsp") + ? WriteRgspFile(manager, group.Key, group) + : WriteMetaFile(manager, group.Key, group); + if (bytes.Length == 0) continue; - } - ret.Add( group.Key, bytes ); + ret.Add(group.Key, bytes); } return ret; } - private static byte[] WriteRgspFile( MetaFileManager manager, string path, IEnumerable< MetaManipulation > manips ) + private static byte[] WriteRgspFile(MetaFileManager manager, string path, IEnumerable manips) { - var list = manips.GroupBy( m => m.Rsp.Attribute ).ToDictionary( m => m.Key, m => m.Last().Rsp ); - using var m = new MemoryStream( 45 ); - using var b = new BinaryWriter( m ); + var list = manips.GroupBy(m => m.Rsp.Attribute).ToDictionary(m => m.Key, m => m.Last().Rsp); + using var m = new MemoryStream(45); + using var b = new BinaryWriter(m); // Version - b.Write( byte.MaxValue ); - b.Write( ( ushort )2 ); + b.Write(byte.MaxValue); + b.Write((ushort)2); var race = list.First().Value.SubRace; var gender = list.First().Value.Attribute.ToGender(); - b.Write( ( byte )(race - 1) ); // offset by one due to Unknown - b.Write( ( byte )(gender - 1) ); // offset by one due to Unknown + b.Write((byte)(race - 1)); // offset by one due to Unknown + b.Write((byte)(gender - 1)); // offset by one due to Unknown - void Add( params RspAttribute[] attributes ) + void Add(params RspAttribute[] attributes) { - foreach( var attribute in attributes ) + foreach (var attribute in attributes) { - var value = list.TryGetValue( attribute, out var tmp ) ? tmp.Entry : CmpFile.GetDefault( manager, race, attribute ); - b.Write( value ); + var value = list.TryGetValue(attribute, out var tmp) ? tmp.Entry : CmpFile.GetDefault(manager, race, attribute); + b.Write(value); } } - if( gender == Gender.Male ) + if (gender == Gender.Male) { - Add( RspAttribute.MaleMinSize, RspAttribute.MaleMaxSize, RspAttribute.MaleMinTail, RspAttribute.MaleMaxTail ); + Add(RspAttribute.MaleMinSize, RspAttribute.MaleMaxSize, RspAttribute.MaleMinTail, RspAttribute.MaleMaxTail); } else { - Add( RspAttribute.FemaleMinSize, RspAttribute.FemaleMaxSize, RspAttribute.FemaleMinTail, RspAttribute.FemaleMaxTail ); - Add( RspAttribute.BustMinX, RspAttribute.BustMinY, RspAttribute.BustMinZ, RspAttribute.BustMaxX, RspAttribute.BustMaxY, RspAttribute.BustMaxZ ); + Add(RspAttribute.FemaleMinSize, RspAttribute.FemaleMaxSize, RspAttribute.FemaleMinTail, RspAttribute.FemaleMaxTail); + Add(RspAttribute.BustMinX, RspAttribute.BustMinY, RspAttribute.BustMinZ, RspAttribute.BustMaxX, RspAttribute.BustMaxY, + RspAttribute.BustMaxZ); } return m.GetBuffer(); } - private static byte[] WriteMetaFile( MetaFileManager manager, string path, IEnumerable< MetaManipulation > manips ) + private static byte[] WriteMetaFile(MetaFileManager manager, string path, IEnumerable manips) { - var filteredManips = manips.GroupBy( m => m.ManipulationType ).ToDictionary( p => p.Key, p => p.Select( x => x ) ); + var filteredManips = manips.GroupBy(m => m.ManipulationType).ToDictionary(p => p.Key, p => p.Select(x => x)); using var m = new MemoryStream(); - using var b = new BinaryWriter( m ); + using var b = new BinaryWriter(m); // Header // Current TT Metadata version. - b.Write( 2u ); + b.Write(2u); // Null-terminated ASCII path. - var utf8Path = Encoding.ASCII.GetBytes( path ); - b.Write( utf8Path ); - b.Write( ( byte )0 ); + var utf8Path = Encoding.ASCII.GetBytes(path); + b.Write(utf8Path); + b.Write((byte)0); // Number of Headers - b.Write( ( uint )filteredManips.Count ); + b.Write((uint)filteredManips.Count); // Current TT Size of Headers - b.Write( ( uint )12 ); + b.Write((uint)12); // Start of Header Entries for some reason, which is absolutely useless. var headerStart = b.BaseStream.Position + 4; - b.Write( ( uint )headerStart ); + b.Write((uint)headerStart); - var offset = ( uint )( b.BaseStream.Position + 12 * filteredManips.Count ); - foreach( var (header, data) in filteredManips ) + var offset = (uint)(b.BaseStream.Position + 12 * filteredManips.Count); + foreach (var (header, data) in filteredManips) { - b.Write( ( uint )header ); - b.Write( offset ); + b.Write((uint)header); + b.Write(offset); - var size = WriteData( manager, b, offset, header, data ); - b.Write( size ); + var size = WriteData(manager, b, offset, header, data); + b.Write(size); offset += size; } return m.ToArray(); } - private static uint WriteData( MetaFileManager manager, BinaryWriter b, uint offset, MetaManipulation.Type type, IEnumerable< MetaManipulation > manips ) + private static uint WriteData(MetaFileManager manager, BinaryWriter b, uint offset, MetaManipulation.Type type, + IEnumerable manips) { var oldPos = b.BaseStream.Position; - b.Seek( ( int )offset, SeekOrigin.Begin ); + b.Seek((int)offset, SeekOrigin.Begin); - switch( type ) + switch (type) { case MetaManipulation.Type.Imc: var allManips = manips.ToList(); - var baseFile = new ImcFile( manager, allManips[ 0 ].Imc ); - foreach( var manip in allManips ) - { - manip.Imc.Apply( baseFile ); - } + var baseFile = new ImcFile(manager, allManips[0].Imc); + foreach (var manip in allManips) + manip.Imc.Apply(baseFile); - var partIdx = allManips[ 0 ].Imc.ObjectType is ObjectType.Equipment or ObjectType.Accessory - ? ImcFile.PartIndex( allManips[ 0 ].Imc.EquipSlot ) + var partIdx = allManips[0].Imc.ObjectType is ObjectType.Equipment or ObjectType.Accessory + ? ImcFile.PartIndex(allManips[0].Imc.EquipSlot) : 0; - for( var i = 0; i <= baseFile.Count; ++i ) + for (var i = 0; i <= baseFile.Count; ++i) { - var entry = baseFile.GetEntry( partIdx, (Variant)i ); - b.Write( entry.MaterialId ); - b.Write( entry.DecalId ); - b.Write( entry.AttributeAndSound ); - b.Write( entry.VfxId ); - b.Write( entry.MaterialAnimationId ); + var entry = baseFile.GetEntry(partIdx, (Variant)i); + b.Write(entry.MaterialId); + b.Write(entry.DecalId); + b.Write(entry.AttributeAndSound); + b.Write(entry.VfxId); + b.Write(entry.MaterialAnimationId); } break; case MetaManipulation.Type.Eqdp: - foreach( var manip in manips ) + foreach (var manip in manips) { - b.Write( ( uint )Names.CombinedRace( manip.Eqdp.Gender, manip.Eqdp.Race ) ); - var entry = ( byte )(( ( uint )manip.Eqdp.Entry >> Eqdp.Offset( manip.Eqdp.Slot ) ) & 0x03); - b.Write( entry ); + b.Write((uint)Names.CombinedRace(manip.Eqdp.Gender, manip.Eqdp.Race)); + var entry = (byte)(((uint)manip.Eqdp.Entry >> Eqdp.Offset(manip.Eqdp.Slot)) & 0x03); + b.Write(entry); } break; case MetaManipulation.Type.Eqp: - foreach( var manip in manips ) + foreach (var manip in manips) { - var bytes = BitConverter.GetBytes( (ulong) manip.Eqp.Entry ); - var (numBytes, byteOffset) = Eqp.BytesAndOffset( manip.Eqp.Slot ); - for( var i = byteOffset; i < numBytes + byteOffset; ++i ) - b.Write( bytes[ i ] ); + var bytes = BitConverter.GetBytes((ulong)manip.Eqp.Entry); + var (numBytes, byteOffset) = Eqp.BytesAndOffset(manip.Eqp.Slot); + for (var i = byteOffset; i < numBytes + byteOffset; ++i) + b.Write(bytes[i]); } break; case MetaManipulation.Type.Est: - foreach( var manip in manips ) + foreach (var manip in manips) { - b.Write( ( ushort )Names.CombinedRace( manip.Est.Gender, manip.Est.Race ) ); - b.Write( manip.Est.SetId.Id ); - b.Write( manip.Est.Entry ); + b.Write((ushort)Names.CombinedRace(manip.Est.Gender, manip.Est.Race)); + b.Write(manip.Est.SetId.Id); + b.Write(manip.Est.Entry); } break; case MetaManipulation.Type.Gmp: - foreach( var manip in manips ) + foreach (var manip in manips) { - b.Write( ( uint )manip.Gmp.Entry.Value ); - b.Write( manip.Gmp.Entry.UnknownTotal ); + b.Write((uint)manip.Gmp.Entry.Value); + b.Write(manip.Gmp.Entry.UnknownTotal); } break; } var size = b.BaseStream.Position - offset; - b.Seek( ( int )oldPos, SeekOrigin.Begin ); - return ( uint )size; + b.Seek((int)oldPos, SeekOrigin.Begin); + return (uint)size; } - private static string ManipToPath( MetaManipulation manip ) + private static string ManipToPath(MetaManipulation manip) => manip.ManipulationType switch { - MetaManipulation.Type.Imc => ManipToPath( manip.Imc ), - MetaManipulation.Type.Eqdp => ManipToPath( manip.Eqdp ), - MetaManipulation.Type.Eqp => ManipToPath( manip.Eqp ), - MetaManipulation.Type.Est => ManipToPath( manip.Est ), - MetaManipulation.Type.Gmp => ManipToPath( manip.Gmp ), - MetaManipulation.Type.Rsp => ManipToPath( manip.Rsp ), + MetaManipulation.Type.Imc => ManipToPath(manip.Imc), + MetaManipulation.Type.Eqdp => ManipToPath(manip.Eqdp), + MetaManipulation.Type.Eqp => ManipToPath(manip.Eqp), + MetaManipulation.Type.Est => ManipToPath(manip.Est), + MetaManipulation.Type.Gmp => ManipToPath(manip.Gmp), + MetaManipulation.Type.Rsp => ManipToPath(manip.Rsp), _ => string.Empty, }; - private static string ManipToPath( ImcManipulation manip ) + private static string ManipToPath(ImcManipulation manip) { var path = manip.GamePath().ToString(); var replacement = manip.ObjectType switch @@ -227,22 +223,22 @@ public partial class TexToolsMeta _ => ".meta", }; - return path.Replace( ".imc", replacement ); + return path.Replace(".imc", replacement); } - private static string ManipToPath( EqdpManipulation manip ) + private static string ManipToPath(EqdpManipulation manip) => manip.Slot.IsAccessory() ? $"chara/accessory/a{manip.SetId:D4}/a{manip.SetId:D4}_{manip.Slot.ToSuffix()}.meta" : $"chara/equipment/e{manip.SetId:D4}/e{manip.SetId:D4}_{manip.Slot.ToSuffix()}.meta"; - private static string ManipToPath( EqpManipulation manip ) + private static string ManipToPath(EqpManipulation manip) => manip.Slot.IsAccessory() ? $"chara/accessory/a{manip.SetId:D4}/a{manip.SetId:D4}_{manip.Slot.ToSuffix()}.meta" : $"chara/equipment/e{manip.SetId:D4}/e{manip.SetId:D4}_{manip.Slot.ToSuffix()}.meta"; - private static string ManipToPath( EstManipulation manip ) + private static string ManipToPath(EstManipulation manip) { - var raceCode = Names.CombinedRace( manip.Gender, manip.Race ).ToRaceCode(); + var raceCode = Names.CombinedRace(manip.Gender, manip.Race).ToRaceCode(); return manip.Slot switch { EstManipulation.EstType.Hair => $"chara/human/c{raceCode}/obj/hair/h{manip.SetId:D4}/c{raceCode}h{manip.SetId:D4}_hir.meta", @@ -253,10 +249,10 @@ public partial class TexToolsMeta }; } - private static string ManipToPath( GmpManipulation manip ) + private static string ManipToPath(GmpManipulation manip) => $"chara/equipment/e{manip.SetId:D4}/e{manip.SetId:D4}_{EquipSlot.Head.ToSuffix()}.meta"; - private static string ManipToPath( RspManipulation manip ) - => $"chara/xls/charamake/rgsp/{( int )manip.SubRace - 1}-{( int )manip.Attribute.ToGender() - 1}.rgsp"; -} \ No newline at end of file + private static string ManipToPath(RspManipulation manip) + => $"chara/xls/charamake/rgsp/{(int)manip.SubRace - 1}-{(int)manip.Attribute.ToGender() - 1}.rgsp"; +} diff --git a/Penumbra/Import/Textures/TextureDrawer.cs b/Penumbra/Import/Textures/TextureDrawer.cs index fead989e..d1b78268 100644 --- a/Penumbra/Import/Textures/TextureDrawer.cs +++ b/Penumbra/Import/Textures/TextureDrawer.cs @@ -10,7 +10,7 @@ using OtterGui; using OtterGui.Raii; using OtterGui.Widgets; using OtterTex; -using Penumbra.Mods; +using Penumbra.Mods.Editor; using Penumbra.String.Classes; using Penumbra.UI; using Penumbra.UI.Classes; diff --git a/Penumbra/Meta/MetaFileManager.cs b/Penumbra/Meta/MetaFileManager.cs index 7287204c..171ea729 100644 --- a/Penumbra/Meta/MetaFileManager.cs +++ b/Penumbra/Meta/MetaFileManager.cs @@ -4,6 +4,7 @@ using System.Runtime.CompilerServices; using Dalamud.Plugin.Services; using Dalamud.Utility.Signatures; using FFXIVClientStructs.FFXIV.Client.System.Memory; +using OtterGui.Compression; using Penumbra.Collections; using Penumbra.Collections.Manager; using Penumbra.GameData; @@ -26,9 +27,11 @@ public unsafe class MetaFileManager internal readonly ActiveCollectionData ActiveCollections; internal readonly ValidityChecker ValidityChecker; internal readonly IdentifierService Identifier; + internal readonly FileCompactor Compactor; public MetaFileManager(CharacterUtility characterUtility, ResidentResourceManager residentResources, IDataManager gameData, - ActiveCollectionData activeCollections, Configuration config, ValidityChecker validityChecker, IdentifierService identifier) + ActiveCollectionData activeCollections, Configuration config, ValidityChecker validityChecker, IdentifierService identifier, + FileCompactor compactor) { CharacterUtility = characterUtility; ResidentResources = residentResources; @@ -37,6 +40,7 @@ public unsafe class MetaFileManager Config = config; ValidityChecker = validityChecker; Identifier = identifier; + Compactor = compactor; SignatureHelper.Initialise(this); } @@ -91,7 +95,7 @@ public unsafe class MetaFileManager ResidentResources.Reload(); if (collection?._cache == null) - CharacterUtility.ResetAll(); + CharacterUtility.ResetAll(); else collection._cache.Meta.SetFiles(); } diff --git a/Penumbra/Mods/Editor/MdlMaterialEditor.cs b/Penumbra/Mods/Editor/MdlMaterialEditor.cs index f616d128..0fe9ec46 100644 --- a/Penumbra/Mods/Editor/MdlMaterialEditor.cs +++ b/Penumbra/Mods/Editor/MdlMaterialEditor.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; using OtterGui; +using OtterGui.Compression; using Penumbra.GameData.Enums; using Penumbra.GameData.Files; using Penumbra.Mods.Editor; @@ -26,10 +27,10 @@ public partial class MdlMaterialEditor public MdlMaterialEditor(ModFileCollection files) => _files = files; - public void SaveAllModels() + public void SaveAllModels(FileCompactor compactor) { foreach (var info in _modelFiles) - info.Save(); + info.Save(compactor); } public void RestoreAllModels() diff --git a/Penumbra/Mods/Editor/ModEditor.cs b/Penumbra/Mods/Editor/ModEditor.cs index bd774607..f65ce280 100644 --- a/Penumbra/Mods/Editor/ModEditor.cs +++ b/Penumbra/Mods/Editor/ModEditor.cs @@ -1,10 +1,10 @@ using System; using System.IO; using OtterGui; -using Penumbra.Mods.Editor; +using OtterGui.Compression; using Penumbra.Mods.Subclasses; -namespace Penumbra.Mods; +namespace Penumbra.Mods.Editor; public class ModEditor : IDisposable { @@ -15,6 +15,7 @@ public class ModEditor : IDisposable public readonly ModFileCollection Files; public readonly ModSwapEditor SwapEditor; public readonly MdlMaterialEditor MdlMaterialEditor; + public readonly FileCompactor Compactor; public Mod? Mod { get; private set; } public int GroupIdx { get; private set; } @@ -24,7 +25,8 @@ public class ModEditor : IDisposable public ISubMod? Option { get; private set; } public ModEditor(ModNormalizer modNormalizer, ModMetaEditor metaEditor, ModFileCollection files, - ModFileEditor fileEditor, DuplicateManager duplicates, ModSwapEditor swapEditor, MdlMaterialEditor mdlMaterialEditor) + ModFileEditor fileEditor, DuplicateManager duplicates, ModSwapEditor swapEditor, MdlMaterialEditor mdlMaterialEditor, + FileCompactor compactor) { ModNormalizer = modNormalizer; MetaEditor = metaEditor; @@ -33,6 +35,7 @@ public class ModEditor : IDisposable Duplicates = duplicates; SwapEditor = swapEditor; MdlMaterialEditor = mdlMaterialEditor; + Compactor = compactor; } public void LoadMod(Mod mod) diff --git a/Penumbra/Mods/Editor/ModelMaterialInfo.cs b/Penumbra/Mods/Editor/ModelMaterialInfo.cs index dc01ae7d..38e76deb 100644 --- a/Penumbra/Mods/Editor/ModelMaterialInfo.cs +++ b/Penumbra/Mods/Editor/ModelMaterialInfo.cs @@ -2,10 +2,11 @@ using System; using System.Collections.Generic; using System.Linq; using OtterGui; +using OtterGui.Compression; using Penumbra.GameData.Files; using Penumbra.String.Classes; -namespace Penumbra.Mods; +namespace Penumbra.Mods.Editor; /// A class that collects information about skin materials in a model file and handle changes on them. public class ModelMaterialInfo @@ -40,7 +41,7 @@ public class ModelMaterialInfo } // Save a changed .mdl file. - public void Save() + public void Save(FileCompactor compactor) { if (!Changed) return; @@ -50,7 +51,7 @@ public class ModelMaterialInfo try { - System.IO.File.WriteAllBytes(Path.FullName, File.Write()); + compactor.WriteAllBytes(Path.FullName, File.Write()); Changed = false; } catch (Exception e) diff --git a/Penumbra/Mods/ItemSwap/ItemSwapContainer.cs b/Penumbra/Mods/ItemSwap/ItemSwapContainer.cs index 00bbaac8..d1af1d09 100644 --- a/Penumbra/Mods/ItemSwap/ItemSwapContainer.cs +++ b/Penumbra/Mods/ItemSwap/ItemSwapContainer.cs @@ -63,7 +63,6 @@ public class ItemSwapContainer continue; } - if( writeType == WriteType.UseSwaps && file.SwapToModdedExistsInGame && !file.DataWasChanged ) { convertedSwaps.TryAdd( file.SwapFromRequestPath, file.SwapToModded ); @@ -73,7 +72,7 @@ public class ItemSwapContainer var path = file.GetNewPath( directory.FullName ); var bytes = file.FileData.Write(); Directory.CreateDirectory( Path.GetDirectoryName( path )! ); - File.WriteAllBytes( path, bytes ); + _manager.Compactor.WriteAllBytes( path, bytes ); convertedFiles.TryAdd( file.SwapFromRequestPath, new FullPath( path ) ); } diff --git a/Penumbra/Mods/Manager/ModImportManager.cs b/Penumbra/Mods/Manager/ModImportManager.cs index bbe881b0..45b3f2ec 100644 --- a/Penumbra/Mods/Manager/ModImportManager.cs +++ b/Penumbra/Mods/Manager/ModImportManager.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; using Dalamud.Interface.Internal.Notifications; using Penumbra.Import; +using Penumbra.Mods.Editor; namespace Penumbra.Mods.Manager; @@ -57,7 +58,7 @@ public class ModImportManager : IDisposable if (files.Length == 0) return; - _import = new TexToolsImporter(files.Length, files, AddNewMod, _config, _modEditor, _modManager); + _import = new TexToolsImporter(files.Length, files, AddNewMod, _config, _modEditor, _modManager, _modEditor.Compactor); } public bool Importing diff --git a/Penumbra/Services/ServiceManager.cs b/Penumbra/Services/ServiceManager.cs index 8bea52e3..07c7394c 100644 --- a/Penumbra/Services/ServiceManager.cs +++ b/Penumbra/Services/ServiceManager.cs @@ -1,6 +1,7 @@ using Dalamud.Plugin; using Microsoft.Extensions.DependencyInjection; using OtterGui.Classes; +using OtterGui.Compression; using OtterGui.Log; using Penumbra.Api; using Penumbra.Collections.Cache; @@ -62,7 +63,8 @@ public static class ServiceManager .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(); private static IServiceCollection AddGameData(this IServiceCollection services) diff --git a/Penumbra/UI/AdvancedWindow/FileEditor.cs b/Penumbra/UI/AdvancedWindow/FileEditor.cs index d0e9504c..8ed59fed 100644 --- a/Penumbra/UI/AdvancedWindow/FileEditor.cs +++ b/Penumbra/UI/AdvancedWindow/FileEditor.cs @@ -9,6 +9,7 @@ using Dalamud.Plugin.Services; using ImGuiNET; using OtterGui; using OtterGui.Classes; +using OtterGui.Compression; using OtterGui.Raii; using OtterGui.Widgets; using Penumbra.GameData.Files; @@ -18,15 +19,16 @@ using Penumbra.UI.Classes; namespace Penumbra.UI.AdvancedWindow; -public class FileEditor : IDisposable where T : class, IWritable +public class FileEditor : IDisposable where T : class, IWritable { private readonly FileDialogService _fileDialog; - private readonly IDataManager _gameData; + private readonly IDataManager _gameData; private readonly ModEditWindow _owner; + private readonly FileCompactor _compactor; - public FileEditor(ModEditWindow owner, IDataManager gameData, Configuration config, FileDialogService fileDialog, string tabName, - string fileType, Func> getFiles, Func drawEdit, Func getInitialPath, - Func parseFile) + public FileEditor(ModEditWindow owner, IDataManager gameData, Configuration config, FileCompactor compactor, FileDialogService fileDialog, + string tabName, string fileType, Func> getFiles, Func drawEdit, Func getInitialPath, + Func parseFile) { _owner = owner; _gameData = gameData; @@ -36,6 +38,7 @@ public class FileEditor : IDisposable where T : class, IWritable _drawEdit = drawEdit; _getInitialPath = getInitialPath; _parseFile = parseFile; + _compactor = compactor; _combo = new Combo(config, getFiles); } @@ -60,19 +63,19 @@ public class FileEditor : IDisposable where T : class, IWritable DrawFilePanel(); } - public void Dispose() - { - (_currentFile as IDisposable)?.Dispose(); + public void Dispose() + { + (_currentFile as IDisposable)?.Dispose(); _currentFile = null; (_defaultFile as IDisposable)?.Dispose(); _defaultFile = null; - } - - private readonly string _tabName; - private readonly string _fileType; - private readonly Func _drawEdit; - private readonly Func _getInitialPath; - private readonly Func _parseFile; + } + + private readonly string _tabName; + private readonly string _fileType; + private readonly Func _drawEdit; + private readonly Func _getInitialPath; + private readonly Func _parseFile; private FileRegistry? _currentPath; private T? _currentFile; @@ -107,9 +110,9 @@ public class FileEditor : IDisposable where T : class, IWritable if (file != null) { _defaultException = null; - (_defaultFile as IDisposable)?.Dispose(); - _defaultFile = null; // Avoid double disposal if an exception occurs during the parsing of the new file. - _defaultFile = _parseFile(file.Data, _defaultPath, false); + (_defaultFile as IDisposable)?.Dispose(); + _defaultFile = null; // Avoid double disposal if an exception occurs during the parsing of the new file. + _defaultFile = _parseFile(file.Data, _defaultPath, false); } else { @@ -135,7 +138,7 @@ public class FileEditor : IDisposable where T : class, IWritable try { - File.WriteAllBytes(name, _defaultFile?.Write() ?? throw new Exception("File invalid.")); + _compactor.WriteAllBytes(name, _defaultFile?.Write() ?? throw new Exception("File invalid.")); } catch (Exception e) { @@ -168,9 +171,9 @@ public class FileEditor : IDisposable where T : class, IWritable { _currentException = null; _currentPath = null; - (_currentFile as IDisposable)?.Dispose(); - _currentFile = null; - _changed = false; + (_currentFile as IDisposable)?.Dispose(); + _currentFile = null; + _changed = false; } private void DrawFileSelectCombo() @@ -192,13 +195,13 @@ public class FileEditor : IDisposable where T : class, IWritable try { var bytes = File.ReadAllBytes(_currentPath.File.FullName); - (_currentFile as IDisposable)?.Dispose(); - _currentFile = null; // Avoid double disposal if an exception occurs during the parsing of the new file. - _currentFile = _parseFile(bytes, _currentPath.File.FullName, true); + (_currentFile as IDisposable)?.Dispose(); + _currentFile = null; // Avoid double disposal if an exception occurs during the parsing of the new file. + _currentFile = _parseFile(bytes, _currentPath.File.FullName, true); } catch (Exception e) { - (_currentFile as IDisposable)?.Dispose(); + (_currentFile as IDisposable)?.Dispose(); _currentFile = null; _currentException = e; } @@ -209,7 +212,7 @@ public class FileEditor : IDisposable where T : class, IWritable if (ImGuiUtil.DrawDisabledButton("Save to File", Vector2.Zero, $"Save the selected {_fileType} file with all changes applied. This is not revertible.", !_changed)) { - File.WriteAllBytes(_currentPath!.File.FullName, _currentFile!.Write()); + _compactor.WriteAllBytes(_currentPath!.File.FullName, _currentFile!.Write()); _changed = false; } } diff --git a/Penumbra/UI/AdvancedWindow/ModEditWindow.Materials.cs b/Penumbra/UI/AdvancedWindow/ModEditWindow.Materials.cs index 102a6778..84779570 100644 --- a/Penumbra/UI/AdvancedWindow/ModEditWindow.Materials.cs +++ b/Penumbra/UI/AdvancedWindow/ModEditWindow.Materials.cs @@ -195,7 +195,7 @@ public partial class ModEditWindow ImGui.TableNextColumn(); if (ImGuiUtil.DrawDisabledButton(FontAwesomeIcon.Save.ToIconString(), iconSize, "Save the changed mdl file.\nUse at own risk!", !info.Changed, true)) - info.Save(); + info.Save(_editor.Compactor); ImGui.TableNextColumn(); if (ImGuiUtil.DrawDisabledButton(FontAwesomeIcon.Recycle.ToIconString(), iconSize, diff --git a/Penumbra/UI/AdvancedWindow/ModEditWindow.Meta.cs b/Penumbra/UI/AdvancedWindow/ModEditWindow.Meta.cs index 3e81901b..fd4c082e 100644 --- a/Penumbra/UI/AdvancedWindow/ModEditWindow.Meta.cs +++ b/Penumbra/UI/AdvancedWindow/ModEditWindow.Meta.cs @@ -13,6 +13,7 @@ using Penumbra.Meta; using Penumbra.Meta.Files; using Penumbra.Meta.Manipulations; using Penumbra.Mods; +using Penumbra.Mods.Editor; using Penumbra.UI.Classes; namespace Penumbra.UI.AdvancedWindow; diff --git a/Penumbra/UI/AdvancedWindow/ModEditWindow.QuickImport.cs b/Penumbra/UI/AdvancedWindow/ModEditWindow.QuickImport.cs index 29647c53..88ed10df 100644 --- a/Penumbra/UI/AdvancedWindow/ModEditWindow.QuickImport.cs +++ b/Penumbra/UI/AdvancedWindow/ModEditWindow.QuickImport.cs @@ -11,6 +11,7 @@ using OtterGui.Raii; using Penumbra.GameData.Files; using Penumbra.Interop.ResourceTree; using Penumbra.Mods; +using Penumbra.Mods.Editor; using Penumbra.String.Classes; namespace Penumbra.UI.AdvancedWindow; @@ -72,7 +73,7 @@ public partial class ModEditWindow try { - File.WriteAllBytes(name, writable!.Write()); + _editor.Compactor.WriteAllBytes(name, writable!.Write()); } catch (Exception e) { @@ -194,7 +195,7 @@ public partial class ModEditWindow var directory = Path.GetDirectoryName(_targetPath); if (directory != null) Directory.CreateDirectory(directory); - File.WriteAllBytes(_targetPath!, _file!.Write()); + _editor.Compactor.WriteAllBytes(_targetPath!, _file!.Write()); _editor.FileEditor.Revert(_editor.Mod!, _editor.Option!); var fileRegistry = _editor.Files.Available.First(file => file.File.FullName == _targetPath); _editor.FileEditor.AddPathsToSelected(_editor.Option!, new[] diff --git a/Penumbra/UI/AdvancedWindow/ModEditWindow.cs b/Penumbra/UI/AdvancedWindow/ModEditWindow.cs index 890abfed..6e1b1d24 100644 --- a/Penumbra/UI/AdvancedWindow/ModEditWindow.cs +++ b/Penumbra/UI/AdvancedWindow/ModEditWindow.cs @@ -12,6 +12,7 @@ using Dalamud.Interface.Windowing; using Dalamud.Plugin.Services; using ImGuiNET; using OtterGui; +using OtterGui.Compression; using OtterGui.Raii; using Penumbra.Collections.Manager; using Penumbra.Communication; @@ -22,6 +23,7 @@ using Penumbra.Interop.ResourceTree; using Penumbra.Interop.Services; using Penumbra.Meta; using Penumbra.Mods; +using Penumbra.Mods.Editor; using Penumbra.Mods.Manager; using Penumbra.Services; using Penumbra.String; @@ -236,7 +238,7 @@ public partial class ModEditWindow : Window, IDisposable var anyChanges = editor.MdlMaterialEditor.ModelFiles.Any(m => m.Changed); if (ImGuiUtil.DrawDisabledButton("Save All Changes", buttonSize, anyChanges ? "Irreversibly rewrites all currently applied changes to model files." : "No changes made yet.", !anyChanges)) - editor.MdlMaterialEditor.SaveAllModels(); + editor.MdlMaterialEditor.SaveAllModels(editor.Compactor); ImGui.SameLine(); if (ImGuiUtil.DrawDisabledButton("Revert All Changes", buttonSize, @@ -572,17 +574,18 @@ public partial class ModEditWindow : Window, IDisposable _textures = textures; _fileDialog = fileDialog; _gameEvents = gameEvents; - _materialTab = new FileEditor(this, gameData, config, _fileDialog, "Materials", ".mtrl", + _materialTab = new FileEditor(this, gameData, config, _editor.Compactor, _fileDialog, "Materials", ".mtrl", () => _editor.Files.Mtrl, DrawMaterialPanel, () => _mod?.ModPath.FullName ?? string.Empty, (bytes, path, writable) => new MtrlTab(this, new MtrlFile(bytes), path, writable)); - _modelTab = new FileEditor(this, gameData, config, _fileDialog, "Models", ".mdl", + _modelTab = new FileEditor(this, gameData, config, _editor.Compactor, _fileDialog, "Models", ".mdl", () => _editor.Files.Mdl, DrawModelPanel, () => _mod?.ModPath.FullName ?? string.Empty, (bytes, _, _) => new MdlFile(bytes)); - _shaderPackageTab = new FileEditor(this, gameData, config, _fileDialog, "Shaders", ".shpk", + _shaderPackageTab = new FileEditor(this, gameData, config, _editor.Compactor, _fileDialog, "Shaders", ".shpk", () => _editor.Files.Shpk, DrawShaderPackagePanel, () => _mod?.ModPath.FullName ?? string.Empty, (bytes, _, _) => new ShpkTab(_fileDialog, bytes)); _center = new CombinedTexture(_left, _right); _textureSelectCombo = new TextureDrawer.PathSelectCombo(textures, editor); - _quickImportViewer = new ResourceTreeViewer(_config, resourceTreeFactory, changedItemDrawer, 2, OnQuickImportRefresh, DrawQuickImportActions); + _quickImportViewer = + new ResourceTreeViewer(_config, resourceTreeFactory, changedItemDrawer, 2, OnQuickImportRefresh, DrawQuickImportActions); _communicator.ModPathChanged.Subscribe(OnModPathChanged, ModPathChanged.Priority.ModEditWindow); } diff --git a/Penumbra/UI/ModsTab/ModPanelEditTab.cs b/Penumbra/UI/ModsTab/ModPanelEditTab.cs index c629de5b..0e239b7f 100644 --- a/Penumbra/UI/ModsTab/ModPanelEditTab.cs +++ b/Penumbra/UI/ModsTab/ModPanelEditTab.cs @@ -138,7 +138,7 @@ public class ModPanelEditTab : ITab _editor.LoadMod(_mod); _editor.MdlMaterialEditor.ReplaceAllMaterials("bibo", "b"); _editor.MdlMaterialEditor.ReplaceAllMaterials("bibopube", "c"); - _editor.MdlMaterialEditor.SaveAllModels(); + _editor.MdlMaterialEditor.SaveAllModels(_editor.Compactor); _editWindow.UpdateModels(); } diff --git a/Penumbra/UI/Tabs/SettingsTab.cs b/Penumbra/UI/Tabs/SettingsTab.cs index ac834e7c..7f27e6ee 100644 --- a/Penumbra/UI/Tabs/SettingsTab.cs +++ b/Penumbra/UI/Tabs/SettingsTab.cs @@ -7,6 +7,7 @@ using Dalamud.Interface.Components; using Dalamud.Utility; using ImGuiNET; using OtterGui; +using OtterGui.Compression; using OtterGui.Custom; using OtterGui.Raii; using OtterGui.Widgets; @@ -39,6 +40,7 @@ public class SettingsTab : ITab private readonly DalamudServices _dalamud; private readonly HttpApi _httpApi; private readonly DalamudSubstitutionProvider _dalamudSubstitutionProvider; + private readonly FileCompactor _compactor; private int _minimumX = int.MaxValue; private int _minimumY = int.MaxValue; @@ -46,7 +48,7 @@ public class SettingsTab : ITab public SettingsTab(Configuration config, FontReloader fontReloader, TutorialService tutorial, Penumbra penumbra, FileDialogService fileDialog, ModManager modManager, ModFileSystemSelector selector, CharacterUtility characterUtility, ResidentResourceManager residentResources, DalamudServices dalamud, ModExportManager modExportManager, HttpApi httpApi, - DalamudSubstitutionProvider dalamudSubstitutionProvider) + DalamudSubstitutionProvider dalamudSubstitutionProvider, FileCompactor compactor) { _config = config; _fontReloader = fontReloader; @@ -61,6 +63,9 @@ public class SettingsTab : ITab _modExportManager = modExportManager; _httpApi = httpApi; _dalamudSubstitutionProvider = dalamudSubstitutionProvider; + _compactor = compactor; + if (_compactor.CanCompact) + _compactor.Enabled = _config.UseFileSystemCompression; } public void DrawHeader() @@ -661,6 +666,7 @@ public class SettingsTab : ITab Checkbox("Auto Deduplicate on Import", "Automatically deduplicate mod files on import. This will make mod file sizes smaller, but deletes (binary identical) files.", _config.AutoDeduplicateOnImport, v => _config.AutoDeduplicateOnImport = v); + DrawCompressionBox(); Checkbox("Keep Default Metadata Changes on Import", "Normally, metadata changes that equal their default values, which are sometimes exported by TexTools, are discarded. " + "Toggle this to keep them, for example if an option in a mod is supposed to disable a metadata change from a prior option.", @@ -673,6 +679,47 @@ public class SettingsTab : ITab ImGui.NewLine(); } + private void DrawCompressionBox() + { + if (!_compactor.CanCompact) + return; + + Checkbox("Use Filesystem Compression", + "Use Windows functionality to transparently reduce storage size of mod files on your computer. This might cost performance, but seems to generally be beneficial to performance by shifting more responsibility to the underused CPU and away from the overused hard drives.", + _config.UseFileSystemCompression, + v => + { + _config.UseFileSystemCompression = v; + _compactor.Enabled = v; + }); + ImGui.SameLine(); + if (ImGuiUtil.DrawDisabledButton("Compress Existing Files", Vector2.Zero, + "Try to compress all files in your root directory. This will take a while.", + _compactor.MassCompactRunning || !_modManager.Valid)) + _compactor.StartMassCompact(_modManager.BasePath.EnumerateFiles("*.*", SearchOption.AllDirectories), CompressionAlgorithm.Xpress8K); + + ImGui.SameLine(); + if (ImGuiUtil.DrawDisabledButton("Decompress Existing Files", Vector2.Zero, + "Try to decompress all files in your root directory. This will take a while.", + _compactor.MassCompactRunning || !_modManager.Valid)) + _compactor.StartMassCompact(_modManager.BasePath.EnumerateFiles("*.*", SearchOption.AllDirectories), CompressionAlgorithm.None); + + if (_compactor.MassCompactRunning) + { + ImGui.ProgressBar((float)_compactor.CurrentIndex / _compactor.TotalFiles, + new Vector2(ImGui.GetContentRegionAvail().X - ImGui.GetStyle().ItemSpacing.X - UiHelpers.IconButtonSize.X, ImGui.GetFrameHeight()), + _compactor.CurrentFile?.FullName[(_modManager.BasePath.FullName.Length + 1)..] ?? "Gathering Files..."); + ImGui.SameLine(); + if (ImGuiUtil.DrawDisabledButton(FontAwesomeIcon.Ban.ToIconString(), UiHelpers.IconButtonSize, "Cancel the mass action.", + !_compactor.MassCompactRunning, true)) + _compactor.CancelMassCompact(); + } + else + { + ImGui.Dummy(UiHelpers.IconButtonSize); + } + } + /// Draw two integral inputs for minimum dimensions of this window. private void DrawMinimumDimensionConfig() {