From 40be298d67d7a823ea67f62848028843d1b72244 Mon Sep 17 00:00:00 2001 From: Ottermandias Date: Fri, 12 Jul 2024 17:37:19 +0200 Subject: [PATCH] Add automatic reduplication for ui files in pmps, test. --- Penumbra/Configuration.cs | 1 + Penumbra/Import/TexToolsImporter.Archives.cs | 4 +- Penumbra/Import/TexToolsImporter.Gui.cs | 66 +++++------ Penumbra/Mods/Editor/ModNormalizer.cs | 109 ++++++++++++++++++- Penumbra/Penumbra.cs | 1 + Penumbra/UI/Tabs/SettingsTab.cs | 3 + 6 files changed, 145 insertions(+), 39 deletions(-) diff --git a/Penumbra/Configuration.cs b/Penumbra/Configuration.cs index f16569b5..63325433 100644 --- a/Penumbra/Configuration.cs +++ b/Penumbra/Configuration.cs @@ -96,6 +96,7 @@ public class Configuration : IPluginConfiguration, ISavable, IService public DoubleModifier DeleteModModifier { get; set; } = new(ModifierHotkey.Control, ModifierHotkey.Shift); public bool PrintSuccessfulCommandsToChat { get; set; } = true; public bool AutoDeduplicateOnImport { get; set; } = true; + public bool AutoReduplicateUiOnImport { get; set; } = true; public bool UseFileSystemCompression { get; set; } = true; public bool EnableHttpApi { get; set; } = true; diff --git a/Penumbra/Import/TexToolsImporter.Archives.cs b/Penumbra/Import/TexToolsImporter.Archives.cs index 63c170cb..dea343c6 100644 --- a/Penumbra/Import/TexToolsImporter.Archives.cs +++ b/Penumbra/Import/TexToolsImporter.Archives.cs @@ -1,9 +1,7 @@ -using System.IO; using Dalamud.Utility; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using OtterGui.Filesystem; -using Penumbra.GameData.Files; using Penumbra.Import.Structs; using Penumbra.Mods; using SharpCompress.Archives; @@ -11,7 +9,6 @@ using SharpCompress.Archives.Rar; using SharpCompress.Archives.SevenZip; using SharpCompress.Common; using SharpCompress.Readers; -using static FFXIVClientStructs.FFXIV.Client.UI.Misc.ConfigModule; using ZipArchive = SharpCompress.Archives.Zip.ZipArchive; namespace Penumbra.Import; @@ -114,6 +111,7 @@ public partial class TexToolsImporter _currentModDirectory.Refresh(); _modManager.Creator.SplitMultiGroups(_currentModDirectory); + _editor.ModNormalizer.NormalizeUi(_currentModDirectory); return _currentModDirectory; } diff --git a/Penumbra/Import/TexToolsImporter.Gui.cs b/Penumbra/Import/TexToolsImporter.Gui.cs index 78665f30..309f107a 100644 --- a/Penumbra/Import/TexToolsImporter.Gui.cs +++ b/Penumbra/Import/TexToolsImporter.Gui.cs @@ -20,59 +20,61 @@ public partial class TexToolsImporter private string _currentOptionName = string.Empty; private string _currentFileName = string.Empty; - public void DrawProgressInfo(Vector2 size) + public bool DrawProgressInfo(Vector2 size) { if (_modPackCount == 0) { ImGuiUtil.Center("Nothing to extract."); + return false; } - else if (_modPackCount == _currentModPackIdx) - { - DrawEndState(); - } + + if (_modPackCount == _currentModPackIdx) + return DrawEndState(); + + ImGui.NewLine(); + var percentage = (float)_currentModPackIdx / _modPackCount; + ImGui.ProgressBar(percentage, size, $"Mod {_currentModPackIdx + 1} / {_modPackCount}"); + ImGui.NewLine(); + if (State == ImporterState.DeduplicatingFiles) + ImGui.TextUnformatted($"Deduplicating {_currentModName}..."); else + ImGui.TextUnformatted($"Extracting {_currentModName}..."); + + if (_currentNumOptions > 1) { ImGui.NewLine(); - var percentage = (float)_currentModPackIdx / _modPackCount; - ImGui.ProgressBar(percentage, size, $"Mod {_currentModPackIdx + 1} / {_modPackCount}"); ImGui.NewLine(); - if (State == ImporterState.DeduplicatingFiles) - ImGui.TextUnformatted($"Deduplicating {_currentModName}..."); - else - ImGui.TextUnformatted($"Extracting {_currentModName}..."); - - if (_currentNumOptions > 1) - { - ImGui.NewLine(); - ImGui.NewLine(); - percentage = _currentNumOptions == 0 ? 1f : _currentOptionIdx / (float)_currentNumOptions; - ImGui.ProgressBar(percentage, size, $"Option {_currentOptionIdx + 1} / {_currentNumOptions}"); - ImGui.NewLine(); - if (State != ImporterState.DeduplicatingFiles) - ImGui.TextUnformatted( - $"Extracting option {(_currentGroupName.Length == 0 ? string.Empty : $"{_currentGroupName} - ")}{_currentOptionName}..."); - } - - ImGui.NewLine(); - ImGui.NewLine(); - percentage = _currentNumFiles == 0 ? 1f : _currentFileIdx / (float)_currentNumFiles; - ImGui.ProgressBar(percentage, size, $"File {_currentFileIdx + 1} / {_currentNumFiles}"); + percentage = _currentNumOptions == 0 ? 1f : _currentOptionIdx / (float)_currentNumOptions; + ImGui.ProgressBar(percentage, size, $"Option {_currentOptionIdx + 1} / {_currentNumOptions}"); ImGui.NewLine(); if (State != ImporterState.DeduplicatingFiles) - ImGui.TextUnformatted($"Extracting file {_currentFileName}..."); + ImGui.TextUnformatted( + $"Extracting option {(_currentGroupName.Length == 0 ? string.Empty : $"{_currentGroupName} - ")}{_currentOptionName}..."); } + + ImGui.NewLine(); + ImGui.NewLine(); + percentage = _currentNumFiles == 0 ? 1f : _currentFileIdx / (float)_currentNumFiles; + ImGui.ProgressBar(percentage, size, $"File {_currentFileIdx + 1} / {_currentNumFiles}"); + ImGui.NewLine(); + if (State != ImporterState.DeduplicatingFiles) + ImGui.TextUnformatted($"Extracting file {_currentFileName}..."); + return false; } - private void DrawEndState() + private bool DrawEndState() { var success = ExtractedMods.Count(t => t.Error == null); + if (ImGui.IsKeyPressed(ImGuiKey.Escape)) + return true; + ImGui.TextUnformatted($"Successfully extracted {success} / {ExtractedMods.Count} files."); ImGui.NewLine(); using var table = ImRaii.Table("##files", 2); if (!table) - return; + return false; foreach (var (file, dir, ex) in ExtractedMods) { @@ -91,6 +93,8 @@ public partial class TexToolsImporter ImGuiUtil.HoverTooltip(ex.ToString()); } } + + return false; } public bool DrawCancelButton(Vector2 size) diff --git a/Penumbra/Mods/Editor/ModNormalizer.cs b/Penumbra/Mods/Editor/ModNormalizer.cs index 30bf3d3f..43cfc1ee 100644 --- a/Penumbra/Mods/Editor/ModNormalizer.cs +++ b/Penumbra/Mods/Editor/ModNormalizer.cs @@ -3,13 +3,15 @@ using OtterGui; using OtterGui.Classes; using OtterGui.Services; using OtterGui.Tasks; +using Penumbra.Mods.Groups; using Penumbra.Mods.Manager; using Penumbra.Mods.SubMods; +using Penumbra.Services; using Penumbra.String.Classes; namespace Penumbra.Mods.Editor; -public class ModNormalizer(ModManager _modManager, Configuration _config) : IService +public class ModNormalizer(ModManager modManager, Configuration config, SaveService saveService) : IService { private readonly List>> _redirections = []; @@ -39,6 +41,103 @@ public class ModNormalizer(ModManager _modManager, Configuration _config) : ISer Worker = TrackedTask.Run(NormalizeSync); } + public void NormalizeUi(DirectoryInfo modDirectory) + { + if (!config.AutoReduplicateUiOnImport) + return; + + if (modManager.Creator.LoadMod(modDirectory, false) is not { } mod) + return; + + Dictionary> paths = []; + Dictionary containers = []; + foreach (var container in mod.AllDataContainers) + { + foreach (var (gamePath, path) in container.Files) + { + if (!gamePath.Path.StartsWith("ui/"u8)) + continue; + + if (!paths.TryGetValue(path, out var list)) + { + list = []; + paths.Add(path, list); + } + + list.Add((container, gamePath)); + containers.TryAdd(container, string.Empty); + } + } + + foreach (var container in containers.Keys.ToList()) + { + if (container.Group == null) + containers[container] = mod.ModPath.FullName; + else + { + var groupDir = ModCreator.NewOptionDirectory(mod.ModPath, container.Group.Name, config.ReplaceNonAsciiOnImport); + var optionDir = ModCreator.NewOptionDirectory(groupDir, container.GetName(), config.ReplaceNonAsciiOnImport); + containers[container] = optionDir.FullName; + } + } + + var anyChanges = 0; + var modRootLength = mod.ModPath.FullName.Length + 1; + foreach (var (file, gamePaths) in paths) + { + if (gamePaths.Count < 2) + continue; + + var keptPath = false; + foreach (var (container, gamePath) in gamePaths) + { + var directory = containers[container]; + var relPath = new Utf8RelPath(gamePath).ToString(); + var newFilePath = Path.Combine(directory, relPath); + if (newFilePath == file.FullName) + { + Penumbra.Log.Verbose($"[UIReduplication] Kept {file.FullName[modRootLength..]} because new path was identical."); + keptPath = true; + continue; + } + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(newFilePath)!); + File.Copy(file.FullName, newFilePath, false); + Penumbra.Log.Verbose($"[UIReduplication] Copied {file.FullName[modRootLength..]} to {newFilePath[modRootLength..]}."); + container.Files[gamePath] = new FullPath(newFilePath); + ++anyChanges; + } + catch (Exception ex) + { + Penumbra.Log.Error( + $"[UIReduplication] Failed to copy {file.FullName[modRootLength..]} to {newFilePath[modRootLength..]}:\n{ex}"); + } + } + + if (keptPath) + continue; + + try + { + File.Delete(file.FullName); + Penumbra.Log.Verbose($"[UIReduplication] Deleted {file.FullName[modRootLength..]} because no new path matched."); + } + catch (Exception ex) + { + Penumbra.Log.Error($"[UIReduplication] Failed to delete {file.FullName[modRootLength..]}:\n{ex}"); + } + } + + if (anyChanges == 0) + return; + + saveService.Save(SaveType.ImmediateSync, new ModSaveGroup(mod.Default, config.ReplaceNonAsciiOnImport)); + saveService.SaveAllOptionGroups(mod, false, config.ReplaceNonAsciiOnImport); + Penumbra.Log.Information($"[UIReduplication] Saved groups after {anyChanges} changes."); + } + private void NormalizeSync() { try @@ -168,7 +267,7 @@ public class ModNormalizer(ModManager _modManager, Configuration _config) : ISer // Normalize all other options. foreach (var (group, groupIdx) in Mod.Groups.WithIndex()) { - var groupDir = ModCreator.CreateModFolder(directory, group.Name, _config.ReplaceNonAsciiOnImport, true); + var groupDir = ModCreator.CreateModFolder(directory, group.Name, config.ReplaceNonAsciiOnImport, true); _redirections[groupIdx + 1].EnsureCapacity(group.DataContainers.Count); for (var i = _redirections[groupIdx + 1].Count; i < group.DataContainers.Count; ++i) _redirections[groupIdx + 1].Add([]); @@ -188,7 +287,7 @@ public class ModNormalizer(ModManager _modManager, Configuration _config) : ISer void HandleSubMod(DirectoryInfo groupDir, IModDataContainer option, Dictionary newDict) { var name = option.GetName(); - var optionDir = ModCreator.CreateModFolder(groupDir, name, _config.ReplaceNonAsciiOnImport, true); + var optionDir = ModCreator.CreateModFolder(groupDir, name, config.ReplaceNonAsciiOnImport, true); newDict.Clear(); newDict.EnsureCapacity(option.Files.Count); @@ -277,10 +376,10 @@ public class ModNormalizer(ModManager _modManager, Configuration _config) : ISer private void ApplyRedirections() { - _modManager.OptionEditor.SetFiles(Mod.Default, _redirections[0][0]); + modManager.OptionEditor.SetFiles(Mod.Default, _redirections[0][0]); foreach (var (group, groupIdx) in Mod.Groups.WithIndex()) foreach (var (container, containerIdx) in group.DataContainers.WithIndex()) - _modManager.OptionEditor.SetFiles(container, _redirections[groupIdx + 1][containerIdx]); + modManager.OptionEditor.SetFiles(container, _redirections[groupIdx + 1][containerIdx]); ++Step; } diff --git a/Penumbra/Penumbra.cs b/Penumbra/Penumbra.cs index 9f2db2e6..5f8d6805 100644 --- a/Penumbra/Penumbra.cs +++ b/Penumbra/Penumbra.cs @@ -213,6 +213,7 @@ public class Penumbra : IDalamudPlugin sb.Append( $"> **`Free Drive Space: `** {(drive != null ? Functions.HumanReadableSize(drive.AvailableFreeSpace) : "Unknown")}\n"); sb.Append($"> **`Auto-Deduplication: `** {_config.AutoDeduplicateOnImport}\n"); + sb.Append($"> **`Auto-UI-Reduplication: `** {_config.AutoReduplicateUiOnImport}\n"); sb.Append($"> **`Debug Mode: `** {_config.DebugMode}\n"); sb.Append( $"> **`Synchronous Load (Dalamud): `** {(_services.GetService().GetDalamudConfig(DalamudConfigService.WaitingForPluginsOption, out bool v) ? v.ToString() : "Unknown")}\n"); diff --git a/Penumbra/UI/Tabs/SettingsTab.cs b/Penumbra/UI/Tabs/SettingsTab.cs index 8a4d6874..ab47ce7c 100644 --- a/Penumbra/UI/Tabs/SettingsTab.cs +++ b/Penumbra/UI/Tabs/SettingsTab.cs @@ -757,6 +757,9 @@ public class SettingsTab : ITab, IUiService Checkbox("Auto Deduplicate on Import", "Automatically deduplicate mod files on import. This will make mod file sizes smaller, but deletes (binary identical) files.", _config.AutoDeduplicateOnImport, v => _config.AutoDeduplicateOnImport = v); + Checkbox("Auto Reduplicate UI Files on PMP Import", + "Automatically reduplicate and normalize UI-specific files on import from PMP files. This is STRONGLY recommended because deduplicated UI files crash the game.", + _config.AutoReduplicateUiOnImport, v => _config.AutoReduplicateUiOnImport = v); DrawCompressionBox(); Checkbox("Keep Default Metadata Changes on Import", "Normally, metadata changes that equal their default values, which are sometimes exported by TexTools, are discarded. "