diff --git a/Penumbra.Api b/Penumbra.Api index 47bd5424..14652039 160000 --- a/Penumbra.Api +++ b/Penumbra.Api @@ -1 +1 @@ -Subproject commit 47bd5424d04c667d0df1ac1dd1eeb3e50b476c2c +Subproject commit 1465203967d08519c6716292bc5e5094c7fbcacc diff --git a/Penumbra.GameData b/Penumbra.GameData index 4769bbcd..e10d8f33 160000 --- a/Penumbra.GameData +++ b/Penumbra.GameData @@ -1 +1 @@ -Subproject commit 4769bbcdfce9e1d5a461c6b552b5b30ad6bc478e +Subproject commit e10d8f33a676ff4544d7ca05a93d555416f41222 diff --git a/Penumbra/Services/MigrationManager.cs b/Penumbra/Services/MigrationManager.cs index abc059e9..2438c0ad 100644 --- a/Penumbra/Services/MigrationManager.cs +++ b/Penumbra/Services/MigrationManager.cs @@ -1,76 +1,13 @@ using Dalamud.Interface.ImGuiNotification; using OtterGui.Classes; using OtterGui.Services; -using Penumbra.Api.Enums; -using Penumbra.GameData.Files; -using Penumbra.Mods; -using Penumbra.String.Classes; using SharpCompress.Common; using SharpCompress.Readers; +using MdlFile = Penumbra.GameData.Files.MdlFile; +using MtrlFile = Penumbra.GameData.Files.MtrlFile; namespace Penumbra.Services; -public class ModMigrator -{ - private class FileData(string path) - { - public readonly string Path = path; - public readonly List<(string GamePath, int Option)> GamePaths = []; - } - - private sealed class FileDataDict : Dictionary - { - public void Add(string path, string gamePath, int option) - { - if (!TryGetValue(path, out var data)) - { - data = new FileData(path); - data.GamePaths.Add((gamePath, option)); - Add(path, data); - } - else - { - data.GamePaths.Add((gamePath, option)); - } - } - } - - private readonly FileDataDict Textures = []; - private readonly FileDataDict Models = []; - private readonly FileDataDict Materials = []; - - public void Update(IEnumerable mods) - { - CollectFiles(mods); - } - - private void CollectFiles(IEnumerable mods) - { - var option = 0; - foreach (var mod in mods) - { - AddDict(mod.Default.Files, option++); - foreach (var container in mod.Groups.SelectMany(group => group.DataContainers)) - AddDict(container.Files, option++); - } - - return; - - void AddDict(Dictionary dict, int currentOption) - { - foreach (var (gamePath, file) in dict) - { - switch (ResourceTypeExtensions.FromExtension(gamePath.Extension().Span)) - { - case ResourceType.Tex: Textures.Add(file.FullName, gamePath.ToString(), currentOption); break; - case ResourceType.Mdl: Models.Add(file.FullName, gamePath.ToString(), currentOption); break; - case ResourceType.Mtrl: Materials.Add(file.FullName, gamePath.ToString(), currentOption); break; - } - } - } - } -} - public class MigrationManager(Configuration config) : IService { public enum TaskType : byte diff --git a/Penumbra/Services/ModMigrator.cs b/Penumbra/Services/ModMigrator.cs new file mode 100644 index 00000000..20005c8f --- /dev/null +++ b/Penumbra/Services/ModMigrator.cs @@ -0,0 +1,331 @@ +using Dalamud.Plugin.Services; +using OtterGui.Classes; +using OtterGui.Services; +using Penumbra.Api.Enums; +using Penumbra.GameData.Data; +using Penumbra.GameData.Files; +using Penumbra.GameData.Files.MaterialStructs; +using Penumbra.GameData.Structs; +using Penumbra.Import.Textures; +using Penumbra.Mods; +using Penumbra.Mods.SubMods; + +namespace Penumbra.Services; + +public class ModMigrator(IDataManager gameData, TextureManager textures) : IService +{ + private sealed class FileDataDict : MultiDictionary; + + private readonly Lazy _glassReferenceMaterial = new(() => + { + var bytes = gameData.GetFile("chara/equipment/e5001/material/v0001/mt_c0101e5001_met_b.mtrl"); + return new MtrlFile(bytes!.Data); + }); + + private readonly HashSet _changedMods = []; + private readonly HashSet _failedMods = []; + + private readonly FileDataDict Textures = []; + private readonly FileDataDict Models = []; + private readonly FileDataDict Materials = []; + private readonly FileDataDict FileSwaps = []; + + private readonly ConcurrentBag _messages = []; + + public void Update(IEnumerable mods) + { + CollectFiles(mods); + foreach (var (from, (to, container)) in FileSwaps) + MigrateFileSwaps(from, to, container); + foreach (var (model, list) in Models.Grouped) + MigrateModel(model, (Mod)list[0].Container.Mod); + } + + private void CollectFiles(IEnumerable mods) + { + foreach (var mod in mods) + { + foreach (var container in mod.AllDataContainers) + { + foreach (var (gamePath, file) in container.Files) + { + switch (ResourceTypeExtensions.FromExtension(gamePath.Extension().Span)) + { + case ResourceType.Tex: Textures.TryAdd(file.FullName, (gamePath.ToString(), container)); break; + case ResourceType.Mdl: Models.TryAdd(file.FullName, (gamePath.ToString(), container)); break; + case ResourceType.Mtrl: Materials.TryAdd(file.FullName, (gamePath.ToString(), container)); break; + } + } + + foreach (var (swapFrom, swapTo) in container.FileSwaps) + FileSwaps.TryAdd(swapTo.FullName, (swapFrom.ToString(), container)); + } + } + } + + public Task CreateIndexFile(string normalPath, string targetPath) + { + const int rowBlend = 17; + + return Task.Run(async () => + { + var tex = textures.LoadTex(normalPath); + var data = tex.GetPixelData(); + var rgbaData = new RgbaPixelData(data.Width, data.Height, data.Rgba); + if (!BitOperations.IsPow2(rgbaData.Height) || !BitOperations.IsPow2(rgbaData.Width)) + { + var requiredHeight = (int)BitOperations.RoundUpToPowerOf2((uint)rgbaData.Height); + var requiredWidth = (int)BitOperations.RoundUpToPowerOf2((uint)rgbaData.Width); + rgbaData = rgbaData.Resize((requiredWidth, requiredHeight)); + } + + Parallel.ForEach(Enumerable.Range(0, rgbaData.PixelData.Length / 4), idx => + { + var pixelIdx = 4 * idx; + var normal = rgbaData.PixelData[pixelIdx + 3]; + + // Copied from TT + var blendRem = normal % (2 * rowBlend); + var originalRow = normal / rowBlend; + switch (blendRem) + { + // Goes to next row, clamped to the closer row. + case > 25: + blendRem = 0; + ++originalRow; + break; + // Stays in this row, clamped to the closer row. + case > 17: blendRem = 17; break; + } + + var newBlend = (byte)(255 - MathF.Round(blendRem / 17f * 255f)); + + // Slight add here to push the color deeper into the row to ensure BC5 compression doesn't + // cause any artifacting. + var newRow = (byte)(originalRow / 2 * 17 + 4); + + rgbaData.PixelData[pixelIdx] = newRow; + rgbaData.PixelData[pixelIdx] = newBlend; + rgbaData.PixelData[pixelIdx] = 0; + rgbaData.PixelData[pixelIdx] = 255; + }); + await textures.SaveAs(CombinedTexture.TextureSaveType.BC5, true, true, new BaseImage(), targetPath, rgbaData.PixelData, + rgbaData.Width, rgbaData.Height); + }); + } + + private void MigrateModel(string filePath, Mod mod) + { + if (MigrationManager.TryMigrateSingleModel(filePath, true)) + { + _messages.Add($"Migrated model {filePath} in {mod.Name}."); + } + else + { + _messages.Add($"Failed to migrate model {filePath} in {mod.Name}"); + _failedMods.Add(mod); + } + } + + private void SetGlassReferenceValues(MtrlFile mtrl) + { + var reference = _glassReferenceMaterial.Value; + mtrl.ShaderPackage.ShaderKeys = reference.ShaderPackage.ShaderKeys.ToArray(); + mtrl.ShaderPackage.Constants = reference.ShaderPackage.Constants.ToArray(); + mtrl.AdditionalData = reference.AdditionalData.ToArray(); + mtrl.ShaderPackage.Flags &= ~(0x04u | 0x08u); + // From TT. + if (mtrl.Table is ColorTable t) + foreach (ref var row in t.AsRows()) + row.SpecularColor = new HalfColor((Half)0.8100586, (Half)0.8100586, (Half)0.8100586); + } + + private ref struct MaterialPack + { + public readonly MtrlFile File; + public readonly bool UsesMaskAsSpecular; + + private readonly Dictionary Samplers = []; + + public MaterialPack(MtrlFile file) + { + File = file; + UsesMaskAsSpecular = File.ShaderPackage.ShaderKeys.Any(x => x.Key is 0xC8BD1DEF && x.Value is 0xA02F4828 or 0x198D11CD); + Add(Samplers, TextureUsage.Normal, ShpkFile.NormalSamplerId); + Add(Samplers, TextureUsage.Index, ShpkFile.IndexSamplerId); + Add(Samplers, TextureUsage.Mask, ShpkFile.MaskSamplerId); + Add(Samplers, TextureUsage.Diffuse, ShpkFile.DiffuseSamplerId); + Add(Samplers, TextureUsage.Specular, ShpkFile.SpecularSamplerId); + return; + + void Add(Dictionary dict, TextureUsage usage, uint samplerId) + { + var idx = new SamplerIndex(file, samplerId); + if (idx.Texture >= 0) + dict.Add(usage, idx); + } + } + + public readonly record struct SamplerIndex(int Sampler, int Texture) + { + public SamplerIndex(MtrlFile file, uint samplerId) + : this(file.FindSampler(samplerId), -1) + => Texture = Sampler < 0 ? -1 : file.ShaderPackage.Samplers[Sampler].TextureIndex; + } + + public enum TextureUsage + { + Normal, + Index, + Mask, + Diffuse, + Specular, + } + + public static bool AdaptPath(IDataManager data, string path, TextureUsage usage, out string newPath) + { + newPath = path; + if (Path.GetExtension(newPath) is not ".tex") + return false; + + if (data.FileExists(newPath)) + return true; + + switch (usage) + { + case TextureUsage.Normal: + newPath = path.Replace("_n.tex", "_norm.tex"); + if (data.FileExists(newPath)) + return true; + + newPath = path.Replace("_n_", "_norm_"); + if (data.FileExists(newPath)) + return true; + + return false; + case TextureUsage.Index: return false; + case TextureUsage.Mask: + newPath = path.Replace("_m.tex", "_mult.tex"); + if (data.FileExists(newPath)) + return true; + + newPath = path.Replace("_m.tex", "_mask.tex"); + if (data.FileExists(newPath)) + return true; + + newPath = path.Replace("_m_", "_mult_"); + if (data.FileExists(newPath)) + return true; + + newPath = path.Replace("_m_", "_mask_"); + if (data.FileExists(newPath)) + return true; + + return false; + case TextureUsage.Diffuse: + newPath = path.Replace("_d.tex", "_base.tex"); + if (data.FileExists(newPath)) + return true; + + newPath = path.Replace("_d_", "_base_"); + if (data.FileExists(newPath)) + return true; + + return false; + case TextureUsage.Specular: + return false; + default: throw new ArgumentOutOfRangeException(nameof(usage), usage, null); + } + } + } + + private void MigrateMaterial(string filePath, IReadOnlyList<(string GamePath, IModDataContainer Container)> redirections) + { + try + { + var bytes = File.ReadAllBytes(filePath); + var mtrl = new MtrlFile(bytes); + if (!CheckUpdateNeeded(mtrl)) + return; + + // Update colorsets, flags and character shader package. + var changes = mtrl.MigrateToDawntrail(); + + if (!changes) + switch (mtrl.ShaderPackage.Name) + { + case "hair.shpk": break; + case "characterglass.shpk": + SetGlassReferenceValues(mtrl); + changes = true; + break; + } + + // Remove DX11 flags and update paths if necessary. + foreach (ref var tex in mtrl.Textures.AsSpan()) + { + if (tex.DX11) + { + changes = true; + if (GamePaths.Tex.HandleDx11Path(tex, out var newPath)) + tex.Path = newPath; + tex.DX11 = false; + } + + if (gameData.FileExists(tex.Path)) + continue; + } + + // Dyeing, from TT. + if (mtrl.DyeTable is ColorDyeTable dye) + foreach (ref var row in dye.AsRows()) + row.Template += 1000; + } + catch + { + // ignored + } + + static bool CheckUpdateNeeded(MtrlFile mtrl) + { + if (!mtrl.IsDawntrail) + return true; + + if (mtrl.ShaderPackage.Name is not "hair.shpk") + return false; + + var foundOld = 0; + foreach (var c in mtrl.ShaderPackage.Constants) + { + switch (c.Id) + { + case 0x36080AD0: foundOld |= 1; break; // == 1, from TT + case 0x992869AB: foundOld |= 2; break; // == 3 (skin) or 4 (hair) from TT + } + + if (foundOld is 3) + return true; + } + + return false; + } + } + + private void MigrateFileSwaps(string swapFrom, string swapTo, IModDataContainer container) + { + var fromExists = gameData.FileExists(swapFrom); + var toExists = gameData.FileExists(swapTo); + if (fromExists && toExists) + return; + + if (ResourceTypeExtensions.FromExtension(Path.GetExtension(swapFrom.AsSpan())) is not ResourceType.Tex + || ResourceTypeExtensions.FromExtension(Path.GetExtension(swapTo.AsSpan())) is not ResourceType.Tex) + { + _messages.Add( + $"Could not migrate file swap {swapFrom} -> {swapTo} in {container.Mod.Name}: {container.GetFullName()}. Only textures may be migrated.{(fromExists ? "\n\tSource File does not exist." : "")}{(toExists ? "\n\tTarget File does not exist." : "")}"); + return; + } + + // try to migrate texture swaps + } +} diff --git a/Penumbra/UI/Tabs/Debug/DebugTab.cs b/Penumbra/UI/Tabs/Debug/DebugTab.cs index 9dd18ddd..6d6222ec 100644 --- a/Penumbra/UI/Tabs/Debug/DebugTab.cs +++ b/Penumbra/UI/Tabs/Debug/DebugTab.cs @@ -106,6 +106,7 @@ public class DebugTab : Window, ITab, IUiService private readonly SchedulerResourceManagementService _schedulerService; private readonly ObjectIdentification _objectIdentification; private readonly RenderTargetDrawer _renderTargetDrawer; + private readonly ModMigratorDebug _modMigratorDebug; public DebugTab(PerformanceTracker performance, Configuration config, CollectionManager collectionManager, ObjectManager objects, IClientState clientState, IDataManager dataManager, @@ -116,7 +117,8 @@ public class DebugTab : Window, ITab, IUiService TextureManager textureManager, ShaderReplacementFixer shaderReplacementFixer, RedrawService redraws, DictEmote emotes, Diagnostics diagnostics, IpcTester ipcTester, CrashHandlerPanel crashHandlerPanel, TexHeaderDrawer texHeaderDrawer, HookOverrideDrawer hookOverrides, RsfService rsfService, GlobalVariablesDrawer globalVariablesDrawer, - SchedulerResourceManagementService schedulerService, ObjectIdentification objectIdentification, RenderTargetDrawer renderTargetDrawer) + SchedulerResourceManagementService schedulerService, ObjectIdentification objectIdentification, RenderTargetDrawer renderTargetDrawer, + ModMigratorDebug modMigratorDebug) : base("Penumbra Debug Window", ImGuiWindowFlags.NoCollapse) { IsOpen = true; @@ -158,6 +160,7 @@ public class DebugTab : Window, ITab, IUiService _schedulerService = schedulerService; _objectIdentification = objectIdentification; _renderTargetDrawer = renderTargetDrawer; + _modMigratorDebug = modMigratorDebug; _objects = objects; _clientState = clientState; _dataManager = dataManager; @@ -190,6 +193,7 @@ public class DebugTab : Window, ITab, IUiService DrawActorsDebug(); DrawCollectionCaches(); _texHeaderDrawer.Draw(); + _modMigratorDebug.Draw(); DrawShaderReplacementFixer(); DrawData(); DrawCrcCache(); @@ -591,6 +595,7 @@ public class DebugTab : Window, ITab, IUiService { ImUtf8.DrawTableColumn($"{_objects[idx]}"); } + ImUtf8.DrawTableColumn(gameObjectPtr.Utf8Name.Span); var collection = _collectionResolver.IdentifyCollection(gameObjectPtr.AsObject, true); ImUtf8.DrawTableColumn(collection.ModCollection.Identity.Name); @@ -751,7 +756,7 @@ public class DebugTab : Window, ITab, IUiService DrawChangedItemTest(); } - private string _changedItemPath = string.Empty; + private string _changedItemPath = string.Empty; private readonly Dictionary _changedItems = []; private void DrawChangedItemTest() diff --git a/Penumbra/UI/Tabs/Debug/ModMigratorDebug.cs b/Penumbra/UI/Tabs/Debug/ModMigratorDebug.cs new file mode 100644 index 00000000..c8518315 --- /dev/null +++ b/Penumbra/UI/Tabs/Debug/ModMigratorDebug.cs @@ -0,0 +1,55 @@ +using ImGuiNET; +using OtterGui.Services; +using OtterGui.Text; +using Penumbra.Services; + +namespace Penumbra.UI.Tabs.Debug; + +public class ModMigratorDebug(ModMigrator migrator) : IUiService +{ + private string _inputPath = string.Empty; + private string _outputPath = string.Empty; + private Task? _indexTask; + private Task? _mdlTask; + + public void Draw() + { + if (!ImUtf8.CollapsingHeaderId("Mod Migrator"u8)) + return; + + ImUtf8.InputText("##input"u8, ref _inputPath, "Input Path..."u8); + ImUtf8.InputText("##output"u8, ref _outputPath, "Output Path..."u8); + + if (ImUtf8.ButtonEx("Create Index Texture"u8, "Requires input to be a path to a normal texture."u8, default, _inputPath.Length == 0 + || _outputPath.Length == 0 + || _indexTask is + { + IsCompleted: false, + })) + _indexTask = migrator.CreateIndexFile(_inputPath, _outputPath); + + if (_indexTask is not null) + { + ImGui.SameLine(); + ImUtf8.TextFrameAligned($"{_indexTask.Status}"); + } + + if (ImUtf8.ButtonEx("Update Model File"u8, "Requires input to be a path to a mdl."u8, default, _inputPath.Length == 0 + || _outputPath.Length == 0 + || _mdlTask is + { + IsCompleted: false, + })) + _mdlTask = Task.Run(() => + { + File.Copy(_inputPath, _outputPath, true); + MigrationManager.TryMigrateSingleModel(_outputPath, false); + }); + + if (_mdlTask is not null) + { + ImGui.SameLine(); + ImUtf8.TextFrameAligned($"{_mdlTask.Status}"); + } + } +}