Add some migration stuff.

Penumbra.Api

@@ -1 +1 @@
-Subproject commit 47bd5424d04c667d0df1ac1dd1eeb3e50b476c2c
+Subproject commit 1465203967d08519c6716292bc5e5094c7fbcacc

Penumbra.GameData

@@ -1 +1 @@
-Subproject commit 4769bbcdfce9e1d5a461c6b552b5b30ad6bc478e
+Subproject commit e10d8f33a676ff4544d7ca05a93d555416f41222

Penumbra/Services/MigrationManager.cs

@@ -1,76 +1,13 @@
 using Dalamud.Interface.ImGuiNotification;
 using OtterGui.Classes;
 using OtterGui.Services;
-using Penumbra.Api.Enums;
-using Penumbra.GameData.Files;
-using Penumbra.Mods;
-using Penumbra.String.Classes;
 using SharpCompress.Common;
 using SharpCompress.Readers;
+using MdlFile = Penumbra.GameData.Files.MdlFile;
+using MtrlFile = Penumbra.GameData.Files.MtrlFile;
 
 namespace Penumbra.Services;
 
-public class ModMigrator
-{
-    private class FileData(string path)
-    {
-        public readonly string                              Path      = path;
-        public readonly List<(string GamePath, int Option)> GamePaths = [];
-    }
-
-    private sealed class FileDataDict : Dictionary<string, FileData>
-    {
-        public void Add(string path, string gamePath, int option)
-        {
-            if (!TryGetValue(path, out var data))
-            {
-                data = new FileData(path);
-                data.GamePaths.Add((gamePath, option));
-                Add(path, data);
-            }
-            else
-            {
-                data.GamePaths.Add((gamePath, option));
-            }
-        }
-    }
-
-    private readonly FileDataDict Textures  = [];
-    private readonly FileDataDict Models    = [];
-    private readonly FileDataDict Materials = [];
-
-    public void Update(IEnumerable<Mod> mods)
-    {
-        CollectFiles(mods);
-    }
-
-    private void CollectFiles(IEnumerable<Mod> mods)
-    {
-        var option = 0;
-        foreach (var mod in mods)
-        {
-            AddDict(mod.Default.Files, option++);
-            foreach (var container in mod.Groups.SelectMany(group => group.DataContainers))
-                AddDict(container.Files, option++);
-        }
-
-        return;
-
-        void AddDict(Dictionary<Utf8GamePath, FullPath> dict, int currentOption)
-        {
-            foreach (var (gamePath, file) in dict)
-            {
-                switch (ResourceTypeExtensions.FromExtension(gamePath.Extension().Span))
-                {
-                    case ResourceType.Tex:  Textures.Add(file.FullName, gamePath.ToString(), currentOption); break;
-                    case ResourceType.Mdl:  Models.Add(file.FullName, gamePath.ToString(), currentOption); break;
-                    case ResourceType.Mtrl: Materials.Add(file.FullName, gamePath.ToString(), currentOption); break;
-                }
-            }
-        }
-    }
-}
-
 public class MigrationManager(Configuration config) : IService
 {
     public enum TaskType : byte

Penumbra/Services/ModMigrator.cs

@@ -0,0 +1,331 @@
+using Dalamud.Plugin.Services;
+using OtterGui.Classes;
+using OtterGui.Services;
+using Penumbra.Api.Enums;
+using Penumbra.GameData.Data;
+using Penumbra.GameData.Files;
+using Penumbra.GameData.Files.MaterialStructs;
+using Penumbra.GameData.Structs;
+using Penumbra.Import.Textures;
+using Penumbra.Mods;
+using Penumbra.Mods.SubMods;
+
+namespace Penumbra.Services;
+
+public class ModMigrator(IDataManager gameData, TextureManager textures) : IService
+{
+    private sealed class FileDataDict : MultiDictionary<string, (string GamePath, IModDataContainer Container)>;
+
+    private readonly Lazy<MtrlFile> _glassReferenceMaterial = new(() =>
+    {
+        var bytes = gameData.GetFile("chara/equipment/e5001/material/v0001/mt_c0101e5001_met_b.mtrl");
+        return new MtrlFile(bytes!.Data);
+    });
+
+    private readonly HashSet<Mod> _changedMods = [];
+    private readonly HashSet<Mod> _failedMods  = [];
+
+    private readonly FileDataDict Textures  = [];
+    private readonly FileDataDict Models    = [];
+    private readonly FileDataDict Materials = [];
+    private readonly FileDataDict FileSwaps = [];
+
+    private readonly ConcurrentBag<string> _messages = [];
+
+    public void Update(IEnumerable<Mod> mods)
+    {
+        CollectFiles(mods);
+        foreach (var (from, (to, container)) in FileSwaps)
+            MigrateFileSwaps(from, to, container);
+        foreach (var (model, list) in Models.Grouped)
+            MigrateModel(model, (Mod)list[0].Container.Mod);
+    }
+
+    private void CollectFiles(IEnumerable<Mod> mods)
+    {
+        foreach (var mod in mods)
+        {
+            foreach (var container in mod.AllDataContainers)
+            {
+                foreach (var (gamePath, file) in container.Files)
+                {
+                    switch (ResourceTypeExtensions.FromExtension(gamePath.Extension().Span))
+                    {
+                        case ResourceType.Tex:  Textures.TryAdd(file.FullName, (gamePath.ToString(), container)); break;
+                        case ResourceType.Mdl:  Models.TryAdd(file.FullName, (gamePath.ToString(), container)); break;
+                        case ResourceType.Mtrl: Materials.TryAdd(file.FullName, (gamePath.ToString(), container)); break;
+                    }
+                }
+
+                foreach (var (swapFrom, swapTo) in container.FileSwaps)
+                    FileSwaps.TryAdd(swapTo.FullName, (swapFrom.ToString(), container));
+            }
+        }
+    }
+
+    public Task CreateIndexFile(string normalPath, string targetPath)
+    {
+        const int rowBlend = 17;
+
+        return Task.Run(async () =>
+        {
+            var tex      = textures.LoadTex(normalPath);
+            var data     = tex.GetPixelData();
+            var rgbaData = new RgbaPixelData(data.Width, data.Height, data.Rgba);
+            if (!BitOperations.IsPow2(rgbaData.Height) || !BitOperations.IsPow2(rgbaData.Width))
+            {
+                var requiredHeight = (int)BitOperations.RoundUpToPowerOf2((uint)rgbaData.Height);
+                var requiredWidth  = (int)BitOperations.RoundUpToPowerOf2((uint)rgbaData.Width);
+                rgbaData = rgbaData.Resize((requiredWidth, requiredHeight));
+            }
+
+            Parallel.ForEach(Enumerable.Range(0, rgbaData.PixelData.Length / 4), idx =>
+            {
+                var pixelIdx = 4 * idx;
+                var normal   = rgbaData.PixelData[pixelIdx + 3];
+
+                // Copied from TT
+                var blendRem    = normal % (2 * rowBlend);
+                var originalRow = normal / rowBlend;
+                switch (blendRem)
+                {
+                    // Goes to next row, clamped to the closer row.
+                    case > 25:
+                        blendRem = 0;
+                        ++originalRow;
+                        break;
+                    // Stays in this row, clamped to the closer row.
+                    case > 17: blendRem = 17; break;
+                }
+
+                var newBlend = (byte)(255 - MathF.Round(blendRem / 17f * 255f));
+
+                // Slight add here to push the color deeper into the row to ensure BC5 compression doesn't
+                // cause any artifacting.
+                var newRow = (byte)(originalRow / 2 * 17 + 4);
+
+                rgbaData.PixelData[pixelIdx] = newRow;
+                rgbaData.PixelData[pixelIdx] = newBlend;
+                rgbaData.PixelData[pixelIdx] = 0;
+                rgbaData.PixelData[pixelIdx] = 255;
+            });
+            await textures.SaveAs(CombinedTexture.TextureSaveType.BC5, true, true, new BaseImage(), targetPath, rgbaData.PixelData,
+                rgbaData.Width, rgbaData.Height);
+        });
+    }
+
+    private void MigrateModel(string filePath, Mod mod)
+    {
+        if (MigrationManager.TryMigrateSingleModel(filePath, true))
+        {
+            _messages.Add($"Migrated model {filePath} in {mod.Name}.");
+        }
+        else
+        {
+            _messages.Add($"Failed to migrate model {filePath} in {mod.Name}");
+            _failedMods.Add(mod);
+        }
+    }
+
+    private void SetGlassReferenceValues(MtrlFile mtrl)
+    {
+        var reference = _glassReferenceMaterial.Value;
+        mtrl.ShaderPackage.ShaderKeys =  reference.ShaderPackage.ShaderKeys.ToArray();
+        mtrl.ShaderPackage.Constants  =  reference.ShaderPackage.Constants.ToArray();
+        mtrl.AdditionalData           =  reference.AdditionalData.ToArray();
+        mtrl.ShaderPackage.Flags      &= ~(0x04u | 0x08u);
+        // From TT.
+        if (mtrl.Table is ColorTable t)
+            foreach (ref var row in t.AsRows())
+                row.SpecularColor = new HalfColor((Half)0.8100586, (Half)0.8100586, (Half)0.8100586);
+    }
+
+    private ref struct MaterialPack
+    {
+        public readonly MtrlFile File;
+        public readonly bool     UsesMaskAsSpecular;
+
+        private readonly Dictionary<TextureUsage, SamplerIndex> Samplers = [];
+
+        public MaterialPack(MtrlFile file)
+        {
+            File               = file;
+            UsesMaskAsSpecular = File.ShaderPackage.ShaderKeys.Any(x => x.Key is 0xC8BD1DEF && x.Value is 0xA02F4828 or 0x198D11CD);
+            Add(Samplers, TextureUsage.Normal,   ShpkFile.NormalSamplerId);
+            Add(Samplers, TextureUsage.Index,    ShpkFile.IndexSamplerId);
+            Add(Samplers, TextureUsage.Mask,     ShpkFile.MaskSamplerId);
+            Add(Samplers, TextureUsage.Diffuse,  ShpkFile.DiffuseSamplerId);
+            Add(Samplers, TextureUsage.Specular, ShpkFile.SpecularSamplerId);
+            return;
+
+            void Add(Dictionary<TextureUsage, SamplerIndex> dict, TextureUsage usage, uint samplerId)
+            {
+                var idx = new SamplerIndex(file, samplerId);
+                if (idx.Texture >= 0)
+                    dict.Add(usage, idx);
+            }
+        }
+
+        public readonly record struct SamplerIndex(int Sampler, int Texture)
+        {
+            public SamplerIndex(MtrlFile file, uint samplerId)
+                : this(file.FindSampler(samplerId), -1)
+                => Texture = Sampler < 0 ? -1 : file.ShaderPackage.Samplers[Sampler].TextureIndex;
+        }
+
+        public enum TextureUsage
+        {
+            Normal,
+            Index,
+            Mask,
+            Diffuse,
+            Specular,
+        }
+
+        public static bool AdaptPath(IDataManager data, string path, TextureUsage usage, out string newPath)
+        {
+            newPath = path;
+            if (Path.GetExtension(newPath) is not ".tex")
+                return false;
+
+            if (data.FileExists(newPath))
+                return true;
+
+            switch (usage)
+            {
+                case TextureUsage.Normal:
+                    newPath = path.Replace("_n.tex", "_norm.tex");
+                    if (data.FileExists(newPath))
+                        return true;
+
+                    newPath = path.Replace("_n_", "_norm_");
+                    if (data.FileExists(newPath))
+                        return true;
+
+                    return false;
+                case TextureUsage.Index: return false;
+                case TextureUsage.Mask:
+                    newPath = path.Replace("_m.tex", "_mult.tex");
+                    if (data.FileExists(newPath))
+                        return true;
+
+                    newPath = path.Replace("_m.tex", "_mask.tex");
+                    if (data.FileExists(newPath))
+                        return true;
+
+                    newPath = path.Replace("_m_", "_mult_");
+                    if (data.FileExists(newPath))
+                        return true;
+
+                    newPath = path.Replace("_m_", "_mask_");
+                    if (data.FileExists(newPath))
+                        return true;
+
+                    return false;
+                case TextureUsage.Diffuse:
+                    newPath = path.Replace("_d.tex", "_base.tex");
+                    if (data.FileExists(newPath))
+                        return true;
+
+                    newPath = path.Replace("_d_", "_base_");
+                    if (data.FileExists(newPath))
+                        return true;
+
+                    return false;
+                case TextureUsage.Specular:
+                    return false;
+                default:                    throw new ArgumentOutOfRangeException(nameof(usage), usage, null);
+            }
+        }
+    }
+
+    private void MigrateMaterial(string filePath, IReadOnlyList<(string GamePath, IModDataContainer Container)> redirections)
+    {
+        try
+        {
+            var bytes = File.ReadAllBytes(filePath);
+            var mtrl  = new MtrlFile(bytes);
+            if (!CheckUpdateNeeded(mtrl))
+                return;
+
+            // Update colorsets, flags and character shader package.
+            var changes = mtrl.MigrateToDawntrail();
+
+            if (!changes)
+                switch (mtrl.ShaderPackage.Name)
+                {
+                    case "hair.shpk": break;
+                    case "characterglass.shpk":
+                        SetGlassReferenceValues(mtrl);
+                        changes = true;
+                        break;
+                }
+
+            // Remove DX11 flags and update paths if necessary.
+            foreach (ref var tex in mtrl.Textures.AsSpan())
+            {
+                if (tex.DX11)
+                {
+                    changes = true;
+                    if (GamePaths.Tex.HandleDx11Path(tex, out var newPath))
+                        tex.Path = newPath;
+                    tex.DX11 = false;
+                }
+
+                if (gameData.FileExists(tex.Path))
+                    continue;
+            }
+
+            // Dyeing, from TT.
+            if (mtrl.DyeTable is ColorDyeTable dye)
+                foreach (ref var row in dye.AsRows())
+                    row.Template += 1000;
+        }
+        catch
+        {
+            // ignored
+        }
+
+        static bool CheckUpdateNeeded(MtrlFile mtrl)
+        {
+            if (!mtrl.IsDawntrail)
+                return true;
+
+            if (mtrl.ShaderPackage.Name is not "hair.shpk")
+                return false;
+
+            var foundOld = 0;
+            foreach (var c in mtrl.ShaderPackage.Constants)
+            {
+                switch (c.Id)
+                {
+                    case 0x36080AD0: foundOld |= 1; break; // == 1, from TT
+                    case 0x992869AB: foundOld |= 2; break; // == 3 (skin) or 4 (hair) from TT
+                }
+
+                if (foundOld is 3)
+                    return true;
+            }
+
+            return false;
+        }
+    }
+
+    private void MigrateFileSwaps(string swapFrom, string swapTo, IModDataContainer container)
+    {
+        var fromExists = gameData.FileExists(swapFrom);
+        var toExists   = gameData.FileExists(swapTo);
+        if (fromExists && toExists)
+            return;
+
+        if (ResourceTypeExtensions.FromExtension(Path.GetExtension(swapFrom.AsSpan())) is not ResourceType.Tex
+         || ResourceTypeExtensions.FromExtension(Path.GetExtension(swapTo.AsSpan())) is not ResourceType.Tex)
+        {
+            _messages.Add(
+                $"Could not migrate file swap {swapFrom} -> {swapTo} in {container.Mod.Name}: {container.GetFullName()}. Only textures may be migrated.{(fromExists ? "\n\tSource File does not exist." : "")}{(toExists ? "\n\tTarget File does not exist." : "")}");
+            return;
+        }
+
+        // try to migrate texture swaps
+    }
+}

Penumbra/UI/Tabs/Debug/DebugTab.cs

@@ -106,6 +106,7 @@ public class DebugTab : Window, ITab, IUiService
     private readonly SchedulerResourceManagementService _schedulerService;
     private readonly ObjectIdentification               _objectIdentification;
     private readonly RenderTargetDrawer                 _renderTargetDrawer;
+    private readonly ModMigratorDebug                   _modMigratorDebug;
 
     public DebugTab(PerformanceTracker performance, Configuration config, CollectionManager collectionManager, ObjectManager objects,
         IClientState clientState, IDataManager dataManager,
@@ -116,7 +117,8 @@ public class DebugTab : Window, ITab, IUiService
         TextureManager textureManager, ShaderReplacementFixer shaderReplacementFixer, RedrawService redraws, DictEmote emotes,
         Diagnostics diagnostics, IpcTester ipcTester, CrashHandlerPanel crashHandlerPanel, TexHeaderDrawer texHeaderDrawer,
         HookOverrideDrawer hookOverrides, RsfService rsfService, GlobalVariablesDrawer globalVariablesDrawer,
-        SchedulerResourceManagementService schedulerService, ObjectIdentification objectIdentification, RenderTargetDrawer renderTargetDrawer)
+        SchedulerResourceManagementService schedulerService, ObjectIdentification objectIdentification, RenderTargetDrawer renderTargetDrawer,
+        ModMigratorDebug modMigratorDebug)
         : base("Penumbra Debug Window", ImGuiWindowFlags.NoCollapse)
     {
         IsOpen = true;
@@ -158,6 +160,7 @@ public class DebugTab : Window, ITab, IUiService
         _schedulerService          = schedulerService;
         _objectIdentification      = objectIdentification;
         _renderTargetDrawer        = renderTargetDrawer;
+        _modMigratorDebug          = modMigratorDebug;
         _objects                   = objects;
         _clientState               = clientState;
         _dataManager               = dataManager;
@@ -190,6 +193,7 @@ public class DebugTab : Window, ITab, IUiService
         DrawActorsDebug();
         DrawCollectionCaches();
         _texHeaderDrawer.Draw();
+        _modMigratorDebug.Draw();
         DrawShaderReplacementFixer();
         DrawData();
         DrawCrcCache();
@@ -591,6 +595,7 @@ public class DebugTab : Window, ITab, IUiService
                         {
                             ImUtf8.DrawTableColumn($"{_objects[idx]}");
                         }
+
                         ImUtf8.DrawTableColumn(gameObjectPtr.Utf8Name.Span);
                         var collection = _collectionResolver.IdentifyCollection(gameObjectPtr.AsObject, true);
                         ImUtf8.DrawTableColumn(collection.ModCollection.Identity.Name);
@@ -751,7 +756,7 @@ public class DebugTab : Window, ITab, IUiService
         DrawChangedItemTest();
     }
 
-    private          string                                     _changedItemPath = string.Empty;
+    private          string                                    _changedItemPath = string.Empty;
     private readonly Dictionary<string, IIdentifiedObjectData> _changedItems    = [];
 
     private void DrawChangedItemTest()

Penumbra/UI/Tabs/Debug/ModMigratorDebug.cs

@@ -0,0 +1,55 @@
+using ImGuiNET;
+using OtterGui.Services;
+using OtterGui.Text;
+using Penumbra.Services;
+
+namespace Penumbra.UI.Tabs.Debug;
+
+public class ModMigratorDebug(ModMigrator migrator) : IUiService
+{
+    private string _inputPath  = string.Empty;
+    private string _outputPath = string.Empty;
+    private Task?  _indexTask;
+    private Task?  _mdlTask;
+
+    public void Draw()
+    {
+        if (!ImUtf8.CollapsingHeaderId("Mod Migrator"u8))
+            return;
+
+        ImUtf8.InputText("##input"u8,  ref _inputPath,  "Input Path..."u8);
+        ImUtf8.InputText("##output"u8, ref _outputPath, "Output Path..."u8);
+
+        if (ImUtf8.ButtonEx("Create Index Texture"u8, "Requires input to be a path to a normal texture."u8, default, _inputPath.Length == 0
+             || _outputPath.Length == 0
+             || _indexTask is
+                {
+                    IsCompleted: false,
+                }))
+            _indexTask = migrator.CreateIndexFile(_inputPath, _outputPath);
+
+        if (_indexTask is not null)
+        {
+            ImGui.SameLine();
+            ImUtf8.TextFrameAligned($"{_indexTask.Status}");
+        }
+
+        if (ImUtf8.ButtonEx("Update Model File"u8, "Requires input to be a path to a mdl."u8, default, _inputPath.Length == 0
+             || _outputPath.Length == 0
+             || _mdlTask is
+                {
+                    IsCompleted: false,
+                }))
+            _mdlTask = Task.Run(() =>
+            {
+                File.Copy(_inputPath, _outputPath, true);
+                MigrationManager.TryMigrateSingleModel(_outputPath, false);
+            });
+
+        if (_mdlTask is not null)
+        {
+            ImGui.SameLine();
+            ImUtf8.TextFrameAligned($"{_mdlTask.Status}");
+        }
+    }
+}