Improve deduplicator and normalizer.

Penumbra/Mods/Editor/DuplicateManager.cs

@@ -4,6 +4,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Security.Cryptography;
+using System.Threading;
 using System.Threading.Tasks;
 using Penumbra.Mods.Manager;
 using Penumbra.Services;
@@ -28,21 +29,23 @@ public class DuplicateManager
         => _duplicates;
 
     public long SavedSpace { get; private set; } = 0;
-    public bool Finished   { get; private set; } = true;
+    public Task Worker     { get; private set; } = Task.CompletedTask;
+
+    private CancellationTokenSource _cancellationTokenSource = new();
 
     public void StartDuplicateCheck(IEnumerable<FileRegistry> files)
     {
-        if (!Finished)
+        if (!Worker.IsCompleted)
             return;
 
-        Finished = false;
         var filesTmp = files.OrderByDescending(f => f.FileSize).ToArray();
-        Task.Run(() => CheckDuplicates(filesTmp));
+        _cancellationTokenSource = new CancellationTokenSource();
+        Worker                   = Task.Run(() => CheckDuplicates(filesTmp, _cancellationTokenSource.Token), _cancellationTokenSource.Token);
     }
 
     public void DeleteDuplicates(ModFileCollection files, Mod mod, ISubMod option, bool useModManager)
     {
-        if (!Finished || _duplicates.Count == 0)
+        if (!Worker.IsCompleted || _duplicates.Count == 0)
             return;
 
         foreach (var (set, _, _) in _duplicates)
@@ -62,7 +65,9 @@ public class DuplicateManager
 
     public void Clear()
     {
-        Finished   = true;
+        _cancellationTokenSource.Cancel();
+        Worker     = Task.CompletedTask;
+        _duplicates.Clear();
         SavedSpace = 0;
     }
 
@@ -110,7 +115,7 @@ public class DuplicateManager
         return to;
     }
 
-    private void CheckDuplicates(IReadOnlyList<FileRegistry> files)
+    private void CheckDuplicates(IReadOnlyList<FileRegistry> files, CancellationToken token)
     {
         _duplicates.Clear();
         SavedSpace = 0;
@@ -122,8 +127,7 @@ public class DuplicateManager
             if (file.SubModUsage.Any(f => f.Item2.Path.StartsWith("ui/"u8)))
                 continue;
 
-            if (Finished)
+            token.ThrowIfCancellationRequested();
-                return;
 
             if (file.FileSize == lastSize)
             {
@@ -132,7 +136,7 @@ public class DuplicateManager
             }
 
             if (list.Count >= 2)
-                CheckMultiDuplicates(list, lastSize);
+                CheckMultiDuplicates(list, lastSize, token);
 
             lastSize = file.FileSize;
 
@@ -141,26 +145,23 @@ public class DuplicateManager
         }
 
         if (list.Count >= 2)
-            CheckMultiDuplicates(list, lastSize);
+            CheckMultiDuplicates(list, lastSize, token);
 
         _duplicates.Sort((a, b) => a.Size != b.Size ? b.Size.CompareTo(a.Size) : a.Paths[0].CompareTo(b.Paths[0]));
-        Finished = true;
     }
 
-    private void CheckMultiDuplicates(IReadOnlyList<FullPath> list, long size)
+    private void CheckMultiDuplicates(IReadOnlyList<FullPath> list, long size, CancellationToken token)
     {
         var hashes = list.Select(f => (f, ComputeHash(f))).ToList();
         while (hashes.Count > 0)
         {
-            if (Finished)
+            token.ThrowIfCancellationRequested();
-                return;
 
             var set  = new HashSet<FullPath> { hashes[0].Item1 };
             var hash = hashes[0];
             for (var j = 1; j < hashes.Count; ++j)
             {
-                if (Finished)
+                token.ThrowIfCancellationRequested();
-                    return;
 
                 if (CompareHashes(hash.Item2, hashes[j].Item2) && CompareFilesDirectly(hashes[0].Item1, hashes[j].Item1))
                     set.Add(hashes[j].Item1);
@@ -245,10 +246,10 @@ public class DuplicateManager
             var mod = new Mod(modDirectory);
             _modManager.Creator.ReloadMod(mod, true, out _);
 
-            Finished = false;
+            Clear();
             var files = new ModFileCollection();
             files.UpdateAll(mod, mod.Default);
-            CheckDuplicates(files.Available.OrderByDescending(f => f.FileSize).ToArray());
+            CheckDuplicates(files.Available.OrderByDescending(f => f.FileSize).ToArray(), CancellationToken.None);
             DeleteDuplicates(files, mod, mod.Default, false);
         }
         catch (Exception e)

Penumbra/Mods/Editor/ModNormalizer.cs

@@ -19,11 +19,13 @@ public class ModNormalizer
     private string _normalizationDirName = null!;
     private string _oldDirName           = null!;
 
-    public int Step       { get; private set; }
+    public int  Step       { get; private set; }
-    public int TotalSteps { get; private set; }
+    public int  TotalSteps { get; private set; }
+    public Task Worker     { get; private set; } = Task.CompletedTask;
+
 
     public bool Running
-        => Step < TotalSteps;
+        => !Worker.IsCompleted;
 
     public ModNormalizer(ModManager modManager)
         => _modManager = modManager;
@@ -39,7 +41,7 @@ public class ModNormalizer
         Step                  = 0;
         TotalSteps            = mod.TotalFileCount + 5;
 
-        Task.Run(NormalizeSync);
+        Worker = Task.Run(NormalizeSync);
     }
 
     private void NormalizeSync()
@@ -280,7 +282,7 @@ public class ModNormalizer
 
     private void ApplyRedirections()
     {
-        foreach (var option in Mod.AllSubMods.OfType<SubMod>())
+        foreach (var option in Mod.AllSubMods)
             _modManager.OptionEditor.OptionSetFiles(Mod, option.GroupIdx, option.OptionIdx,
                 _redirections[option.GroupIdx + 1][option.OptionIdx]);
 

Penumbra/UI/AdvancedWindow/ModEditWindow.cs

@@ -126,7 +126,7 @@ public partial class ModEditWindow : Window, IDisposable
         if (swaps > 0)
             sb.Append($"   |   {swaps} Swaps");
 
-        _allowReduplicate = redirections != _editor.Files.Available.Count || _editor.Files.Available.Count > 0;
+        _allowReduplicate = redirections != _editor.Files.Available.Count || _editor.Files.Missing.Count > 0 || unused > 0;
         sb.Append(WindowBaseLabel);
         WindowName = sb.ToString();
     }
@@ -275,10 +275,17 @@ public partial class ModEditWindow : Window, IDisposable
         if (!tab)
             return;
 
-        var buttonText = _editor.Duplicates.Finished ? "Scan for Duplicates###ScanButton" : "Scanning for Duplicates...###ScanButton";
+        if (_editor.Duplicates.Worker.IsCompleted)
-        if (ImGuiUtil.DrawDisabledButton(buttonText, Vector2.Zero, "Search for identical files in this mod. This may take a while.",
+        {
-                !_editor.Duplicates.Finished))
+            if (ImGuiUtil.DrawDisabledButton("Scan for Duplicates", Vector2.Zero,
-            _editor.Duplicates.StartDuplicateCheck(_editor.Files.Available);
+                    "Search for identical files in this mod. This may take a while.", false))
+                _editor.Duplicates.StartDuplicateCheck(_editor.Files.Available);
+        }
+        else
+        {
+            if (ImGuiUtil.DrawDisabledButton("Cancel Scanning for Duplicates", Vector2.Zero, "Cancel the current scanning operation...", false))
+                _editor.Duplicates.Clear();
+        }
 
         const string desc =
             "Tries to create a unique copy of a file for every game path manipulated and put them in [Groupname]/[Optionname]/[GamePath] order.\n"
@@ -290,28 +297,21 @@ public partial class ModEditWindow : Window, IDisposable
         var tt = _allowReduplicate ? desc :
             modifier ? desc : desc + $"\n\nNo duplicates detected! Hold {_config.DeleteModModifier} to force normalization anyway.";
 
-        if (ImGuiUtil.DrawDisabledButton("Re-Duplicate and Normalize Mod", Vector2.Zero, tt, !_allowReduplicate && !modifier))
-        {
-            _editor.ModNormalizer.Normalize(_mod!);
-            _editor.LoadMod(_mod!, _editor.GroupIdx, _editor.OptionIdx);
-        }
-
         if (_editor.ModNormalizer.Running)
         {
-            using var popup = ImRaii.Popup("Normalization", ImGuiWindowFlags.Modal);
             ImGui.ProgressBar((float)_editor.ModNormalizer.Step / _editor.ModNormalizer.TotalSteps,
                 new Vector2(300 * UiHelpers.Scale, ImGui.GetFrameHeight()),
                 $"{_editor.ModNormalizer.Step} / {_editor.ModNormalizer.TotalSteps}");
         }
-
+        else if(ImGuiUtil.DrawDisabledButton("Re-Duplicate and Normalize Mod", Vector2.Zero, tt, !_allowReduplicate && !modifier))
-        if (!_editor.Duplicates.Finished)
         {
-            ImGui.SameLine();
+            _editor.ModNormalizer.Normalize(_mod!);
-            if (ImGui.Button("Cancel"))
+            _editor.ModNormalizer.Worker.ContinueWith(_ => _editor.LoadMod(_mod!, _editor.GroupIdx, _editor.OptionIdx));
-                _editor.Duplicates.Clear();
-            return;
         }
 
+        if (!_editor.Duplicates.Worker.IsCompleted)
+            return;
+
         if (_editor.Duplicates.Duplicates.Count == 0)
         {
             ImGui.NewLine();