ToxicMushroom/Penumbra
1
0
Fork 0
mirror of https://github.com/xivdev/Penumbra.git synced 2025-12-12 18:27:24 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve deduplicator and normalizer.

Browse source
This commit is contained in:
Ottermandias 2023-05-27 13:57:01 +02:00
parent f938531e21
commit 0243e7a633
3 changed files with 45 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

39
Penumbra/Mods/Editor/DuplicateManager.cs
View file

@ -4,6 +4,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using Penumbra.Mods.Manager;
using Penumbra.Services;
@ -28,21 +29,23 @@ public class DuplicateManager
=> _duplicates;
public long SavedSpace { get; private set; } = 0;
public bool Finished { get; private set; } = true;
public Task Worker { get; private set; } = Task.CompletedTask;
private CancellationTokenSource _cancellationTokenSource = new();
public void StartDuplicateCheck(IEnumerable<FileRegistry> files)
{
if (!Finished)
if (!Worker.IsCompleted)
return;
Finished = false;
var filesTmp = files.OrderByDescending(f => f.FileSize).ToArray();
Task.Run(() => CheckDuplicates(filesTmp));
_cancellationTokenSource = new CancellationTokenSource();
Worker = Task.Run(() => CheckDuplicates(filesTmp, _cancellationTokenSource.Token), _cancellationTokenSource.Token);
}
public void DeleteDuplicates(ModFileCollection files, Mod mod, ISubMod option, bool useModManager)
{
if (!Finished || _duplicates.Count == 0)
if (!Worker.IsCompleted || _duplicates.Count == 0)
return;
foreach (var (set, _, _) in _duplicates)
@ -62,7 +65,9 @@ public class DuplicateManager
public void Clear()
{
Finished = true;
_cancellationTokenSource.Cancel();
Worker = Task.CompletedTask;
_duplicates.Clear();
SavedSpace = 0;
}
@ -110,7 +115,7 @@ public class DuplicateManager
return to;
}
private void CheckDuplicates(IReadOnlyList<FileRegistry> files)
private void CheckDuplicates(IReadOnlyList<FileRegistry> files, CancellationToken token)
{
_duplicates.Clear();
SavedSpace = 0;
@ -122,8 +127,7 @@ public class DuplicateManager
if (file.SubModUsage.Any(f => f.Item2.Path.StartsWith("ui/"u8)))
continue;
if (Finished)
return;
token.ThrowIfCancellationRequested();
if (file.FileSize == lastSize)
{
@ -132,7 +136,7 @@ public class DuplicateManager
}
if (list.Count >= 2)
CheckMultiDuplicates(list, lastSize);
CheckMultiDuplicates(list, lastSize, token);
lastSize = file.FileSize;
@ -141,26 +145,23 @@ public class DuplicateManager
}
if (list.Count >= 2)
CheckMultiDuplicates(list, lastSize);
CheckMultiDuplicates(list, lastSize, token);
_duplicates.Sort((a, b) => a.Size != b.Size ? b.Size.CompareTo(a.Size) : a.Paths[0].CompareTo(b.Paths[0]));
Finished = true;
}
private void CheckMultiDuplicates(IReadOnlyList<FullPath> list, long size)
private void CheckMultiDuplicates(IReadOnlyList<FullPath> list, long size, CancellationToken token)
{
var hashes = list.Select(f => (f, ComputeHash(f))).ToList();
while (hashes.Count > 0)
{
if (Finished)
return;
token.ThrowIfCancellationRequested();
var set = new HashSet<FullPath> { hashes[0].Item1 };
var hash = hashes[0];
for (var j = 1; j < hashes.Count; ++j)
{
if (Finished)
return;
token.ThrowIfCancellationRequested();
if (CompareHashes(hash.Item2, hashes[j].Item2) && CompareFilesDirectly(hashes[0].Item1, hashes[j].Item1))
set.Add(hashes[j].Item1);
@ -245,10 +246,10 @@ public class DuplicateManager
var mod = new Mod(modDirectory);
_modManager.Creator.ReloadMod(mod, true, out _);
Finished = false;
Clear();
var files = new ModFileCollection();
files.UpdateAll(mod, mod.Default);
CheckDuplicates(files.Available.OrderByDescending(f => f.FileSize).ToArray());
CheckDuplicates(files.Available.OrderByDescending(f => f.FileSize).ToArray(), CancellationToken.None);
DeleteDuplicates(files, mod, mod.Default, false);
}
catch (Exception e)

12
Penumbra/Mods/Editor/ModNormalizer.cs
View file

@ -19,11 +19,13 @@ public class ModNormalizer
private string _normalizationDirName = null!;
private string _oldDirName = null!;
public int Step { get; private set; }
public int TotalSteps { get; private set; }
public int Step { get; private set; }
public int TotalSteps { get; private set; }
public Task Worker { get; private set; } = Task.CompletedTask;
public bool Running
=> Step < TotalSteps;
=> !Worker.IsCompleted;
public ModNormalizer(ModManager modManager)
=> _modManager = modManager;
@ -39,7 +41,7 @@ public class ModNormalizer
Step = 0;
TotalSteps = mod.TotalFileCount + 5;
Task.Run(NormalizeSync);
Worker = Task.Run(NormalizeSync);
}
private void NormalizeSync()
@ -280,7 +282,7 @@ public class ModNormalizer
private void ApplyRedirections()
{
foreach (var option in Mod.AllSubMods.OfType<SubMod>())
foreach (var option in Mod.AllSubMods)
_modManager.OptionEditor.OptionSetFiles(Mod, option.GroupIdx, option.OptionIdx,
_redirections[option.GroupIdx + 1][option.OptionIdx]);

36
Penumbra/UI/AdvancedWindow/ModEditWindow.cs
View file

@ -126,7 +126,7 @@ public partial class ModEditWindow : Window, IDisposable
if (swaps > 0)
sb.Append($" | {swaps} Swaps");
_allowReduplicate = redirections != _editor.Files.Available.Count || _editor.Files.Available.Count > 0;
_allowReduplicate = redirections != _editor.Files.Available.Count || _editor.Files.Missing.Count > 0 || unused > 0;
sb.Append(WindowBaseLabel);
WindowName = sb.ToString();
}
@ -275,10 +275,17 @@ public partial class ModEditWindow : Window, IDisposable
if (!tab)
return;
var buttonText = _editor.Duplicates.Finished ? "Scan for Duplicates###ScanButton" : "Scanning for Duplicates...###ScanButton";
if (ImGuiUtil.DrawDisabledButton(buttonText, Vector2.Zero, "Search for identical files in this mod. This may take a while.",
!_editor.Duplicates.Finished))
_editor.Duplicates.StartDuplicateCheck(_editor.Files.Available);
if (_editor.Duplicates.Worker.IsCompleted)
{
if (ImGuiUtil.DrawDisabledButton("Scan for Duplicates", Vector2.Zero,
"Search for identical files in this mod. This may take a while.", false))
_editor.Duplicates.StartDuplicateCheck(_editor.Files.Available);
}
else
{
if (ImGuiUtil.DrawDisabledButton("Cancel Scanning for Duplicates", Vector2.Zero, "Cancel the current scanning operation...", false))
_editor.Duplicates.Clear();
}
const string desc =
"Tries to create a unique copy of a file for every game path manipulated and put them in [Groupname]/[Optionname]/[GamePath] order.\n"
@ -290,28 +297,21 @@ public partial class ModEditWindow : Window, IDisposable
var tt = _allowReduplicate ? desc :
modifier ? desc : desc + $"\n\nNo duplicates detected! Hold {_config.DeleteModModifier} to force normalization anyway.";
if (ImGuiUtil.DrawDisabledButton("Re-Duplicate and Normalize Mod", Vector2.Zero, tt, !_allowReduplicate && !modifier))
{
_editor.ModNormalizer.Normalize(_mod!);
_editor.LoadMod(_mod!, _editor.GroupIdx, _editor.OptionIdx);
}
if (_editor.ModNormalizer.Running)
{
using var popup = ImRaii.Popup("Normalization", ImGuiWindowFlags.Modal);
ImGui.ProgressBar((float)_editor.ModNormalizer.Step / _editor.ModNormalizer.TotalSteps,
new Vector2(300 * UiHelpers.Scale, ImGui.GetFrameHeight()),
$"{_editor.ModNormalizer.Step} / {_editor.ModNormalizer.TotalSteps}");
}
if (!_editor.Duplicates.Finished)
else if(ImGuiUtil.DrawDisabledButton("Re-Duplicate and Normalize Mod", Vector2.Zero, tt, !_allowReduplicate && !modifier))
{
ImGui.SameLine();
if (ImGui.Button("Cancel"))
_editor.Duplicates.Clear();
return;
_editor.ModNormalizer.Normalize(_mod!);
_editor.ModNormalizer.Worker.ContinueWith(_ => _editor.LoadMod(_mod!, _editor.GroupIdx, _editor.OptionIdx));
}
if (!_editor.Duplicates.Worker.IsCompleted)
return;
if (_editor.Duplicates.Duplicates.Count == 0)
{
ImGui.NewLine();