diff --git a/Penumbra/Models/Deduplicator.cs b/Penumbra/Models/Deduplicator.cs deleted file mode 100644 index deb3e411..00000000 --- a/Penumbra/Models/Deduplicator.cs +++ /dev/null @@ -1,168 +0,0 @@ -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Security.Cryptography; -using Dalamud.Plugin; -using Penumbra.Util; - -namespace Penumbra.Models -{ - public class Deduplicator - { - private readonly DirectoryInfo _baseDir; - private readonly ModMeta _mod; - private SHA256? _hasher; - - private readonly Dictionary< long, List< FileInfo > > _filesBySize = new(); - - private SHA256 Sha() - { - _hasher ??= SHA256.Create(); - return _hasher; - } - - public Deduplicator( DirectoryInfo baseDir, ModMeta mod ) - { - _baseDir = baseDir; - _mod = mod; - BuildDict(); - } - - private void BuildDict() - { - foreach( var file in _baseDir.EnumerateFiles( "*.*", SearchOption.AllDirectories ) ) - { - var fileLength = file.Length; - if( _filesBySize.TryGetValue( fileLength, out var files ) ) - { - files.Add( file ); - } - else - { - _filesBySize[ fileLength ] = new List< FileInfo >() { file }; - } - } - } - - public void Run() - { - foreach( var pair in _filesBySize.Where( pair => pair.Value.Count >= 2 ) ) - { - if( pair.Value.Count == 2 ) - { - if( CompareFilesDirectly( pair.Value[ 0 ], pair.Value[ 1 ] ) ) - { - ReplaceFile( pair.Value[ 0 ], pair.Value[ 1 ] ); - } - } - else - { - var deleted = Enumerable.Repeat( false, pair.Value.Count ).ToArray(); - var hashes = pair.Value.Select( ComputeHash ).ToArray(); - - for( var i = 0; i < pair.Value.Count; ++i ) - { - if( deleted[ i ] ) - { - continue; - } - - for( var j = i + 1; j < pair.Value.Count; ++j ) - { - if( deleted[ j ] || !CompareHashes( hashes[ i ], hashes[ j ] ) ) - { - continue; - } - - ReplaceFile( pair.Value[ i ], pair.Value[ j ] ); - deleted[ j ] = true; - } - } - } - } - - ClearEmptySubDirectories( _baseDir ); - } - - private void ReplaceFile( FileInfo f1, FileInfo f2 ) - { - RelPath relName1 = new( f1, _baseDir ); - RelPath relName2 = new( f2, _baseDir ); - - var inOption = false; - foreach( var group in _mod.Groups.Select( g => g.Value.Options ) ) - { - foreach( var option in group ) - { - if( option.OptionFiles.TryGetValue( relName2, out var values ) ) - { - inOption = true; - foreach( var value in values ) - { - option.AddFile( relName1, value ); - } - - option.OptionFiles.Remove( relName2 ); - } - } - } - - if( !inOption ) - { - const string duplicates = "Duplicates"; - if( !_mod.Groups.ContainsKey( duplicates ) ) - { - OptionGroup info = new() - { - GroupName = duplicates, - SelectionType = SelectType.Single, - Options = new List< Option >() - { - new() - { - OptionName = "Required", - OptionDesc = "", - OptionFiles = new Dictionary< RelPath, HashSet< GamePath > >() - } - } - }; - _mod.Groups.Add( duplicates, info ); - } - - _mod.Groups[ duplicates ].Options[ 0 ].AddFile( relName1, new GamePath( relName2 ) ); - _mod.Groups[ duplicates ].Options[ 0 ].AddFile( relName1, new GamePath( relName1 ) ); - } - - PluginLog.Information( $"File {relName1} and {relName2} are identical. Deleting the second." ); - f2.Delete(); - } - - public static bool CompareFilesDirectly( FileInfo f1, FileInfo f2 ) - => File.ReadAllBytes( f1.FullName ).SequenceEqual( File.ReadAllBytes( f2.FullName ) ); - - public static bool CompareHashes( byte[] f1, byte[] f2 ) - => StructuralComparisons.StructuralEqualityComparer.Equals( f1, f2 ); - - public byte[] ComputeHash( FileInfo f ) - { - var stream = File.OpenRead( f.FullName ); - var ret = Sha().ComputeHash( stream ); - stream.Dispose(); - return ret; - } - - // Does not delete the base directory itself even if it is completely empty at the end. - public static void ClearEmptySubDirectories( DirectoryInfo baseDir ) - { - foreach( var subDir in baseDir.GetDirectories() ) - { - ClearEmptySubDirectories( subDir ); - if( subDir.GetFiles().Length == 0 && subDir.GetDirectories().Length == 0 ) - { - subDir.Delete(); - } - } - } - } -} \ No newline at end of file diff --git a/Penumbra/Models/ModCleanup.cs b/Penumbra/Models/ModCleanup.cs new file mode 100644 index 00000000..6c08e8ee --- /dev/null +++ b/Penumbra/Models/ModCleanup.cs @@ -0,0 +1,378 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using Dalamud.Plugin; +using Penumbra.Util; + +namespace Penumbra.Models +{ + public class ModCleanup + { + private const string Duplicates = "Duplicates"; + private const string Required = "Required"; + + + private readonly DirectoryInfo _baseDir; + private readonly ModMeta _mod; + private SHA256? _hasher; + + private readonly Dictionary< long, List< FileInfo > > _filesBySize = new(); + + private SHA256 Sha() + { + _hasher ??= SHA256.Create(); + return _hasher; + } + + private ModCleanup( DirectoryInfo baseDir, ModMeta mod ) + { + _baseDir = baseDir; + _mod = mod; + BuildDict(); + } + + private void BuildDict() + { + foreach( var file in _baseDir.EnumerateFiles( "*.*", SearchOption.AllDirectories ) ) + { + var fileLength = file.Length; + if( _filesBySize.TryGetValue( fileLength, out var files ) ) + { + files.Add( file ); + } + else + { + _filesBySize[ fileLength ] = new List< FileInfo >() { file }; + } + } + } + + private static Option FindOrCreateDuplicates( ModMeta meta ) + { + static Option RequiredOption() + => new() + { + OptionName = Required, + OptionDesc = "", + OptionFiles = new Dictionary< RelPath, HashSet< GamePath > >(), + }; + + if( meta.Groups.TryGetValue( Duplicates, out var duplicates ) ) + { + var idx = duplicates.Options.FindIndex( o => o.OptionName == Required ); + if( idx >= 0 ) + { + return duplicates.Options[ idx ]; + } + + duplicates.Options.Add( RequiredOption() ); + return duplicates.Options.Last(); + } + + meta.Groups.Add( Duplicates, new OptionGroup + { + GroupName = Duplicates, + SelectionType = SelectType.Single, + Options = new List< Option > { RequiredOption() }, + } ); + + return meta.Groups[ Duplicates ].Options.First(); + } + + public static void Deduplicate( DirectoryInfo baseDir, ModMeta mod ) + { + var dedup = new ModCleanup( baseDir, mod ); + foreach( var pair in dedup._filesBySize.Where( pair => pair.Value.Count >= 2 ) ) + { + if( pair.Value.Count == 2 ) + { + if( CompareFilesDirectly( pair.Value[ 0 ], pair.Value[ 1 ] ) ) + { + dedup.ReplaceFile( pair.Value[ 0 ], pair.Value[ 1 ] ); + } + } + else + { + var deleted = Enumerable.Repeat( false, pair.Value.Count ).ToArray(); + var hashes = pair.Value.Select( dedup.ComputeHash ).ToArray(); + + for( var i = 0; i < pair.Value.Count; ++i ) + { + if( deleted[ i ] ) + { + continue; + } + + for( var j = i + 1; j < pair.Value.Count; ++j ) + { + if( deleted[ j ] || !CompareHashes( hashes[ i ], hashes[ j ] ) ) + { + continue; + } + + dedup.ReplaceFile( pair.Value[ i ], pair.Value[ j ] ); + deleted[ j ] = true; + } + } + } + } + + CleanUpDuplicates( mod ); + ClearEmptySubDirectories( dedup._baseDir ); + } + + private void ReplaceFile( FileInfo f1, FileInfo f2 ) + { + RelPath relName1 = new( f1, _baseDir ); + RelPath relName2 = new( f2, _baseDir ); + + var inOption1 = false; + var inOption2 = false; + foreach( var option in _mod.Groups.SelectMany( g => g.Value.Options ) ) + { + if( option.OptionFiles.ContainsKey( relName1 ) ) + { + inOption1 = true; + } + + if( !option.OptionFiles.TryGetValue( relName2, out var values ) ) + { + continue; + } + + inOption2 = true; + + foreach( var value in values ) + { + option.AddFile( relName1, value ); + } + + option.OptionFiles.Remove( relName2 ); + } + + if( !inOption1 || !inOption2 ) + { + var duplicates = FindOrCreateDuplicates( _mod ); + if( !inOption1 ) + { + duplicates.AddFile( relName1, new GamePath( relName2, 0 ) ); + } + + if( !inOption2 ) + { + duplicates.AddFile( relName1, new GamePath( relName1, 0 ) ); + } + } + + PluginLog.Information( $"File {relName1} and {relName2} are identical. Deleting the second." ); + f2.Delete(); + } + + public static bool CompareFilesDirectly( FileInfo f1, FileInfo f2 ) + => File.ReadAllBytes( f1.FullName ).SequenceEqual( File.ReadAllBytes( f2.FullName ) ); + + public static bool CompareHashes( byte[] f1, byte[] f2 ) + => StructuralComparisons.StructuralEqualityComparer.Equals( f1, f2 ); + + public byte[] ComputeHash( FileInfo f ) + { + var stream = File.OpenRead( f.FullName ); + var ret = Sha().ComputeHash( stream ); + stream.Dispose(); + return ret; + } + + // Does not delete the base directory itself even if it is completely empty at the end. + public static void ClearEmptySubDirectories( DirectoryInfo baseDir ) + { + foreach( var subDir in baseDir.GetDirectories() ) + { + ClearEmptySubDirectories( subDir ); + if( subDir.GetFiles().Length == 0 && subDir.GetDirectories().Length == 0 ) + { + subDir.Delete(); + } + } + } + + private static bool FileIsInAnyGroup( ModMeta meta, RelPath relPath, bool exceptDuplicates = false ) + { + var groupEnumerator = exceptDuplicates + ? meta.Groups.Values.Where( G => G.GroupName != Duplicates ) + : meta.Groups.Values; + return groupEnumerator.SelectMany( group => group.Options ) + .Any( option => option.OptionFiles.ContainsKey( relPath ) ); + } + + private static void CleanUpDuplicates( ModMeta meta ) + { + if( !meta.Groups.TryGetValue( Duplicates, out var info ) ) + { + return; + } + + var requiredIdx = info.Options.FindIndex( o => o.OptionName == Required ); + if( requiredIdx >= 0 ) + { + var required = info.Options[ requiredIdx ]; + foreach( var kvp in required.OptionFiles.ToArray() ) + { + if( kvp.Value.Count > 1 || FileIsInAnyGroup( meta, kvp.Key, true ) ) + { + continue; + } + + if( kvp.Value.Count == 0 || kvp.Value.First().CompareTo( new GamePath( kvp.Key, 0 ) ) == 0 ) + { + required.OptionFiles.Remove( kvp.Key ); + } + } + + if( required.OptionFiles.Count == 0 ) + { + info.Options.RemoveAt( requiredIdx ); + } + } + + if( info.Options.Count == 0 ) + { + meta.Groups.Remove( Duplicates ); + } + } + + public enum GroupType + { + Both = 0, + Single = 1, + Multi = 2, + }; + + private static void RemoveFromGroups( ModMeta meta, RelPath relPath, GamePath gamePath, GroupType type = GroupType.Both, + bool skipDuplicates = true ) + { + if( meta.Groups.Count == 0 ) + { + return; + } + + var enumerator = type switch + { + GroupType.Both => meta.Groups.Values, + GroupType.Single => meta.Groups.Values.Where( g => g.SelectionType == SelectType.Single ), + GroupType.Multi => meta.Groups.Values.Where( g => g.SelectionType == SelectType.Multi ), + _ => throw new InvalidEnumArgumentException( "Invalid Enum in RemoveFromGroups" ), + }; + foreach( var group in enumerator ) + { + var optionEnum = skipDuplicates + ? group.Options.Where( o => group.GroupName != Duplicates || o.OptionName != Required ) + : group.Options; + foreach( var option in optionEnum ) + { + if( option.OptionFiles.TryGetValue( relPath, out var gamePaths ) && gamePaths.Remove( gamePath ) && gamePaths.Count == 0 ) + { + option.OptionFiles.Remove( relPath ); + } + } + } + } + + public static bool MoveFile( ModMeta meta, string basePath, RelPath oldRelPath, RelPath newRelPath ) + { + if( oldRelPath == newRelPath ) + { + return true; + } + + try + { + var newFullPath = Path.Combine( basePath, newRelPath ); + new FileInfo( newFullPath ).Directory!.Create(); + File.Move( Path.Combine( basePath, oldRelPath ), newFullPath ); + } + catch( Exception e ) + { + PluginLog.Error( $"Could not move file from {oldRelPath} to {newRelPath}:\n{e}" ); + return false; + } + + foreach( var option in meta.Groups.Values.SelectMany( group => group.Options ) ) + { + if( option.OptionFiles.TryGetValue( oldRelPath, out var gamePaths ) ) + { + option.OptionFiles.Add( newRelPath, gamePaths ); + option.OptionFiles.Remove( oldRelPath ); + } + } + + return true; + } + + + private static void RemoveUselessGroups( ModMeta meta ) + { + meta.Groups = meta.Groups.Where( kvp => kvp.Value.Options.Any( o => o.OptionFiles.Count > 0 ) ).ToDictionary( kvp => kvp.Key, kvp => kvp.Value ); + } + + // Goes through all Single-Select options and checks if file links are in each of them. + // If they are, it moves those files to the root folder and removes them from the groups (and puts them to duplicates, if necessary). + public static void Normalize( DirectoryInfo baseDir, ModMeta meta ) + { + foreach( var group in meta.Groups.Values.Where( g => g.SelectionType == SelectType.Single && g.GroupName != Duplicates ) ) + { + var firstOption = true; + HashSet< (RelPath, GamePath) > groupList = new(); + foreach( var option in group.Options ) + { + HashSet< (RelPath, GamePath) > optionList = new(); + foreach( var (file, gamePaths) in option.OptionFiles.Select( p => ( p.Key, p.Value ) ) ) + { + optionList.UnionWith( gamePaths.Select( p => ( file, p ) ) ); + } + + if( firstOption ) + { + groupList = optionList; + } + else + { + groupList.IntersectWith( optionList ); + } + + firstOption = false; + } + + var newPath = new Dictionary< RelPath, GamePath >(); + foreach( var (path, gamePath) in groupList ) + { + var relPath = new RelPath( gamePath ); + if( newPath.TryGetValue( path, out var usedGamePath ) ) + { + var required = FindOrCreateDuplicates( meta ); + var usedRelPath = new RelPath( usedGamePath ); + required.AddFile( usedRelPath, gamePath ); + required.AddFile( usedRelPath, usedGamePath ); + RemoveFromGroups( meta, relPath, gamePath, GroupType.Single, true ); + } + else if( MoveFile( meta, baseDir.FullName, path, relPath ) ) + { + newPath[ path ] = gamePath; + if( FileIsInAnyGroup( meta, relPath ) ) + { + FindOrCreateDuplicates( meta ).AddFile( relPath, gamePath ); + } + + RemoveFromGroups( meta, relPath, gamePath, GroupType.Single, true ); + } + } + } + + RemoveUselessGroups( meta ); + ClearEmptySubDirectories( baseDir ); + } + } +} \ No newline at end of file diff --git a/Penumbra/UI/MenuTabs/TabInstalled/TabInstalledModPanel.cs b/Penumbra/UI/MenuTabs/TabInstalled/TabInstalledModPanel.cs index f1eeecdc..58c7fa3c 100644 --- a/Penumbra/UI/MenuTabs/TabInstalled/TabInstalledModPanel.cs +++ b/Penumbra/UI/MenuTabs/TabInstalled/TabInstalledModPanel.cs @@ -26,6 +26,7 @@ namespace Penumbra.UI private const string ButtonEditJson = "Edit JSON"; private const string ButtonReloadJson = "Reload JSON"; private const string ButtonDeduplicate = "Deduplicate"; + private const string ButtonNormalize = "Normalize"; private const string TooltipOpenModFolder = "Open the directory containing this mod in your default file explorer."; private const string TooltipRenameModFolder = "Rename the directory containing this mod without opening another application."; private const string TooltipEditJson = "Open the JSON configuration file in your default application for .json."; @@ -34,7 +35,10 @@ namespace Penumbra.UI private const string TooltipDeduplicate = "Try to find identical files and remove duplicate occurences to reduce the mods disk size.\n" - + "Introduces an invisible single-option Group \"Duplicates\"."; + + "Introduces an invisible single-option Group \"Duplicates\".\nExperimental - use at own risk!"; + + private const string TooltipNormalize = + "Try to reduce unnecessary options or subdirectories to default options if possible.\nExperimental - use at own risk!"; private const float HeaderLineDistance = 10f; private static readonly Vector4 GreyColor = new( 1f, 1f, 1f, 0.66f ); @@ -389,7 +393,7 @@ namespace Penumbra.UI { if( ImGui.Button( ButtonDeduplicate ) ) { - new Deduplicator( Mod!.Mod.ModBasePath, Meta! ).Run(); + ModCleanup.Deduplicate( Mod!.Mod.ModBasePath, Meta! ); _selector.SaveCurrentMod(); Mod.Mod.RefreshModFiles(); Service< ModManager >.Get().CalculateEffectiveFileList(); @@ -401,6 +405,22 @@ namespace Penumbra.UI } } + private void DrawNormalizeButton() + { + if( ImGui.Button( ButtonNormalize ) ) + { + ModCleanup.Normalize( Mod!.Mod.ModBasePath, Meta! ); + _selector.SaveCurrentMod(); + Mod.Mod.RefreshModFiles(); + Service< ModManager >.Get().CalculateEffectiveFileList(); + } + + if( ImGui.IsItemHovered() ) + { + ImGui.SetTooltip( TooltipNormalize ); + } + } + private void DrawEditLine() { DrawOpenModFolderButton(); @@ -412,6 +432,8 @@ namespace Penumbra.UI DrawReloadJsonButton(); ImGui.SameLine(); DrawDeduplicateButton(); + ImGui.SameLine(); + DrawNormalizeButton(); } public void Draw()