Some Deduplicator fixes and introduce Normalize function, untested.

Penumbra/Models/Deduplicator.cs

@@ -1,168 +0,0 @@
-using System.Collections;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Security.Cryptography;
-using Dalamud.Plugin;
-using Penumbra.Util;
-
-namespace Penumbra.Models
-{
-    public class Deduplicator
-    {
-        private readonly DirectoryInfo _baseDir;
-        private readonly ModMeta       _mod;
-        private          SHA256?       _hasher;
-
-        private readonly Dictionary< long, List< FileInfo > > _filesBySize = new();
-
-        private SHA256 Sha()
-        {
-            _hasher ??= SHA256.Create();
-            return _hasher;
-        }
-
-        public Deduplicator( DirectoryInfo baseDir, ModMeta mod )
-        {
-            _baseDir = baseDir;
-            _mod     = mod;
-            BuildDict();
-        }
-
-        private void BuildDict()
-        {
-            foreach( var file in _baseDir.EnumerateFiles( "*.*", SearchOption.AllDirectories ) )
-            {
-                var fileLength = file.Length;
-                if( _filesBySize.TryGetValue( fileLength, out var files ) )
-                {
-                    files.Add( file );
-                }
-                else
-                {
-                    _filesBySize[ fileLength ] = new List< FileInfo >() { file };
-                }
-            }
-        }
-
-        public void Run()
-        {
-            foreach( var pair in _filesBySize.Where( pair => pair.Value.Count >= 2 ) )
-            {
-                if( pair.Value.Count == 2 )
-                {
-                    if( CompareFilesDirectly( pair.Value[ 0 ], pair.Value[ 1 ] ) )
-                    {
-                        ReplaceFile( pair.Value[ 0 ], pair.Value[ 1 ] );
-                    }
-                }
-                else
-                {
-                    var deleted = Enumerable.Repeat( false, pair.Value.Count ).ToArray();
-                    var hashes  = pair.Value.Select( ComputeHash ).ToArray();
-
-                    for( var i = 0; i < pair.Value.Count; ++i )
-                    {
-                        if( deleted[ i ] )
-                        {
-                            continue;
-                        }
-
-                        for( var j = i + 1; j < pair.Value.Count; ++j )
-                        {
-                            if( deleted[ j ] || !CompareHashes( hashes[ i ], hashes[ j ] ) )
-                            {
-                                continue;
-                            }
-
-                            ReplaceFile( pair.Value[ i ], pair.Value[ j ] );
-                            deleted[ j ] = true;
-                        }
-                    }
-                }
-            }
-
-            ClearEmptySubDirectories( _baseDir );
-        }
-
-        private void ReplaceFile( FileInfo f1, FileInfo f2 )
-        {
-            RelPath relName1 = new( f1, _baseDir );
-            RelPath relName2 = new( f2, _baseDir );
-
-            var inOption = false;
-            foreach( var group in _mod.Groups.Select( g => g.Value.Options ) )
-            {
-                foreach( var option in group )
-                {
-                    if( option.OptionFiles.TryGetValue( relName2, out var values ) )
-                    {
-                        inOption = true;
-                        foreach( var value in values )
-                        {
-                            option.AddFile( relName1, value );
-                        }
-
-                        option.OptionFiles.Remove( relName2 );
-                    }
-                }
-            }
-
-            if( !inOption )
-            {
-                const string duplicates = "Duplicates";
-                if( !_mod.Groups.ContainsKey( duplicates ) )
-                {
-                    OptionGroup info = new()
-                    {
-                        GroupName     = duplicates,
-                        SelectionType = SelectType.Single,
-                        Options = new List< Option >()
-                        {
-                            new()
-                            {
-                                OptionName  = "Required",
-                                OptionDesc  = "",
-                                OptionFiles = new Dictionary< RelPath, HashSet< GamePath > >()
-                            }
-                        }
-                    };
-                    _mod.Groups.Add( duplicates, info );
-                }
-
-                _mod.Groups[ duplicates ].Options[ 0 ].AddFile( relName1, new GamePath( relName2 ) );
-                _mod.Groups[ duplicates ].Options[ 0 ].AddFile( relName1, new GamePath( relName1 ) );
-            }
-
-            PluginLog.Information( $"File {relName1} and {relName2} are identical. Deleting the second." );
-            f2.Delete();
-        }
-
-        public static bool CompareFilesDirectly( FileInfo f1, FileInfo f2 )
-            => File.ReadAllBytes( f1.FullName ).SequenceEqual( File.ReadAllBytes( f2.FullName ) );
-
-        public static bool CompareHashes( byte[] f1, byte[] f2 )
-            => StructuralComparisons.StructuralEqualityComparer.Equals( f1, f2 );
-
-        public byte[] ComputeHash( FileInfo f )
-        {
-            var stream = File.OpenRead( f.FullName );
-            var ret    = Sha().ComputeHash( stream );
-            stream.Dispose();
-            return ret;
-        }
-
-        // Does not delete the base directory itself even if it is completely empty at the end.
-        public static void ClearEmptySubDirectories( DirectoryInfo baseDir )
-        {
-            foreach( var subDir in baseDir.GetDirectories() )
-            {
-                ClearEmptySubDirectories( subDir );
-                if( subDir.GetFiles().Length == 0 && subDir.GetDirectories().Length == 0 )
-                {
-                    subDir.Delete();
-                }
-            }
-        }
-    }
-}

Penumbra/Models/ModCleanup.cs

@@ -0,0 +1,378 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using Dalamud.Plugin;
+using Penumbra.Util;
+
+namespace Penumbra.Models
+{
+    public class ModCleanup
+    {
+        private const string Duplicates = "Duplicates";
+        private const string Required   = "Required";
+
+
+        private readonly DirectoryInfo _baseDir;
+        private readonly ModMeta       _mod;
+        private          SHA256?       _hasher;
+
+        private readonly Dictionary< long, List< FileInfo > > _filesBySize = new();
+
+        private SHA256 Sha()
+        {
+            _hasher ??= SHA256.Create();
+            return _hasher;
+        }
+
+        private ModCleanup( DirectoryInfo baseDir, ModMeta mod )
+        {
+            _baseDir = baseDir;
+            _mod     = mod;
+            BuildDict();
+        }
+
+        private void BuildDict()
+        {
+            foreach( var file in _baseDir.EnumerateFiles( "*.*", SearchOption.AllDirectories ) )
+            {
+                var fileLength = file.Length;
+                if( _filesBySize.TryGetValue( fileLength, out var files ) )
+                {
+                    files.Add( file );
+                }
+                else
+                {
+                    _filesBySize[ fileLength ] = new List< FileInfo >() { file };
+                }
+            }
+        }
+
+        private static Option FindOrCreateDuplicates( ModMeta meta )
+        {
+            static Option RequiredOption()
+                => new()
+                {
+                    OptionName  = Required,
+                    OptionDesc  = "",
+                    OptionFiles = new Dictionary< RelPath, HashSet< GamePath > >(),
+                };
+
+            if( meta.Groups.TryGetValue( Duplicates, out var duplicates ) )
+            {
+                var idx = duplicates.Options.FindIndex( o => o.OptionName == Required );
+                if( idx >= 0 )
+                {
+                    return duplicates.Options[ idx ];
+                }
+
+                duplicates.Options.Add( RequiredOption() );
+                return duplicates.Options.Last();
+            }
+
+            meta.Groups.Add( Duplicates, new OptionGroup
+            {
+                GroupName     = Duplicates,
+                SelectionType = SelectType.Single,
+                Options       = new List< Option > { RequiredOption() },
+            } );
+
+            return meta.Groups[ Duplicates ].Options.First();
+        }
+
+        public static void Deduplicate( DirectoryInfo baseDir, ModMeta mod )
+        {
+            var dedup = new ModCleanup( baseDir, mod );
+            foreach( var pair in dedup._filesBySize.Where( pair => pair.Value.Count >= 2 ) )
+            {
+                if( pair.Value.Count == 2 )
+                {
+                    if( CompareFilesDirectly( pair.Value[ 0 ], pair.Value[ 1 ] ) )
+                    {
+                        dedup.ReplaceFile( pair.Value[ 0 ], pair.Value[ 1 ] );
+                    }
+                }
+                else
+                {
+                    var deleted = Enumerable.Repeat( false, pair.Value.Count ).ToArray();
+                    var hashes  = pair.Value.Select( dedup.ComputeHash ).ToArray();
+
+                    for( var i = 0; i < pair.Value.Count; ++i )
+                    {
+                        if( deleted[ i ] )
+                        {
+                            continue;
+                        }
+
+                        for( var j = i + 1; j < pair.Value.Count; ++j )
+                        {
+                            if( deleted[ j ] || !CompareHashes( hashes[ i ], hashes[ j ] ) )
+                            {
+                                continue;
+                            }
+
+                            dedup.ReplaceFile( pair.Value[ i ], pair.Value[ j ] );
+                            deleted[ j ] = true;
+                        }
+                    }
+                }
+            }
+
+            CleanUpDuplicates( mod );
+            ClearEmptySubDirectories( dedup._baseDir );
+        }
+
+        private void ReplaceFile( FileInfo f1, FileInfo f2 )
+        {
+            RelPath relName1 = new( f1, _baseDir );
+            RelPath relName2 = new( f2, _baseDir );
+
+            var inOption1 = false;
+            var inOption2 = false;
+            foreach( var option in _mod.Groups.SelectMany( g => g.Value.Options ) )
+            {
+                if( option.OptionFiles.ContainsKey( relName1 ) )
+                {
+                    inOption1 = true;
+                }
+
+                if( !option.OptionFiles.TryGetValue( relName2, out var values ) )
+                {
+                    continue;
+                }
+
+                inOption2 = true;
+
+                foreach( var value in values )
+                {
+                    option.AddFile( relName1, value );
+                }
+
+                option.OptionFiles.Remove( relName2 );
+            }
+
+            if( !inOption1 || !inOption2 )
+            {
+                var duplicates = FindOrCreateDuplicates( _mod );
+                if( !inOption1 )
+                {
+                    duplicates.AddFile( relName1, new GamePath( relName2, 0 ) );
+                }
+
+                if( !inOption2 )
+                {
+                    duplicates.AddFile( relName1, new GamePath( relName1, 0 ) );
+                }
+            }
+
+            PluginLog.Information( $"File {relName1} and {relName2} are identical. Deleting the second." );
+            f2.Delete();
+        }
+
+        public static bool CompareFilesDirectly( FileInfo f1, FileInfo f2 )
+            => File.ReadAllBytes( f1.FullName ).SequenceEqual( File.ReadAllBytes( f2.FullName ) );
+
+        public static bool CompareHashes( byte[] f1, byte[] f2 )
+            => StructuralComparisons.StructuralEqualityComparer.Equals( f1, f2 );
+
+        public byte[] ComputeHash( FileInfo f )
+        {
+            var stream = File.OpenRead( f.FullName );
+            var ret    = Sha().ComputeHash( stream );
+            stream.Dispose();
+            return ret;
+        }
+
+        // Does not delete the base directory itself even if it is completely empty at the end.
+        public static void ClearEmptySubDirectories( DirectoryInfo baseDir )
+        {
+            foreach( var subDir in baseDir.GetDirectories() )
+            {
+                ClearEmptySubDirectories( subDir );
+                if( subDir.GetFiles().Length == 0 && subDir.GetDirectories().Length == 0 )
+                {
+                    subDir.Delete();
+                }
+            }
+        }
+
+        private static bool FileIsInAnyGroup( ModMeta meta, RelPath relPath, bool exceptDuplicates = false )
+        {
+            var groupEnumerator = exceptDuplicates
+                ? meta.Groups.Values.Where( G => G.GroupName != Duplicates )
+                : meta.Groups.Values;
+            return groupEnumerator.SelectMany( group => group.Options )
+               .Any( option => option.OptionFiles.ContainsKey( relPath ) );
+        }
+
+        private static void CleanUpDuplicates( ModMeta meta )
+        {
+            if( !meta.Groups.TryGetValue( Duplicates, out var info ) )
+            {
+                return;
+            }
+
+            var requiredIdx = info.Options.FindIndex( o => o.OptionName == Required );
+            if( requiredIdx >= 0 )
+            {
+                var required = info.Options[ requiredIdx ];
+                foreach( var kvp in required.OptionFiles.ToArray() )
+                {
+                    if( kvp.Value.Count > 1 || FileIsInAnyGroup( meta, kvp.Key, true ) )
+                    {
+                        continue;
+                    }
+
+                    if( kvp.Value.Count == 0 || kvp.Value.First().CompareTo( new GamePath( kvp.Key, 0 ) ) == 0 )
+                    {
+                        required.OptionFiles.Remove( kvp.Key );
+                    }
+                }
+
+                if( required.OptionFiles.Count == 0 )
+                {
+                    info.Options.RemoveAt( requiredIdx );
+                }
+            }
+
+            if( info.Options.Count == 0 )
+            {
+                meta.Groups.Remove( Duplicates );
+            }
+        }
+
+        public enum GroupType
+        {
+            Both   = 0,
+            Single = 1,
+            Multi  = 2,
+        };
+
+        private static void RemoveFromGroups( ModMeta meta, RelPath relPath, GamePath gamePath, GroupType type = GroupType.Both,
+            bool                                      skipDuplicates = true )
+        {
+            if( meta.Groups.Count == 0 )
+            {
+                return;
+            }
+
+            var enumerator = type switch
+            {
+                GroupType.Both   => meta.Groups.Values,
+                GroupType.Single => meta.Groups.Values.Where( g => g.SelectionType == SelectType.Single ),
+                GroupType.Multi  => meta.Groups.Values.Where( g => g.SelectionType == SelectType.Multi ),
+                _                => throw new InvalidEnumArgumentException( "Invalid Enum in RemoveFromGroups" ),
+            };
+            foreach( var group in enumerator )
+            {
+                var optionEnum = skipDuplicates
+                    ? group.Options.Where( o => group.GroupName != Duplicates || o.OptionName != Required )
+                    : group.Options;
+                foreach( var option in optionEnum )
+                {
+                    if( option.OptionFiles.TryGetValue( relPath, out var gamePaths ) && gamePaths.Remove( gamePath ) && gamePaths.Count == 0 )
+                    {
+                        option.OptionFiles.Remove( relPath );
+                    }
+                }
+            }
+        }
+
+        public static bool MoveFile( ModMeta meta, string basePath, RelPath oldRelPath, RelPath newRelPath )
+        {
+            if( oldRelPath == newRelPath )
+            {
+                return true;
+            }
+
+            try
+            {
+                var newFullPath = Path.Combine( basePath, newRelPath );
+                new FileInfo( newFullPath ).Directory!.Create();
+                File.Move( Path.Combine( basePath, oldRelPath ), newFullPath );
+            }
+            catch( Exception e )
+            {
+                PluginLog.Error( $"Could not move file from {oldRelPath} to {newRelPath}:\n{e}" );
+                return false;
+            }
+
+            foreach( var option in meta.Groups.Values.SelectMany( group => group.Options ) )
+            {
+                if( option.OptionFiles.TryGetValue( oldRelPath, out var gamePaths ) )
+                {
+                    option.OptionFiles.Add( newRelPath, gamePaths );
+                    option.OptionFiles.Remove( oldRelPath );
+                }
+            }
+
+            return true;
+        }
+
+
+        private static void RemoveUselessGroups( ModMeta meta )
+        {
+            meta.Groups = meta.Groups.Where( kvp => kvp.Value.Options.Any( o => o.OptionFiles.Count > 0  ) ).ToDictionary( kvp => kvp.Key, kvp => kvp.Value );
+        }
+
+        // Goes through all Single-Select options and checks if file links are in each of them.
+        // If they are, it moves those files to the root folder and removes them from the groups (and puts them to duplicates, if necessary).
+        public static void Normalize( DirectoryInfo baseDir, ModMeta meta )
+        {
+            foreach( var group in meta.Groups.Values.Where( g => g.SelectionType == SelectType.Single && g.GroupName != Duplicates ) )
+            {
+                var                            firstOption = true;
+                HashSet< (RelPath, GamePath) > groupList   = new();
+                foreach( var option in group.Options )
+                {
+                    HashSet< (RelPath, GamePath) > optionList = new();
+                    foreach( var (file, gamePaths) in option.OptionFiles.Select( p => ( p.Key, p.Value ) ) )
+                    {
+                        optionList.UnionWith( gamePaths.Select( p => ( file, p ) ) );
+                    }
+
+                    if( firstOption )
+                    {
+                        groupList = optionList;
+                    }
+                    else
+                    {
+                        groupList.IntersectWith( optionList );
+                    }
+
+                    firstOption = false;
+                }
+
+                var newPath = new Dictionary< RelPath, GamePath >();
+                foreach( var (path, gamePath) in groupList )
+                {
+                    var relPath = new RelPath( gamePath );
+                    if( newPath.TryGetValue( path, out var usedGamePath ) )
+                    {
+                        var required    = FindOrCreateDuplicates( meta );
+                        var usedRelPath = new RelPath( usedGamePath );
+                        required.AddFile( usedRelPath, gamePath );
+                        required.AddFile( usedRelPath, usedGamePath );
+                        RemoveFromGroups( meta, relPath, gamePath, GroupType.Single, true );
+                    }
+                    else if( MoveFile( meta, baseDir.FullName, path, relPath ) )
+                    {
+                        newPath[ path ] = gamePath;
+                        if( FileIsInAnyGroup( meta, relPath ) )
+                        {
+                            FindOrCreateDuplicates( meta ).AddFile( relPath, gamePath );
+                        }
+
+                        RemoveFromGroups( meta, relPath, gamePath, GroupType.Single, true );
+                    }
+                }
+            }
+
+            RemoveUselessGroups( meta );
+            ClearEmptySubDirectories( baseDir );
+        }
+    }
+}

Penumbra/UI/MenuTabs/TabInstalled/TabInstalledModPanel.cs

@@ -26,6 +26,7 @@ namespace Penumbra.UI
             private const string ButtonEditJson         = "Edit JSON";
             private const string ButtonReloadJson       = "Reload JSON";
             private const string ButtonDeduplicate      = "Deduplicate";
+            private const string ButtonNormalize        = "Normalize";
             private const string TooltipOpenModFolder   = "Open the directory containing this mod in your default file explorer.";
             private const string TooltipRenameModFolder = "Rename the directory containing this mod without opening another application.";
             private const string TooltipEditJson        = "Open the JSON configuration file in your default application for .json.";
@@ -34,7 +35,10 @@ namespace Penumbra.UI
 
             private const string TooltipDeduplicate =
                 "Try to find identical files and remove duplicate occurences to reduce the mods disk size.\n"
-              + "Introduces an invisible single-option Group \"Duplicates\".";
+              + "Introduces an invisible single-option Group \"Duplicates\".\nExperimental - use at own risk!";
+
+            private const string TooltipNormalize =
+                "Try to reduce unnecessary options or subdirectories to default options if possible.\nExperimental - use at own risk!";
 
             private const           float   HeaderLineDistance = 10f;
             private static readonly Vector4 GreyColor          = new( 1f, 1f, 1f, 0.66f );
@@ -389,7 +393,7 @@ namespace Penumbra.UI
             {
                 if( ImGui.Button( ButtonDeduplicate ) )
                 {
-                    new Deduplicator( Mod!.Mod.ModBasePath, Meta! ).Run();
+                    ModCleanup.Deduplicate( Mod!.Mod.ModBasePath, Meta! );
                     _selector.SaveCurrentMod();
                     Mod.Mod.RefreshModFiles();
                     Service< ModManager >.Get().CalculateEffectiveFileList();
@@ -401,6 +405,22 @@ namespace Penumbra.UI
                 }
             }
 
+            private void DrawNormalizeButton()
+            {
+                if( ImGui.Button( ButtonNormalize ) )
+                {
+                    ModCleanup.Normalize( Mod!.Mod.ModBasePath, Meta! );
+                    _selector.SaveCurrentMod();
+                    Mod.Mod.RefreshModFiles();
+                    Service< ModManager >.Get().CalculateEffectiveFileList();
+                }
+
+                if( ImGui.IsItemHovered() )
+                {
+                    ImGui.SetTooltip( TooltipNormalize );
+                }
+            }
+
             private void DrawEditLine()
             {
                 DrawOpenModFolderButton();
@@ -412,6 +432,8 @@ namespace Penumbra.UI
                 DrawReloadJsonButton();
                 ImGui.SameLine();
                 DrawDeduplicateButton();
+                ImGui.SameLine();
+                DrawNormalizeButton();
             }
 
             public void Draw()