Add option to auto-deduplicate on import.

Penumbra/Configuration.cs

@@ -48,6 +48,7 @@ public partial class Configuration : IPluginConfiguration
 
     public bool FixMainWindow { get; set; } = false;
     public bool ShowAdvanced { get; set; }
+    public bool AutoDeduplicateOnImport { get; set; } = false;
     public bool DisableSoundStreaming { get; set; } = true;
     public bool EnableHttpApi { get; set; }
 

Penumbra/Import/ImporterState.cs

@@ -5,5 +5,6 @@ public enum ImporterState
     None,
     WritingPackToDisk,
     ExtractingModFiles,
+    DeduplicatingFiles,
     Done,
 }

Penumbra/Import/TexToolsImport.cs

@@ -1,14 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Dalamud.Logging;
 using ICSharpCode.SharpZipLib.Zip;
 using Newtonsoft.Json;
-using Penumbra.Util;
+using Penumbra.Mods;
 using FileMode = System.IO.FileMode;
 
 namespace Penumbra.Import;
@@ -95,6 +94,11 @@ public partial class TexToolsImporter : IDisposable
             {
                 var directory = VerifyVersionAndImport( file );
                 ExtractedMods.Add( ( file, directory, null ) );
+                if( Penumbra.Config.AutoDeduplicateOnImport )
+                {
+                    State = ImporterState.DeduplicatingFiles;
+                    Mod.Editor.DeduplicateMod( directory );
+                }
             }
             catch( Exception e )
             {

Penumbra/Import/TexToolsImporter.Gui.cs

@@ -38,7 +38,14 @@ public partial class TexToolsImporter
             var percentage = _modPackCount / ( float )_currentModPackIdx;
             ImGui.ProgressBar( percentage, size, $"Mod {_currentModPackIdx + 1} / {_modPackCount}" );
             ImGui.NewLine();
-            ImGui.TextUnformatted( $"Extracting {_currentModName}..." );
+            if( State == ImporterState.DeduplicatingFiles )
+            {
+                ImGui.TextUnformatted( $"Deduplicating {_currentModName}..." );
+            }
+            else
+            {
+                ImGui.TextUnformatted( $"Extracting {_currentModName}..." );
+            }
 
             if( _currentNumOptions > 1 )
             {
@@ -47,8 +54,11 @@ public partial class TexToolsImporter
                 percentage = _currentNumOptions == 0 ? 1f : _currentOptionIdx / ( float )_currentNumOptions;
                 ImGui.ProgressBar( percentage, size, $"Option {_currentOptionIdx + 1} / {_currentNumOptions}" );
                 ImGui.NewLine();
-                ImGui.TextUnformatted(
+                if( State != ImporterState.DeduplicatingFiles )
-                    $"Extracting option {( _currentGroupName.Length == 0 ? string.Empty : $"{_currentGroupName} - " )}{_currentOptionName}..." );
+                {
+                    ImGui.TextUnformatted(
+                        $"Extracting option {( _currentGroupName.Length == 0 ? string.Empty : $"{_currentGroupName} - " )}{_currentOptionName}..." );
+                }
             }
 
             ImGui.NewLine();
@@ -56,7 +66,10 @@ public partial class TexToolsImporter
             percentage = _currentNumFiles == 0 ? 1f : _currentFileIdx / ( float )_currentNumFiles;
             ImGui.ProgressBar( percentage, size, $"File {_currentFileIdx + 1} / {_currentNumFiles}" );
             ImGui.NewLine();
-            ImGui.TextUnformatted( $"Extracting file {_currentFileName}..." );
+            if( State != ImporterState.DeduplicatingFiles )
+            {
+                ImGui.TextUnformatted( $"Extracting file {_currentFileName}..." );
+            }
         }
     }
 

Penumbra/Mods/Editor/Mod.Editor.Duplicates.cs

@@ -24,7 +24,7 @@ public partial class Mod
 
         public bool DuplicatesFinished { get; private set; } = true;
 
-        public void DeleteDuplicates()
+        public void DeleteDuplicates( bool useModManager = true )
         {
             if( !DuplicatesFinished || _duplicates.Count == 0 )
             {
@@ -41,15 +41,16 @@ public partial class Mod
                 var remaining = set[ 0 ];
                 foreach( var duplicate in set.Skip( 1 ) )
                 {
-                    HandleDuplicate( duplicate, remaining );
+                    HandleDuplicate( duplicate, remaining, useModManager );
                 }
             }
 
             _availableFiles.RemoveAll( p => !p.File.Exists );
             _duplicates.Clear();
+            DeleteEmptyDirectories( _mod.ModPath );
         }
 
-        private void HandleDuplicate( FullPath duplicate, FullPath remaining )
+        private void HandleDuplicate( FullPath duplicate, FullPath remaining, bool useModManager )
         {
             void HandleSubMod( ISubMod subMod, int groupIdx, int optionIdx )
             {
@@ -58,7 +59,23 @@ public partial class Mod
                     kvp => ChangeDuplicatePath( kvp.Value, duplicate, remaining, kvp.Key, ref changes ) );
                 if( changes )
                 {
-                    Penumbra.ModManager.OptionSetFiles( _mod, groupIdx, optionIdx, dict );
+                    if( useModManager )
+                    {
+                        Penumbra.ModManager.OptionSetFiles( _mod, groupIdx, optionIdx, dict );
+                    }
+                    else
+                    {
+                        var sub = ( SubMod )subMod;
+                        sub.FileData = dict;
+                        if( groupIdx == -1 )
+                        {
+                            _mod.SaveDefaultMod();
+                        }
+                        else
+                        {
+                            IModGroup.Save( _mod.Groups[ groupIdx ], _mod.ModPath, groupIdx );
+                        }
+                    }
                 }
             }
 
@@ -94,7 +111,7 @@ public partial class Mod
             {
                 DuplicatesFinished = false;
                 UpdateFiles();
-                var files = _availableFiles.OrderByDescending(f => f.FileSize).ToArray();
+                var files = _availableFiles.OrderByDescending( f => f.FileSize ).ToArray();
                 Task.Run( () => CheckDuplicates( files ) );
             }
         }
@@ -215,5 +232,53 @@ public partial class Mod
             using var stream = File.OpenRead( f.FullName );
             return _hasher.ComputeHash( stream );
         }
+
+        // Recursively delete all empty directories starting from the given directory.
+        // Deletes inner directories first, so that a tree of empty directories is actually deleted.
+        private void DeleteEmptyDirectories( DirectoryInfo baseDir )
+        {
+            try
+            {
+                if( !baseDir.Exists )
+                {
+                    return;
+                }
+
+                foreach( var dir in baseDir.EnumerateDirectories( "*", SearchOption.TopDirectoryOnly ) )
+                {
+                    DeleteEmptyDirectories( dir );
+                }
+
+                baseDir.Refresh();
+                if( !baseDir.EnumerateFileSystemInfos().Any() )
+                {
+                    Directory.Delete( baseDir.FullName, false );
+                }
+            }
+            catch( Exception e )
+            {
+                PluginLog.Error( $"Could not delete empty directories in {baseDir.FullName}:\n{e}" );
+            }
+        }
+
+
+
+        // Deduplicate a mod simply by its directory without any confirmation or waiting time.
+        internal static void DeduplicateMod( DirectoryInfo modDirectory )
+        {
+            try
+            {
+                var mod = new Mod( modDirectory );
+                mod.Reload( out _ );
+                var editor = new Editor( mod, 0, 0 );
+                editor.DuplicatesFinished = false;
+                editor.CheckDuplicates( editor.AvailableFiles.OrderByDescending( f => f.FileSize ).ToArray() );
+                editor.DeleteDuplicates( false );
+            }
+            catch( Exception e )
+            {
+                PluginLog.Warning( $"Could not deduplicate mod {modDirectory.Name}:\n{e}" );
+            }
+        }
     }
 }

Penumbra/UI/ConfigWindow.SettingsTab.Advanced.cs

@@ -20,6 +20,9 @@ public partial class ConfigWindow
                 return;
             }
 
+            Checkbox( "Auto Deduplicate on Import",
+                "Automatically deduplicate mod files on import. This will make mod file sizes smaller, but deletes (binary identical) files.",
+                Penumbra.Config.AutoDeduplicateOnImport, v => Penumbra.Config.AutoDeduplicateOnImport = v );
             DrawRequestedResourceLogging();
             DrawDisableSoundStreamingBox();
             DrawEnableHttpApiBox();