From 06b0fb7e0c75b174f18491aa1760a751970e182d Mon Sep 17 00:00:00 2001
From: Ottermandias <nickroederspam@gmail.com>
Date: Mon, 18 Jan 2021 14:53:28 +0100
Subject: [PATCH] Added Deduplication button and the ability to point a hdd
 file to multiple game paths in groups.

---
 Penumbra/Importer/TexToolsImport.cs     |  14 +-
 Penumbra/Models/Deduplicator.cs         | 162 ++++++++++++++++++++++++
 Penumbra/Models/GroupInformation.cs     |  20 ++-
 Penumbra/Models/ModMeta.cs              |   8 +-
 Penumbra/Mods/ModCollection.cs          |   4 +-
 Penumbra/Mods/ModManager.cs             | 137 ++++++++++----------
 Penumbra/UI/SettingsInterface.cs        |  61 +++++----
 Penumbra/Util/SingleOrArrayConverter.cs |  55 ++++++++
 8 files changed, 358 insertions(+), 103 deletions(-)
 create mode 100644 Penumbra/Models/Deduplicator.cs
 create mode 100644 Penumbra/Util/SingleOrArrayConverter.cs

diff --git a/Penumbra/Importer/TexToolsImport.cs b/Penumbra/Importer/TexToolsImport.cs
index 9ec04a57..066ad305 100644
--- a/Penumbra/Importer/TexToolsImport.cs
+++ b/Penumbra/Importer/TexToolsImport.cs
@@ -237,18 +237,18 @@ namespace Penumbra.Importer
                 GroupName = group.GroupName,
                 Options = new List<Option>(),
             };
-            foreach( var opt in group.OptionList )
+            foreach( var opt in group.OptionList ) 
             {
                 var optio = new Option
                 {
                     OptionName = opt.Name,
-                    OptionDesc = String.IsNullOrEmpty( opt.Description ) ? "" : opt.Description,
-                    OptionFiles = new Dictionary<string, string>()
+                    OptionDesc = String.IsNullOrEmpty(opt.Description) ? "" : opt.Description,
+                    OptionFiles = new Dictionary<string, HashSet<string>>()
                 };
-                var optDir = new DirectoryInfo( Path.Combine( groupFolder.FullName, opt.Name ) );
-                foreach( var file in optDir.EnumerateFiles( "*.*", SearchOption.AllDirectories ) )
+                var optDir = new DirectoryInfo(Path.Combine( groupFolder.FullName, opt.Name));
+                foreach ( var file in optDir.EnumerateFiles("*.*", SearchOption.AllDirectories) ) 
                 {
-                    optio.OptionFiles[file.FullName.Substring( baseFolder.FullName.Length ).TrimStart( '\\' )] = file.FullName.Substring( optDir.FullName.Length ).TrimStart( '\\' ).Replace( '\\', '/' );
+                    optio.AddFile(file.FullName.Substring(baseFolder.FullName.Length).TrimStart('\\'), file.FullName.Substring(optDir.FullName.Length).TrimStart('\\').Replace('\\','/'));
                 }
                 Inf.Options.Add( optio );
             }
@@ -333,4 +333,4 @@ namespace Penumbra.Importer
             return encoding.GetString( ms.ToArray() );
         }
     }
-}
\ No newline at end of file
+}
diff --git a/Penumbra/Models/Deduplicator.cs b/Penumbra/Models/Deduplicator.cs
new file mode 100644
index 00000000..cfebd60b
--- /dev/null
+++ b/Penumbra/Models/Deduplicator.cs
@@ -0,0 +1,162 @@
+using System.Collections.Generic;
+using System.Security.Cryptography;
+using System.IO;
+using System.Linq;
+using System.Collections;
+using Dalamud.Plugin;
+
+namespace Penumbra.Models
+{
+    public class Deduplicator
+    {
+        private DirectoryInfo baseDir;
+        private int           baseDirLength;
+        private ModMeta       mod;
+        private SHA256        hasher = null;
+
+        private Dictionary<long, List<FileInfo>> filesBySize;
+
+        private ref SHA256 Sha()
+        {
+            if (hasher == null)
+                hasher = SHA256.Create();
+            return ref hasher;
+        }
+
+        public Deduplicator(DirectoryInfo baseDir, ModMeta mod)
+        {
+            this.baseDir       = baseDir;
+            this.baseDirLength = baseDir.FullName.Length;
+            this.mod           = mod;
+            filesBySize        = new();
+
+            BuildDict();  
+        }
+
+        private void BuildDict()
+        {
+            foreach( var file in baseDir.EnumerateFiles( "*.*", SearchOption.AllDirectories ) )
+            {
+                var fileLength = file.Length;
+                if (filesBySize.TryGetValue(fileLength, out var files))
+                    files.Add(file);
+                else
+                    filesBySize[fileLength] = new(){ file };
+            }
+        }
+
+        public void Run()
+        {
+            foreach (var pair in filesBySize)
+            {
+                if (pair.Value.Count < 2)
+                    continue;
+
+                if (pair.Value.Count == 2)
+                {
+                    if (CompareFilesDirectly(pair.Value[0], pair.Value[1]))
+                        ReplaceFile(pair.Value[0], pair.Value[1]);
+                }
+                else
+                {
+                    var deleted = Enumerable.Repeat(false, pair.Value.Count).ToArray();
+                    var hashes  = pair.Value.Select( F => ComputeHash(F)).ToArray();
+
+                    for (var i = 0; i < pair.Value.Count; ++i)
+                    {
+                        if (deleted[i])
+                            continue;
+
+                        for (var j = i + 1; j < pair.Value.Count; ++j)
+                        {
+                            if (deleted[j])
+                                continue;
+
+                            if (!CompareHashes(hashes[i], hashes[j]))
+                                continue;
+
+                            ReplaceFile(pair.Value[i], pair.Value[j]);
+                            deleted[j] = true;
+                        }
+                    }
+                }
+            }
+            ClearEmptySubDirectories(baseDir);
+        }
+
+        private void ReplaceFile(FileInfo f1, FileInfo f2)
+        {
+            var relName1 = f1.FullName.Substring(baseDirLength).TrimStart('\\');
+            var relName2 = f2.FullName.Substring(baseDirLength).TrimStart('\\');
+
+            var inOption = false;
+            foreach (var group in mod.Groups.Select( g => g.Value.Options))
+            {
+                foreach (var option in group)
+                {
+                    if (option.OptionFiles.TryGetValue(relName2, out var values))
+                    {
+                        inOption = true;
+                        foreach (var value in values)
+                            option.AddFile(relName1, value);
+                    }
+                }
+            }
+            if (!inOption)
+            {
+                const string duplicates = "Duplicates";
+                if (!mod.Groups.ContainsKey(duplicates))
+                {
+                    InstallerInfo info = new()
+                    {
+                        GroupName = duplicates,
+                        SelectionType = SelectType.Single,
+                        Options = new()
+                        { 
+                            new()
+                            {
+                                OptionName  = "Required", 
+                                OptionDesc  = "", 
+                                OptionFiles = new()
+                            }
+                        }
+                    };
+                    mod.Groups.Add(duplicates, info);
+                }
+                mod.Groups[duplicates].Options[0].AddFile(relName1, relName2.Replace('\\', '/'));
+                mod.Groups[duplicates].Options[0].AddFile(relName1, relName1.Replace('\\', '/'));
+            }
+            PluginLog.Information($"File {relName1} and {relName2} are identical. Deleting the second.");
+            f2.Delete();
+        }
+
+        public static bool CompareFilesDirectly(FileInfo f1, FileInfo f2)
+        {
+            return File.ReadAllBytes(f1.FullName).SequenceEqual(File.ReadAllBytes(f2.FullName));
+        }
+
+        public static bool CompareHashes(byte[] f1, byte[] f2)
+        {
+            return StructuralComparisons.StructuralEqualityComparer.Equals(f1, f2);
+        }
+
+        public byte[] ComputeHash(FileInfo f)
+        {
+            var stream = File.OpenRead( f.FullName );
+            var ret = Sha().ComputeHash(stream);
+            stream.Dispose();
+            return ret;
+        }
+
+        // Does not delete the base directory itself even if it is completely empty at the end.
+        public static void ClearEmptySubDirectories(DirectoryInfo baseDir)
+        {
+            foreach (var subDir in baseDir.GetDirectories())
+            {
+                ClearEmptySubDirectories(subDir);
+                if (subDir.GetFiles().Length == 0 && subDir.GetDirectories().Length == 0)
+                    subDir.Delete();
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/Penumbra/Models/GroupInformation.cs b/Penumbra/Models/GroupInformation.cs
index f7eb19e3..982dc72d 100644
--- a/Penumbra/Models/GroupInformation.cs
+++ b/Penumbra/Models/GroupInformation.cs
@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using Newtonsoft.Json;
 
 namespace Penumbra.Models
 {
@@ -10,12 +11,23 @@ namespace Penumbra.Models
     {
         public string OptionName;
         public string OptionDesc;
-        public Dictionary<string, string> OptionFiles;
+
+        [JsonProperty(ItemConverterType = typeof(SingleOrArrayConverter<string>))]
+        public Dictionary<string, HashSet<string>> OptionFiles;
+
+        public bool AddFile(string filePath, string gamePath)
+        {
+            if (OptionFiles.TryGetValue(filePath, out var set))
+                return set.Add(gamePath);
+            else
+                OptionFiles[filePath] = new(){ gamePath };
+            return true;
+        }
     }
-    public struct InstallerInfo
-    {
+
+    public struct InstallerInfo {
         public string GroupName;
         public SelectType SelectionType;
         public List<Option> Options;
     }
-}
\ No newline at end of file
+}
diff --git a/Penumbra/Models/ModMeta.cs b/Penumbra/Models/ModMeta.cs
index 6128b241..c027c467 100644
--- a/Penumbra/Models/ModMeta.cs
+++ b/Penumbra/Models/ModMeta.cs
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
+using Newtonsoft.Json;
 
 namespace Penumbra.Models
 {
@@ -17,6 +18,9 @@ namespace Penumbra.Models
 
         public Dictionary< string, string > FileSwaps { get; } = new();
 
-        public Dictionary<string, InstallerInfo> Groups { get; set; } = new();
+        public Dictionary<string, InstallerInfo> Groups { get; set; } = new();
+
+        [JsonIgnore]
+        public bool HasGroupWithConfig { get; set; } = false;
     }
 }
\ No newline at end of file
diff --git a/Penumbra/Mods/ModCollection.cs b/Penumbra/Mods/ModCollection.cs
index 4c6eea78..f3e828a9 100644
--- a/Penumbra/Mods/ModCollection.cs
+++ b/Penumbra/Mods/ModCollection.cs
@@ -69,6 +69,8 @@ namespace Penumbra.Mods
                 try
                 {
                     meta = JsonConvert.DeserializeObject< ModMeta >( File.ReadAllText( metaFile.FullName ) );
+                    meta.HasGroupWithConfig = meta.Groups != null && meta.Groups.Count > 0 
+                        && meta.Groups.Values.Any( G => G.SelectionType == SelectType.Multi || G.Options.Count > 1);
                 }
                 catch( Exception e )
                 {
@@ -218,4 +220,4 @@ namespace Penumbra.Mods
                 .Select( x => (x.Mod, x) );
         }
     }
-}
\ No newline at end of file
+}
diff --git a/Penumbra/Mods/ModManager.cs b/Penumbra/Mods/ModManager.cs
index c63d338f..9b2013be 100644
--- a/Penumbra/Mods/ModManager.cs
+++ b/Penumbra/Mods/ModManager.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using Penumbra.Models;
 
 namespace Penumbra.Mods
@@ -102,21 +103,7 @@ namespace Penumbra.Mods
             // Needed to reload body textures with mods
             //_plugin.GameUtils.ReloadPlayerResources();
         }
-        private (InstallerInfo, Option, string) GlobalPosition( string rel, Dictionary<string, InstallerInfo> gps )
-        {
-            string filePath = null;
-            foreach( var g in gps )
-            {
-                foreach( var opt in g.Value.Options )
-                {
-                    if( opt.OptionFiles.TryGetValue( rel, out filePath ) )
-                    {
-                        return (g.Value, opt, filePath);
-                    }
-                }
-            }
-            return (default( InstallerInfo ), default( Option ), null);
-        }
+
         public void CalculateEffectiveFileList()
         {
             ResolvedFiles.Clear();
@@ -131,65 +118,83 @@ namespace Penumbra.Mods
                 // fixup path
                 var baseDir = mod.ModBasePath.FullName;
 
+                if(settings.Conf == null) {
+                    settings.Conf = new();
+                    _plugin.ModManager.Mods.Save();
+                }
+
                 foreach( var file in mod.ModFiles )
                 {
                     var relativeFilePath = file.FullName.Substring( baseDir.Length ).TrimStart( '\\' );
+                    
+                    bool doNotAdd = false;
+                    void AddFiles(HashSet<string> gamePaths)
+                    {
+                        doNotAdd = true;
+                        foreach (var gamePath in gamePaths)
+                        {
+                            if( !ResolvedFiles.ContainsKey( gamePath ) )                            {
+                                ResolvedFiles[ gamePath.ToLowerInvariant() ] = file;
+                                registeredFiles[ gamePath ] = mod.Meta.Name;
+                            }
+                            else if( registeredFiles.TryGetValue( gamePath, out var modName ) )
+                            {
+                                mod.AddConflict( modName, gamePath );
+                            }
+                        }
+                    }
 
-                    string gamePath;
-                    bool addFile = true;
-                    var gps = mod.Meta.Groups;
-                    if( gps.Count >= 1 )
-                    {
-                        var negivtron = GlobalPosition( relativeFilePath, gps );
-                        if( negivtron.Item3 != null )
+                    HashSet<string> paths;
+                    foreach (var group in mod.Meta.Groups.Select( G => G.Value))
+                    {
+                        if (!settings.Conf.TryGetValue(group.GroupName, out var setting) 
+                            || (group.SelectionType == SelectType.Single && settings.Conf[group.GroupName] >= group.Options.Count))
                         {
-                            if( settings.Conf == null )
-                            {
-                                settings.Conf = new();
-                                _plugin.ModManager.Mods.Save();
-                            }
-                            if( !settings.Conf.ContainsKey( negivtron.Item1.GroupName ) )
-                            {
-                                settings.Conf[negivtron.Item1.GroupName] = 0;
-                                _plugin.ModManager.Mods.Save();
-                            }
-                            var current = settings.Conf[negivtron.Item1.GroupName];
-                            var flag = negivtron.Item1.Options.IndexOf( negivtron.Item2 );
-                            switch( negivtron.Item1.SelectionType )
-                            {
-                                case SelectType.Single:
-                                    {
-                                        addFile = current == flag;
-                                        break;
-                                    }
-                                case SelectType.Multi:
-                                    {
-                                        flag = 1 << negivtron.Item1.Options.IndexOf( negivtron.Item2 );
-                                        addFile = ( flag & current ) != 0;
-                                        break;
-                                    }
-                            }
-                            gamePath = negivtron.Item3;
-                        }
-                        else
+                            settings.Conf[group.GroupName] = 0;
+                            _plugin.ModManager.Mods.Save();
+                            setting = 0;
+                        }
+
+                        if (group.Options.Count == 0)
+                            continue;
+
+                        if (group.SelectionType == SelectType.Multi)
+                            settings.Conf[group.GroupName] &= ((1 << group.Options.Count) - 1);
+                        
+                        switch(group.SelectionType)
                         {
-                            gamePath = relativeFilePath.Replace( '\\', '/' );
-                        }
-                    }
-                    else
-                        gamePath = relativeFilePath.Replace( '\\', '/' );
-                    if( addFile )
-                    {
-                        if( !ResolvedFiles.ContainsKey( gamePath ) )                            {
-                            ResolvedFiles[ gamePath.ToLowerInvariant() ] = file;
-                            registeredFiles[ gamePath ] = mod.Meta.Name;
-                        }
-                        else if( registeredFiles.TryGetValue( gamePath, out var modName ) )
-                        {
-                            mod.AddConflict( modName, gamePath );
+                            case SelectType.Single:
+                                if (group.Options[setting].OptionFiles.TryGetValue(relativeFilePath, out paths))
+                                    AddFiles(paths);
+                                else
+                                {
+                                    for(var i = 0; i < group.Options.Count; ++i)
+                                    {
+                                        if (i == setting)
+                                            continue;
+                                        if(group.Options[i].OptionFiles.ContainsKey(relativeFilePath))
+                                        {
+                                            doNotAdd = true;
+                                            break;
+                                        }
+                                    }
+                                }
+                                break;
+                            case SelectType.Multi:
+                                for(var i = 0; i < group.Options.Count; ++i)
+                                {
+                                    if ((setting & (1 << i)) != 0)
+                                        if (group.Options[i].OptionFiles.TryGetValue(relativeFilePath, out paths))
+                                            AddFiles(paths);
+                                }
+                                break;
                         }
                     }
+                    if (!doNotAdd)
+                        AddFiles(new() { relativeFilePath.Replace( '\\', '/' ) });
                 }
+
+
                 foreach( var swap in mod.Meta.FileSwaps )
                 {
                     // just assume people put not fucked paths in here lol
@@ -253,4 +258,4 @@ namespace Penumbra.Mods
             // _fileSystemWatcher?.Dispose();
         }
     }
-}
\ No newline at end of file
+}
diff --git a/Penumbra/UI/SettingsInterface.cs b/Penumbra/UI/SettingsInterface.cs
index 484865b4..54bf55bb 100644
--- a/Penumbra/UI/SettingsInterface.cs
+++ b/Penumbra/UI/SettingsInterface.cs
@@ -1,4 +1,5 @@
 using System;
+using System.ComponentModel.Design;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
@@ -547,11 +548,7 @@ namespace Penumbra.UI
                 }
 
                 if( ImGui.IsItemHovered() )
-                {
-                    ImGui.BeginTooltip();
-                    ImGui.Text( _selectedMod.Mod.Meta.Website );
-                    ImGui.EndTooltip();
-                }
+                    ImGui.SetTooltip( _selectedMod.Mod.Meta.Website );
             }
             else
             {
@@ -568,7 +565,9 @@ namespace Penumbra.UI
             if( ImGui.Button( "Open Mod Folder" ) )
             {
                 Process.Start( _selectedMod.Mod.ModBasePath.FullName );
-            }
+            }
+            if( ImGui.IsItemHovered() )
+                ImGui.SetTooltip( "Open the directory containing this mod in your default file explorer." );
 
             ImGui.SameLine();
             if( ImGui.Button( "Edit JSON" ) )
@@ -576,24 +575,28 @@ namespace Penumbra.UI
                 var metaPath = Path.Combine( _selectedMod.Mod.ModBasePath.FullName, "meta.json" );
                 File.WriteAllText( metaPath, JsonConvert.SerializeObject( _selectedMod.Mod.Meta, Formatting.Indented ) );
                 Process.Start( metaPath );
-            }
+            }
+            if( ImGui.IsItemHovered() )
+                ImGui.SetTooltip( "Open the JSON configuration file in your default application for .json." );
 
             ImGui.SameLine();
             if( ImGui.Button( "Reload JSON" ) )
             {
                 ReloadMods();
-
-                // May select a different mod than before if mods were added or deleted, but will not crash.
-                if( _selectedModIndex < _plugin.ModManager.Mods.ModSettings.Count )
-                {
-                    _selectedMod = _plugin.ModManager.Mods.ModSettings[ _selectedModIndex ];
-                }
-                else
-                {
-                    _selectedModIndex = 0;
-                    _selectedMod = null;
-                }
-            }
+            }
+            if( ImGui.IsItemHovered() )
+                ImGui.SetTooltip( "Reload the configuration of all mods." );
+
+            ImGui.SameLine();
+            if( ImGui.Button( "Deduplicate" ) )
+            {
+                new Deduplicator(_selectedMod.Mod.ModBasePath, _selectedMod.Mod.Meta).Run();
+                var metaPath = Path.Combine( _selectedMod.Mod.ModBasePath.FullName, "meta.json" );
+                File.WriteAllText( metaPath, JsonConvert.SerializeObject( _selectedMod.Mod.Meta, Formatting.Indented ) );
+                ReloadMods();
+            }
+            if( ImGui.IsItemHovered() )
+                ImGui.SetTooltip( "Try to find identical files and remove duplicate occurences to reduce the mods disk size." );
         }
 
         private void DrawGroupSelectors()
@@ -724,11 +727,12 @@ namespace Penumbra.UI
                             ImGui.EndTabItem();
                         }
                     }
-                    if(_selectedMod.Mod.Meta.Groups.Count >=1) {
-                        if(ImGui.BeginTabItem( "Configuration" )) {
+                    if(_selectedMod.Mod.Meta.HasGroupWithConfig) {
+                        if(ImGui.BeginTabItem( "Configuration" )) 
+                        {
                             DrawGroupSelectors();
                             ImGui.EndTabItem();
-                        }
+                        }
                     }
                     if( ImGui.BeginTabItem( "Files" ) )
                     {
@@ -850,7 +854,18 @@ namespace Penumbra.UI
             // create the directory if it doesn't exist
             Directory.CreateDirectory( _plugin.Configuration.CurrentCollection );
 
-            _plugin.ModManager.DiscoverMods( _plugin.Configuration.CurrentCollection );
+            _plugin.ModManager.DiscoverMods( _plugin.Configuration.CurrentCollection );
+
+            // May select a different mod than before if mods were added or deleted, but will not crash.
+            if( _selectedModIndex < _plugin.ModManager.Mods.ModSettings.Count )
+            {
+                _selectedMod = _plugin.ModManager.Mods.ModSettings[ _selectedModIndex ];
+            }
+            else
+            {
+                _selectedModIndex = 0;
+                _selectedMod = null;
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/Penumbra/Util/SingleOrArrayConverter.cs b/Penumbra/Util/SingleOrArrayConverter.cs
new file mode 100644
index 00000000..3742a4da
--- /dev/null
+++ b/Penumbra/Util/SingleOrArrayConverter.cs
@@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+public class SingleOrArrayConverter<T> : JsonConverter
+{
+    public override bool CanConvert( Type objectType )
+    {
+        return (objectType == typeof(HashSet<T>));
+    }
+
+    public override object ReadJson( JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer )
+    {
+        var token = JToken.Load(reader);
+        if (token.Type == JTokenType.Array)
+        {
+            return token.ToObject<HashSet<T>>();
+        }
+        return new HashSet<T>{ token.ToObject<T>() };
+    }
+
+    public override bool CanWrite => false;
+
+    public override void WriteJson( JsonWriter writer, object value, JsonSerializer serializer )
+    {
+        throw new NotImplementedException();
+    }
+}
+
+public class DictSingleOrArrayConverter<T,U> : JsonConverter
+{
+    public override bool CanConvert( Type objectType )
+    {
+        return (objectType == typeof(Dictionary<T, HashSet<U>>));
+    }
+
+    public override object ReadJson( JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer )
+    {
+        var token = JToken.Load(reader);
+        
+        if (token.Type == JTokenType.Array)
+        {
+            return token.ToObject<HashSet<T>>();
+        }
+        return new HashSet<T>{ token.ToObject<T>() };
+    }
+
+    public override bool CanWrite => false;
+
+    public override void WriteJson( JsonWriter writer, object value, JsonSerializer serializer )
+    {
+        throw new NotImplementedException();
+    }
+}