ToxicMushroom/Penumbra
1
0
Fork 0
mirror of https://github.com/xivdev/Penumbra.git synced 2025-12-12 10:17:22 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added Deduplication button and the ability to point a hdd file to multiple game paths in groups.

Browse source
This commit is contained in:
Ottermandias 2021-01-18 14:53:28 +01:00 • committed by Ottermandias
parent fd2e020eec
commit 06b0fb7e0c
8 changed files with 358 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Penumbra/Importer/TexToolsImport.cs
View file

@ -243,12 +243,12 @@ namespace Penumbra.Importer
{
OptionName = opt.Name,
OptionDesc = String.IsNullOrEmpty(opt.Description) ? "" : opt.Description,
OptionFiles = new Dictionary<string, string>()
OptionFiles = new Dictionary<string, HashSet<string>>()
};
var optDir = new DirectoryInfo(Path.Combine( groupFolder.FullName, opt.Name));
foreach ( var file in optDir.EnumerateFiles("*.*", SearchOption.AllDirectories) )
{
optio.OptionFiles[file.FullName.Substring( baseFolder.FullName.Length ).TrimStart( '\\' )] = file.FullName.Substring( optDir.FullName.Length ).TrimStart( '\\' ).Replace( '\\', '/' );
optio.AddFile(file.FullName.Substring(baseFolder.FullName.Length).TrimStart('\\'), file.FullName.Substring(optDir.FullName.Length).TrimStart('\\').Replace('\\','/'));
}
Inf.Options.Add( optio );
}

162
Penumbra/Models/Deduplicator.cs Normal file
View file

@ -0,0 +1,162 @@
using System.Collections.Generic;
using System.Security.Cryptography;
using System.IO;
using System.Linq;
using System.Collections;
using Dalamud.Plugin;
namespace Penumbra.Models
{
public class Deduplicator
{
private DirectoryInfo baseDir;
private int baseDirLength;
private ModMeta mod;
private SHA256 hasher = null;
private Dictionary<long, List<FileInfo>> filesBySize;
private ref SHA256 Sha()
{
if (hasher == null)
hasher = SHA256.Create();
return ref hasher;
}
public Deduplicator(DirectoryInfo baseDir, ModMeta mod)
{
this.baseDir = baseDir;
this.baseDirLength = baseDir.FullName.Length;
this.mod = mod;
filesBySize = new();
BuildDict();
}
private void BuildDict()
{
foreach( var file in baseDir.EnumerateFiles( "*.*", SearchOption.AllDirectories ) )
{
var fileLength = file.Length;
if (filesBySize.TryGetValue(fileLength, out var files))
files.Add(file);
else
filesBySize[fileLength] = new(){ file };
}
}
public void Run()
{
foreach (var pair in filesBySize)
{
if (pair.Value.Count < 2)
continue;
if (pair.Value.Count == 2)
{
if (CompareFilesDirectly(pair.Value[0], pair.Value[1]))
ReplaceFile(pair.Value[0], pair.Value[1]);
}
else
{
var deleted = Enumerable.Repeat(false, pair.Value.Count).ToArray();
var hashes = pair.Value.Select( F => ComputeHash(F)).ToArray();
for (var i = 0; i < pair.Value.Count; ++i)
{
if (deleted[i])
continue;
for (var j = i + 1; j < pair.Value.Count; ++j)
{
if (deleted[j])
continue;
if (!CompareHashes(hashes[i], hashes[j]))
continue;
ReplaceFile(pair.Value[i], pair.Value[j]);
deleted[j] = true;
}
}
}
}
ClearEmptySubDirectories(baseDir);
}
private void ReplaceFile(FileInfo f1, FileInfo f2)
{
var relName1 = f1.FullName.Substring(baseDirLength).TrimStart('\\');
var relName2 = f2.FullName.Substring(baseDirLength).TrimStart('\\');
var inOption = false;
foreach (var group in mod.Groups.Select( g => g.Value.Options))
{
foreach (var option in group)
{
if (option.OptionFiles.TryGetValue(relName2, out var values))
{
inOption = true;
foreach (var value in values)
option.AddFile(relName1, value);
}
}
}
if (!inOption)
{
const string duplicates = "Duplicates";
if (!mod.Groups.ContainsKey(duplicates))
{
InstallerInfo info = new()
{
GroupName = duplicates,
SelectionType = SelectType.Single,
Options = new()
{
new()
{
OptionName = "Required",
OptionDesc = "",
OptionFiles = new()
}
}
};
mod.Groups.Add(duplicates, info);
}
mod.Groups[duplicates].Options[0].AddFile(relName1, relName2.Replace('\\', '/'));
mod.Groups[duplicates].Options[0].AddFile(relName1, relName1.Replace('\\', '/'));
}
PluginLog.Information($"File {relName1} and {relName2} are identical. Deleting the second.");
f2.Delete();
}
public static bool CompareFilesDirectly(FileInfo f1, FileInfo f2)
{
return File.ReadAllBytes(f1.FullName).SequenceEqual(File.ReadAllBytes(f2.FullName));
}
public static bool CompareHashes(byte[] f1, byte[] f2)
{
return StructuralComparisons.StructuralEqualityComparer.Equals(f1, f2);
}
public byte[] ComputeHash(FileInfo f)
{
var stream = File.OpenRead( f.FullName );
var ret = Sha().ComputeHash(stream);
stream.Dispose();
return ret;
}
// Does not delete the base directory itself even if it is completely empty at the end.
public static void ClearEmptySubDirectories(DirectoryInfo baseDir)
{
foreach (var subDir in baseDir.GetDirectories())
{
ClearEmptySubDirectories(subDir);
if (subDir.GetFiles().Length == 0 && subDir.GetDirectories().Length == 0)
subDir.Delete();
}
}
}
}

18
Penumbra/Models/GroupInformation.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using Newtonsoft.Json;
namespace Penumbra.Models
{
@ -10,10 +11,21 @@ namespace Penumbra.Models
{
public string OptionName;
public string OptionDesc;
public Dictionary<string, string> OptionFiles;
}
public struct InstallerInfo
[JsonProperty(ItemConverterType = typeof(SingleOrArrayConverter<string>))]
public Dictionary<string, HashSet<string>> OptionFiles;
public bool AddFile(string filePath, string gamePath)
{
if (OptionFiles.TryGetValue(filePath, out var set))
return set.Add(gamePath);
else
OptionFiles[filePath] = new(){ gamePath };
return true;
}
}
public struct InstallerInfo {
public string GroupName;
public SelectType SelectionType;
public List<Option> Options;

4
Penumbra/Models/ModMeta.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using Newtonsoft.Json;
namespace Penumbra.Models
{
@ -18,5 +19,8 @@ namespace Penumbra.Models
public Dictionary< string, string > FileSwaps { get; } = new();
public Dictionary<string, InstallerInfo> Groups { get; set; } = new();
[JsonIgnore]
public bool HasGroupWithConfig { get; set; } = false;
}
}

2
Penumbra/Mods/ModCollection.cs
View file

@ -69,6 +69,8 @@ namespace Penumbra.Mods
try
{
meta = JsonConvert.DeserializeObject< ModMeta >( File.ReadAllText( metaFile.FullName ) );
meta.HasGroupWithConfig = meta.Groups != null && meta.Groups.Count > 0
&& meta.Groups.Values.Any( G => G.SelectionType == SelectType.Multi || G.Options.Count > 1);
}
catch( Exception e )
{

121
Penumbra/Mods/ModManager.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Penumbra.Models;
namespace Penumbra.Mods
@ -102,21 +103,7 @@ namespace Penumbra.Mods
// Needed to reload body textures with mods
//_plugin.GameUtils.ReloadPlayerResources();
}
private (InstallerInfo, Option, string) GlobalPosition( string rel, Dictionary<string, InstallerInfo> gps )
{
string filePath = null;
foreach( var g in gps )
{
foreach( var opt in g.Value.Options )
{
if( opt.OptionFiles.TryGetValue( rel, out filePath ) )
{
return (g.Value, opt, filePath);
}
}
}
return (default( InstallerInfo ), default( Option ), null);
}
public void CalculateEffectiveFileList()
{
ResolvedFiles.Clear();
@ -131,54 +118,20 @@ namespace Penumbra.Mods
// fixup path
var baseDir = mod.ModBasePath.FullName;
if(settings.Conf == null) {
settings.Conf = new();
_plugin.ModManager.Mods.Save();
}
foreach( var file in mod.ModFiles )
{
var relativeFilePath = file.FullName.Substring( baseDir.Length ).TrimStart( '\\' );
string gamePath;
bool addFile = true;
var gps = mod.Meta.Groups;
if( gps.Count >= 1 )
bool doNotAdd = false;
void AddFiles(HashSet<string> gamePaths)
{
var negivtron = GlobalPosition( relativeFilePath, gps );
if( negivtron.Item3 != null )
{
if( settings.Conf == null )
{
settings.Conf = new();
_plugin.ModManager.Mods.Save();
}
if( !settings.Conf.ContainsKey( negivtron.Item1.GroupName ) )
{
settings.Conf[negivtron.Item1.GroupName] = 0;
_plugin.ModManager.Mods.Save();
}
var current = settings.Conf[negivtron.Item1.GroupName];
var flag = negivtron.Item1.Options.IndexOf( negivtron.Item2 );
switch( negivtron.Item1.SelectionType )
{
case SelectType.Single:
{
addFile = current == flag;
break;
}
case SelectType.Multi:
{
flag = 1 << negivtron.Item1.Options.IndexOf( negivtron.Item2 );
addFile = ( flag & current ) != 0;
break;
}
}
gamePath = negivtron.Item3;
}
else
{
gamePath = relativeFilePath.Replace( '\\', '/' );
}
}
else
gamePath = relativeFilePath.Replace( '\\', '/' );
if( addFile )
doNotAdd = true;
foreach (var gamePath in gamePaths)
{
if( !ResolvedFiles.ContainsKey( gamePath ) ) {
ResolvedFiles[ gamePath.ToLowerInvariant() ] = file;
@ -190,6 +143,58 @@ namespace Penumbra.Mods
}
}
}
HashSet<string> paths;
foreach (var group in mod.Meta.Groups.Select( G => G.Value))
{
if (!settings.Conf.TryGetValue(group.GroupName, out var setting)
|| (group.SelectionType == SelectType.Single && settings.Conf[group.GroupName] >= group.Options.Count))
{
settings.Conf[group.GroupName] = 0;
_plugin.ModManager.Mods.Save();
setting = 0;
}
if (group.Options.Count == 0)
continue;
if (group.SelectionType == SelectType.Multi)
settings.Conf[group.GroupName] &= ((1 << group.Options.Count) - 1);
switch(group.SelectionType)
{
case SelectType.Single:
if (group.Options[setting].OptionFiles.TryGetValue(relativeFilePath, out paths))
AddFiles(paths);
else
{
for(var i = 0; i < group.Options.Count; ++i)
{
if (i == setting)
continue;
if(group.Options[i].OptionFiles.ContainsKey(relativeFilePath))
{
doNotAdd = true;
break;
}
}
}
break;
case SelectType.Multi:
for(var i = 0; i < group.Options.Count; ++i)
{
if ((setting & (1 << i)) != 0)
if (group.Options[i].OptionFiles.TryGetValue(relativeFilePath, out paths))
AddFiles(paths);
}
break;
}
}
if (!doNotAdd)
AddFiles(new() { relativeFilePath.Replace( '\\', '/' ) });
}
foreach( var swap in mod.Meta.FileSwaps )
{
// just assume people put not fucked paths in here lol

47
Penumbra/UI/SettingsInterface.cs
View file

@ -1,4 +1,5 @@
using System;
using System.ComponentModel.Design;
using System.Diagnostics;
using System.IO;
using System.Linq;
@ -547,11 +548,7 @@ namespace Penumbra.UI
}
if( ImGui.IsItemHovered() )
{
ImGui.BeginTooltip();
ImGui.Text( _selectedMod.Mod.Meta.Website );
ImGui.EndTooltip();
}
ImGui.SetTooltip( _selectedMod.Mod.Meta.Website );
}
else
{
@ -569,6 +566,8 @@ namespace Penumbra.UI
{
Process.Start( _selectedMod.Mod.ModBasePath.FullName );
}
if( ImGui.IsItemHovered() )
ImGui.SetTooltip( "Open the directory containing this mod in your default file explorer." );
ImGui.SameLine();
if( ImGui.Button( "Edit JSON" ) )
@ -577,23 +576,27 @@ namespace Penumbra.UI
File.WriteAllText( metaPath, JsonConvert.SerializeObject( _selectedMod.Mod.Meta, Formatting.Indented ) );
Process.Start( metaPath );
}
if( ImGui.IsItemHovered() )
ImGui.SetTooltip( "Open the JSON configuration file in your default application for .json." );
ImGui.SameLine();
if( ImGui.Button( "Reload JSON" ) )
{
ReloadMods();
}
if( ImGui.IsItemHovered() )
ImGui.SetTooltip( "Reload the configuration of all mods." );
// May select a different mod than before if mods were added or deleted, but will not crash.
if( _selectedModIndex < _plugin.ModManager.Mods.ModSettings.Count )
ImGui.SameLine();
if( ImGui.Button( "Deduplicate" ) )
{
_selectedMod = _plugin.ModManager.Mods.ModSettings[ _selectedModIndex ];
}
else
{
_selectedModIndex = 0;
_selectedMod = null;
}
new Deduplicator(_selectedMod.Mod.ModBasePath, _selectedMod.Mod.Meta).Run();
var metaPath = Path.Combine( _selectedMod.Mod.ModBasePath.FullName, "meta.json" );
File.WriteAllText( metaPath, JsonConvert.SerializeObject( _selectedMod.Mod.Meta, Formatting.Indented ) );
ReloadMods();
}
if( ImGui.IsItemHovered() )
ImGui.SetTooltip( "Try to find identical files and remove duplicate occurences to reduce the mods disk size." );
}
private void DrawGroupSelectors()
@ -724,8 +727,9 @@ namespace Penumbra.UI
ImGui.EndTabItem();
}
}
if(_selectedMod.Mod.Meta.Groups.Count >=1) {
if(ImGui.BeginTabItem( "Configuration" )) {
if(_selectedMod.Mod.Meta.HasGroupWithConfig) {
if(ImGui.BeginTabItem( "Configuration" ))
{
DrawGroupSelectors();
ImGui.EndTabItem();
}
@ -851,6 +855,17 @@ namespace Penumbra.UI
Directory.CreateDirectory( _plugin.Configuration.CurrentCollection );
_plugin.ModManager.DiscoverMods( _plugin.Configuration.CurrentCollection );
// May select a different mod than before if mods were added or deleted, but will not crash.
if( _selectedModIndex < _plugin.ModManager.Mods.ModSettings.Count )
{
_selectedMod = _plugin.ModManager.Mods.ModSettings[ _selectedModIndex ];
}
else
{
_selectedModIndex = 0;
_selectedMod = null;
}
}
}
}

55
Penumbra/Util/SingleOrArrayConverter.cs Normal file
View file

@ -0,0 +1,55 @@
using System;
using System.Collections.Generic;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
public class SingleOrArrayConverter<T> : JsonConverter
{
public override bool CanConvert( Type objectType )
{
return (objectType == typeof(HashSet<T>));
}
public override object ReadJson( JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer )
{
var token = JToken.Load(reader);
if (token.Type == JTokenType.Array)
{
return token.ToObject<HashSet<T>>();
}
return new HashSet<T>{ token.ToObject<T>() };
}
public override bool CanWrite => false;
public override void WriteJson( JsonWriter writer, object value, JsonSerializer serializer )
{
throw new NotImplementedException();
}
}
public class DictSingleOrArrayConverter<T,U> : JsonConverter
{
public override bool CanConvert( Type objectType )
{
return (objectType == typeof(Dictionary<T, HashSet<U>>));
}
public override object ReadJson( JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer )
{
var token = JToken.Load(reader);
if (token.Type == JTokenType.Array)
{
return token.ToObject<HashSet<T>>();
}
return new HashSet<T>{ token.ToObject<T>() };
}
public override bool CanWrite => false;
public override void WriteJson( JsonWriter writer, object value, JsonSerializer serializer )
{
throw new NotImplementedException();
}
}