ToxicMushroom/Penumbra
1
0
Fork 0
mirror of https://github.com/xivdev/Penumbra.git synced 2025-12-12 10:17:22 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make extracting mods cancelable, some fixes.

Browse source
This commit is contained in:
Ottermandias 2022-04-30 16:26:39 +02:00
parent cf54bc7f57
commit 5e46f43d7d
9 changed files with 182 additions and 108 deletions
Download patch file Download diff file Expand all files Collapse all files

2
OtterGui

@ -1 +1 @@
Subproject commit 627e313232a2e602432dcc4d090dccd5e27993a1
Subproject commit a1ff5ca207080786225f716a0e2487e206923a52

5
Penumbra/Configuration.Migration.cs
View file

@ -30,6 +30,11 @@ public partial class Configuration
public static void Migrate( Configuration config )
{
if( !File.Exists( Dalamud.PluginInterface.ConfigFile.FullName ) )
{
return;
}
var m = new Migration
{
_config = config,

81
Penumbra/Import/TexToolsImport.cs
View file

@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Dalamud.Logging;
using ICSharpCode.SharpZipLib.Zip;
@ -11,7 +13,7 @@ using FileMode = System.IO.FileMode;
namespace Penumbra.Import;
public partial class TexToolsImporter
public partial class TexToolsImporter : IDisposable
{
private const string TempFileName = "textools-import";
private static readonly JsonSerializerSettings JsonSettings = new() { NullValueHandling = NullValueHandling.Ignore };
@ -21,22 +23,59 @@ public partial class TexToolsImporter
private readonly IEnumerable< FileInfo > _modPackFiles;
private readonly int _modPackCount;
private FileStream? _tmpFileStream;
private StreamDisposer? _streamDisposer;
private readonly CancellationTokenSource _cancellation = new();
private readonly CancellationToken _token;
public ImporterState State { get; private set; }
public readonly List< (FileInfo File, DirectoryInfo? Mod, Exception? Error) > ExtractedMods;
public TexToolsImporter( DirectoryInfo baseDirectory, ICollection< FileInfo > files )
: this( baseDirectory, files.Count, files )
public TexToolsImporter( DirectoryInfo baseDirectory, ICollection< FileInfo > files,
Action< FileInfo, DirectoryInfo?, Exception? > handler )
: this( baseDirectory, files.Count, files, handler )
{ }
public TexToolsImporter( DirectoryInfo baseDirectory, int count, IEnumerable< FileInfo > modPackFiles )
public TexToolsImporter( DirectoryInfo baseDirectory, int count, IEnumerable< FileInfo > modPackFiles,
Action< FileInfo, DirectoryInfo?, Exception? > handler )
{
_baseDirectory = baseDirectory;
_tmpFile = Path.Combine( _baseDirectory.FullName, TempFileName );
_modPackFiles = modPackFiles;
_modPackCount = count;
ExtractedMods = new List< (FileInfo, DirectoryInfo?, Exception?) >( count );
Task.Run( ImportFiles );
_token = _cancellation.Token;
Task.Run( ImportFiles, _token )
.ContinueWith( _ => CloseStreams() )
.ContinueWith( _ =>
{
foreach( var (file, dir, error) in ExtractedMods )
{
handler( file, dir, error );
}
} );
}
private void CloseStreams()
{
_tmpFileStream?.Dispose();
_tmpFileStream = null;
ResetStreamDisposer();
}
public void Dispose()
{
_cancellation.Cancel( true );
if( State != ImporterState.WritingPackToDisk )
{
_tmpFileStream?.Dispose();
_tmpFileStream = null;
}
if( State != ImporterState.ExtractingModFiles )
{
ResetStreamDisposer();
}
}
private void ImportFiles()
@ -45,6 +84,13 @@ public partial class TexToolsImporter
_currentModPackIdx = 0;
foreach( var file in _modPackFiles )
{
_currentModDirectory = null;
if( _token.IsCancellationRequested )
{
ExtractedMods.Add( ( file, null, new TaskCanceledException( "Task canceled by user." ) ) );
continue;
}
try
{
var directory = VerifyVersionAndImport( file );
@ -52,7 +98,7 @@ public partial class TexToolsImporter
}
catch( Exception e )
{
ExtractedMods.Add( ( file, null, e ) );
ExtractedMods.Add( ( file, _currentModDirectory, e ) );
_currentNumOptions = 0;
_currentOptionIdx = 0;
_currentFileIdx = 0;
@ -88,7 +134,7 @@ public partial class TexToolsImporter
PluginLog.Warning( $"File {modPackFile.FullName} seems to be a V2 TTMP, but has the wrong extension." );
}
return ImportV2ModPack( _: modPackFile, extractedModPack, modRaw );
return ImportV2ModPack( modPackFile, extractedModPack, modRaw );
}
if( modPackFile.Extension != ".ttmp" )
@ -129,11 +175,19 @@ public partial class TexToolsImporter
private void WriteZipEntryToTempFile( Stream s )
{
using var fs = new FileStream( _tmpFile, FileMode.Create );
s.CopyTo( fs );
_tmpFileStream?.Dispose(); // should not happen
_tmpFileStream = new FileStream( _tmpFile, FileMode.Create );
if( _token.IsCancellationRequested )
{
return;
}
s.CopyTo( _tmpFileStream );
_tmpFileStream.Dispose();
_tmpFileStream = null;
}
private PenumbraSqPackStream GetSqPackStreamStream( ZipFile file, string entryName )
private StreamDisposer GetSqPackStreamStream( ZipFile file, string entryName )
{
State = ImporterState.WritingPackToDisk;
@ -148,7 +202,14 @@ public partial class TexToolsImporter
WriteZipEntryToTempFile( s );
_streamDisposer?.Dispose(); // Should not happen.
var fs = new FileStream( _tmpFile, FileMode.Open );
return new StreamDisposer( fs );
}
private void ResetStreamDisposer()
{
_streamDisposer?.Dispose();
_streamDisposer = null;
}
}

3
Penumbra/Import/TexToolsImporter.Gui.cs
View file

@ -91,4 +91,7 @@ public partial class TexToolsImporter
}
}
}
public bool DrawCancelButton( Vector2 size )
=> ImGuiUtil.DrawDisabledButton( "Cancel", size, string.Empty, _token.IsCancellationRequested );
}

114
Penumbra/Import/TexToolsImporter.ModPack.cs
View file

@ -13,6 +13,8 @@ namespace Penumbra.Import;
public partial class TexToolsImporter
{
private DirectoryInfo? _currentModDirectory;
// Version 1 mod packs are a simple collection of files without much information.
private DirectoryInfo ImportV1ModPack( FileInfo modPackFile, ZipFile extractedModPack, string modRaw )
{
@ -31,16 +33,16 @@ public partial class TexToolsImporter
var modList = modListRaw.Select( m => JsonConvert.DeserializeObject< SimpleMod >( m, JsonSettings )! ).ToList();
// Open the mod data file from the mod pack as a SqPackStream
using var modData = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" );
var ret = Mod.CreateModFolder( _baseDirectory, Path.GetFileNameWithoutExtension( modPackFile.Name ) );
_currentModDirectory = Mod.CreateModFolder( _baseDirectory, Path.GetFileNameWithoutExtension( modPackFile.Name ) );
// Create a new ModMeta from the TTMP mod list info
Mod.CreateMeta( ret, _currentModName, DefaultTexToolsData.Author, DefaultTexToolsData.Description, null, null );
Mod.CreateMeta( _currentModDirectory, _currentModName, DefaultTexToolsData.Author, DefaultTexToolsData.Description, null, null );
ExtractSimpleModList( ret, modList, modData );
Mod.CreateDefaultFiles( ret );
return ret;
// Open the mod data file from the mod pack as a SqPackStream
_streamDisposer = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" );
ExtractSimpleModList( _currentModDirectory, modList );
Mod.CreateDefaultFiles( _currentModDirectory );
ResetStreamDisposer();
return _currentModDirectory;
}
// Version 2 mod packs can either be simple or extended, import accordingly.
@ -87,17 +89,17 @@ public partial class TexToolsImporter
_currentOptionName = DefaultTexToolsData.DefaultOption;
PluginLog.Log( " -> Importing Simple V2 ModPack" );
// Open the mod data file from the mod pack as a SqPackStream
using var modData = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" );
var ret = Mod.CreateModFolder( _baseDirectory, _currentModName );
Mod.CreateMeta( ret, _currentModName, modList.Author, string.IsNullOrEmpty( modList.Description )
_currentModDirectory = Mod.CreateModFolder( _baseDirectory, _currentModName );
Mod.CreateMeta( _currentModDirectory, _currentModName, modList.Author, string.IsNullOrEmpty( modList.Description )
? "Mod imported from TexTools mod pack"
: modList.Description, null, null );
ExtractSimpleModList( ret, modList.SimpleModsList, modData );
Mod.CreateDefaultFiles( ret );
return ret;
// Open the mod data file from the mod pack as a SqPackStream
_streamDisposer = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" );
ExtractSimpleModList( _currentModDirectory, modList.SimpleModsList );
Mod.CreateDefaultFiles( _currentModDirectory );
ResetStreamDisposer();
return _currentModDirectory;
}
// Obtain the number of relevant options to extract.
@ -118,23 +120,24 @@ public partial class TexToolsImporter
var modList = JsonConvert.DeserializeObject< ExtendedModPack >( modRaw, JsonSettings )!;
_currentNumOptions = GetOptionCount( modList );
_currentModName = modList.Name;
// Open the mod data file from the mod pack as a SqPackStream
using var modData = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" );
var ret = Mod.CreateModFolder( _baseDirectory, _currentModName );
Mod.CreateMeta( ret, _currentModName, modList.Author, modList.Description, modList.Version, null );
_currentModDirectory = Mod.CreateModFolder( _baseDirectory, _currentModName );
Mod.CreateMeta( _currentModDirectory, _currentModName, modList.Author, modList.Description, modList.Version, null );
if( _currentNumOptions == 0 )
{
return ret;
return _currentModDirectory;
}
// Open the mod data file from the mod pack as a SqPackStream
_streamDisposer = GetSqPackStreamStream( extractedModPack, "TTMPD.mpd" );
// It can contain a simple list, still.
if( modList.SimpleModsList.Length > 0 )
{
_currentGroupName = string.Empty;
_currentOptionName = "Default";
ExtractSimpleModList( ret, modList.SimpleModsList, modData );
ExtractSimpleModList( _currentModDirectory, modList.SimpleModsList );
}
// Iterate through all pages
@ -147,18 +150,19 @@ public partial class TexToolsImporter
_currentGroupName = group.GroupName;
options.Clear();
var description = new StringBuilder();
var groupFolder = Mod.NewSubFolderName( ret, group.GroupName )
?? new DirectoryInfo( Path.Combine( ret.FullName, $"Group {groupPriority + 1}" ) );
var groupFolder = Mod.NewSubFolderName( _currentModDirectory, group.GroupName )
?? new DirectoryInfo( Path.Combine( _currentModDirectory.FullName, $"Group {groupPriority + 1}" ) );
var optionIdx = 1;
foreach( var option in group.OptionList.Where( option => option.Name.Length > 0 && option.ModsJsons.Length > 0 ) )
{
_token.ThrowIfCancellationRequested();
_currentOptionName = option.Name;
var optionFolder = Mod.NewSubFolderName( groupFolder, option.Name )
?? new DirectoryInfo( Path.Combine( groupFolder.FullName, $"Option {optionIdx}" ) );
ExtractSimpleModList( optionFolder, option.ModsJsons, modData );
options.Add( Mod.CreateSubMod( ret, optionFolder, option ) );
ExtractSimpleModList( optionFolder, option.ModsJsons );
options.Add( Mod.CreateSubMod( _currentModDirectory, optionFolder, option ) );
description.Append( option.Description );
if( !string.IsNullOrEmpty( option.Description ) )
{
@ -169,15 +173,16 @@ public partial class TexToolsImporter
++_currentOptionIdx;
}
Mod.CreateOptionGroup( ret, group, groupPriority++, description.ToString(), options );
Mod.CreateOptionGroup( _currentModDirectory, group, groupPriority++, description.ToString(), options );
}
}
Mod.CreateDefaultFiles( ret );
return ret;
ResetStreamDisposer();
Mod.CreateDefaultFiles( _currentModDirectory );
return _currentModDirectory;
}
private void ExtractSimpleModList( DirectoryInfo outDirectory, ICollection< SimpleMod > mods, PenumbraSqPackStream dataStream )
private void ExtractSimpleModList( DirectoryInfo outDirectory, ICollection< SimpleMod > mods )
{
State = ImporterState.ExtractingModFiles;
@ -187,35 +192,34 @@ public partial class TexToolsImporter
// Extract each SimpleMod into the new mod folder
foreach( var simpleMod in mods )
{
ExtractMod( outDirectory, simpleMod, dataStream );
ExtractMod( outDirectory, simpleMod );
++_currentFileIdx;
}
}
private void ExtractMod( DirectoryInfo outDirectory, SimpleMod mod, PenumbraSqPackStream dataStream )
private void ExtractMod( DirectoryInfo outDirectory, SimpleMod mod )
{
if( _streamDisposer is not PenumbraSqPackStream stream )
{
return;
}
PluginLog.Log( " -> Extracting {0} at {1}", mod.FullPath, mod.ModOffset.ToString( "X" ) );
try
_token.ThrowIfCancellationRequested();
var data = stream.ReadFile< PenumbraSqPackStream.PenumbraFileResource >( mod.ModOffset );
_currentFileName = mod.FullPath;
var extractedFile = new FileInfo( Path.Combine( outDirectory.FullName, mod.FullPath ) );
extractedFile.Directory?.Create();
if( extractedFile.FullName.EndsWith( ".mdl" ) )
{
var data = dataStream.ReadFile< PenumbraSqPackStream.PenumbraFileResource >( mod.ModOffset );
_currentFileName = mod.FullPath;
var extractedFile = new FileInfo( Path.Combine( outDirectory.FullName, mod.FullPath ) );
extractedFile.Directory?.Create();
if( extractedFile.FullName.EndsWith( ".mdl" ) )
{
ProcessMdl( data.Data );
}
File.WriteAllBytes( extractedFile.FullName, data.Data );
}
catch( Exception ex )
{
PluginLog.LogError( ex, "Could not extract mod." );
ProcessMdl( data.Data );
}
File.WriteAllBytes( extractedFile.FullName, data.Data );
}
private static void ProcessMdl( byte[] mdl )
@ -226,11 +230,11 @@ public partial class TexToolsImporter
mdl[ 64 ] = 1;
// Model header LOD num
var stackSize = BitConverter.ToUInt32( mdl, 4 );
var runtimeBegin = stackSize + 0x44;
var stringsLengthOffset = runtimeBegin + 4;
var stringsLength = BitConverter.ToUInt32( mdl, ( int )stringsLengthOffset );
var modelHeaderStart = stringsLengthOffset + stringsLength + 4;
var stackSize = BitConverter.ToUInt32( mdl, 4 );
var runtimeBegin = stackSize + 0x44;
var stringsLengthOffset = runtimeBegin + 4;
var stringsLength = BitConverter.ToUInt32( mdl, ( int )stringsLengthOffset );
var modelHeaderStart = stringsLengthOffset + stringsLength + 4;
mdl[ modelHeaderStart + modelHeaderLodOffset ] = 1;
}
}

10
Penumbra/Interop/Loader/ResourceLoader.Replacement.cs
View file

@ -145,7 +145,7 @@ public unsafe partial class ResourceLoader
return ReadSqPackHook.Original( resourceManager, fileDescriptor, priority, isSync );
}
if( !Utf8GamePath.FromSpan( fileDescriptor->ResourceHandle->FileNameSpan(), out var gamePath, false ) )
if( !Utf8GamePath.FromSpan( fileDescriptor->ResourceHandle->FileNameSpan(), out var gamePath, false ) || gamePath.Length == 0 )
{
return ReadSqPackHook.Original( resourceManager, fileDescriptor, priority, isSync );
}
@ -164,17 +164,13 @@ public unsafe partial class ResourceLoader
fileDescriptor->ResourceHandle->FileNameData = split[ 2 ].Path;
fileDescriptor->ResourceHandle->FileNameLength = split[ 2 ].Length;
// Force isSync = true for these calls. I don't really understand why,
// or where the difference even comes from.
// Was called with True on my client and with false on other peoples clients,
// which caused problems.
var funcFound = ResourceLoadCustomization.GetInvocationList()
.Any( f => ( ( ResourceLoadCustomizationDelegate )f )
.Invoke( split[ 1 ], split[ 2 ], resourceManager, fileDescriptor, priority, true, out ret ) );
.Invoke( split[ 1 ], split[ 2 ], resourceManager, fileDescriptor, priority, isSync, out ret ) );
if( !funcFound )
{
ret = DefaultLoadResource( split[ 2 ], resourceManager, fileDescriptor, priority, true );
ret = DefaultLoadResource( split[ 2 ], resourceManager, fileDescriptor, priority, isSync );
}
// Return original resource handle path so that they can be loaded separately.

7
Penumbra/Interop/Resolver/PathResolver.Material.cs
View file

@ -90,8 +90,11 @@ public unsafe partial class PathResolver
PluginLog.Verbose( "Using MtrlLoadHandler with no collection for path {$Path:l}.", path );
}
ret = Penumbra.ResourceLoader.DefaultLoadResource( path, resourceManager, fileDescriptor, priority, isSync );
// Force isSync = true for this call. I don't really understand why,
// or where the difference even comes from.
// Was called with True on my client and with false on other peoples clients,
// which caused problems.
ret = Penumbra.ResourceLoader.DefaultLoadResource( path, resourceManager, fileDescriptor, priority, true );
PathCollections.TryRemove( path, out _ );
return true;
}

2
Penumbra/Penumbra.cs
View file

@ -365,7 +365,7 @@ public class Penumbra : IDalamudPlugin
private static IReadOnlyList< FileInfo > PenumbraBackupFiles()
{
var collectionDir = ModCollection.CollectionDirectory;
var list = Directory.Exists(collectionDir)
var list = Directory.Exists( collectionDir )
? new DirectoryInfo( collectionDir ).EnumerateFiles( "*.json" ).ToList()
: new List< FileInfo >();
list.Add( Dalamud.PluginInterface.ConfigFile );

66
Penumbra/UI/Classes/ModFileSystemSelector.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Threading.Tasks;
using Dalamud.Interface;
using Dalamud.Interface.ImGuiFileDialog;
using Dalamud.Logging;
@ -55,6 +56,8 @@ public sealed partial class ModFileSystemSelector : FileSystemSelector< Mod, Mod
Penumbra.CollectionManager.Current.ModSettingChanged -= OnSettingChange;
Penumbra.CollectionManager.Current.InheritanceChanged -= OnInheritanceChange;
Penumbra.CollectionManager.CollectionChanged -= OnCollectionChange;
_import?.Dispose();
_import = null;
}
public new ModFileSystem.Leaf? SelectedLeaf
@ -160,7 +163,8 @@ public sealed partial class ModFileSystemSelector : FileSystemSelector< Mod, Mod
{
if( s )
{
_import = new TexToolsImporter( Penumbra.ModManager.BasePath, f.Count, f.Select( file => new FileInfo( file ) ) );
_import = new TexToolsImporter( Penumbra.ModManager.BasePath, f.Count, f.Select( file => new FileInfo( file ) ),
AddNewMod );
ImGui.OpenPopup( "Import Status" );
}
}, 0, modPath );
@ -177,51 +181,49 @@ public sealed partial class ModFileSystemSelector : FileSystemSelector< Mod, Mod
ImGui.SetNextWindowSize( display / 4 );
ImGui.SetNextWindowPos( 3 * display / 8 );
using var popup = ImRaii.Popup( "Import Status", ImGuiWindowFlags.Modal );
if( _import != null && popup.Success )
if( _import == null || !popup.Success )
{
_import.DrawProgressInfo( new Vector2( -1, ImGui.GetFrameHeight() ) );
if( _import.State == ImporterState.Done )
{
ImGui.SetCursorPosY( ImGui.GetWindowHeight() - ImGui.GetFrameHeight() * 2 );
if( ImGui.Button( "Close", -Vector2.UnitX ) )
{
AddNewMods( _import.ExtractedMods );
_import = null;
ImGui.CloseCurrentPopup();
}
}
return;
}
_import.DrawProgressInfo( new Vector2( -1, ImGui.GetFrameHeight() ) );
ImGui.SetCursorPosY( ImGui.GetWindowHeight() - ImGui.GetFrameHeight() * 2 );
if( _import.State == ImporterState.Done && ImGui.Button( "Close", -Vector2.UnitX )
|| _import.State != ImporterState.Done && _import.DrawCancelButton( -Vector2.UnitX ) )
{
_import?.Dispose();
_import = null;
ImGui.CloseCurrentPopup();
}
}
// Clean up invalid directories if necessary.
// Add all successfully extracted mods.
private static void AddNewMods( IEnumerable< (FileInfo File, DirectoryInfo? Mod, Exception? Error) > list )
// Clean up invalid directory if necessary.
// Add successfully extracted mods.
private static void AddNewMod( FileInfo file, DirectoryInfo? dir, Exception? error )
{
foreach( var (file, dir, error) in list )
if( error != null )
{
if( error != null )
if( dir != null && Directory.Exists( dir.FullName ) )
{
if( dir != null && Directory.Exists( dir.FullName ) )
try
{
try
{
Directory.Delete( dir.FullName );
}
catch( Exception e )
{
PluginLog.Error( $"Error cleaning up failed mod extraction of {file.FullName} to {dir.FullName}:\n{e}" );
}
Directory.Delete( dir.FullName, true );
}
catch( Exception e )
{
PluginLog.Error( $"Error cleaning up failed mod extraction of {file.FullName} to {dir.FullName}:\n{e}" );
}
PluginLog.Error( $"Error extracting {file.FullName}, mod skipped:\n{error}" );
continue;
}
if( dir != null )
if( error is not OperationCanceledException )
{
Penumbra.ModManager.AddMod( dir );
PluginLog.Error( $"Error extracting {file.FullName}, mod skipped:\n{error}" );
}
}
else if( dir != null )
{
Penumbra.ModManager.AddMod( dir );
}
}
private void DeleteModButton( Vector2 size )