diff --git a/Dalamud/Configuration/Internal/DalamudConfiguration.cs b/Dalamud/Configuration/Internal/DalamudConfiguration.cs index 2d0a08942..63494931c 100644 --- a/Dalamud/Configuration/Internal/DalamudConfiguration.cs +++ b/Dalamud/Configuration/Internal/DalamudConfiguration.cs @@ -7,6 +7,7 @@ using System.Linq; using Dalamud.Game.Text; using Dalamud.Interface.Style; using Dalamud.Plugin.Internal.Profiles; +using Dalamud.Storage; using Dalamud.Utility; using Newtonsoft.Json; using Serilog; @@ -18,7 +19,7 @@ namespace Dalamud.Configuration.Internal; /// Class containing Dalamud settings. /// [Serializable] -internal sealed class DalamudConfiguration : IServiceType +internal sealed class DalamudConfiguration : IServiceType, IDisposable { private static readonly JsonSerializerSettings SerializerSettings = new() { @@ -422,23 +423,37 @@ internal sealed class DalamudConfiguration : IServiceType /// /// Load a configuration from the provided path. /// - /// The path to load the configuration file from. + /// Path to read from. + /// File storage. /// The deserialized configuration file. - public static DalamudConfiguration Load(string path) + public static DalamudConfiguration Load(string path, ReliableFileStorage fs) { DalamudConfiguration deserialized = null; + try { - deserialized = JsonConvert.DeserializeObject(File.ReadAllText(path), SerializerSettings); + fs.ReadAllText(path, text => + { + deserialized = + JsonConvert.DeserializeObject(text, SerializerSettings); + + // If this reads as null, the file was empty, that's no good + if (deserialized == null) + throw new Exception("Read config was null."); + }); } - catch (Exception ex) + catch (FileNotFoundException) { - Log.Warning(ex, "Failed to load DalamudConfiguration at {0}", path); + // ignored + } + catch (Exception e) + { + Log.Error(e, "Could not load DalamudConfiguration at {Path}, creating new", path); } deserialized ??= new DalamudConfiguration(); deserialized.configPath = path; - + return deserialized; } @@ -457,6 +472,13 @@ internal sealed class DalamudConfiguration : IServiceType { this.Save(); } + + /// + public void Dispose() + { + // Make sure that we save, if a save is queued while we are shutting down + this.Update(); + } /// /// Save the file, if needed. Only needs to be done once a frame. @@ -476,7 +498,8 @@ internal sealed class DalamudConfiguration : IServiceType { ThreadSafety.AssertMainThread(); - Util.WriteAllTextSafe(this.configPath, JsonConvert.SerializeObject(this, SerializerSettings)); + Service.Get().WriteAllText( + this.configPath, JsonConvert.SerializeObject(this, SerializerSettings)); this.DalamudConfigurationSaved?.Invoke(this); } } diff --git a/Dalamud/Configuration/PluginConfigurations.cs b/Dalamud/Configuration/PluginConfigurations.cs index 957a7c99e..de5e071c1 100644 --- a/Dalamud/Configuration/PluginConfigurations.cs +++ b/Dalamud/Configuration/PluginConfigurations.cs @@ -1,6 +1,6 @@ using System.IO; -using Dalamud.Utility; +using Dalamud.Storage; using Newtonsoft.Json; namespace Dalamud.Configuration; @@ -31,24 +31,39 @@ public sealed class PluginConfigurations /// /// Plugin configuration. /// Plugin name. - public void Save(IPluginConfiguration config, string pluginName) + /// WorkingPluginId of the plugin. + public void Save(IPluginConfiguration config, string pluginName, Guid workingPluginId) { - Util.WriteAllTextSafe(this.GetConfigFile(pluginName).FullName, SerializeConfig(config)); + Service.Get() + .WriteAllText(this.GetConfigFile(pluginName).FullName, SerializeConfig(config), workingPluginId); } /// /// Load plugin configuration. /// /// Plugin name. + /// WorkingPluginID of the plugin. /// Plugin configuration. - public IPluginConfiguration? Load(string pluginName) + public IPluginConfiguration? Load(string pluginName, Guid workingPluginId) { var path = this.GetConfigFile(pluginName); - if (!path.Exists) - return null; + IPluginConfiguration? config = null; + try + { + Service.Get().ReadAllText(path.FullName, text => + { + config = DeserializeConfig(text); + if (config == null) + throw new Exception("Read config was null."); + }, workingPluginId); + } + catch (FileNotFoundException) + { + // ignored + } - return DeserializeConfig(File.ReadAllText(path.FullName)); + return config; } /// diff --git a/Dalamud/Dalamud.cs b/Dalamud/Dalamud.cs index a9d822f55..2187f0da2 100644 --- a/Dalamud/Dalamud.cs +++ b/Dalamud/Dalamud.cs @@ -12,6 +12,7 @@ using Dalamud.Game; using Dalamud.Game.Gui.Internal; using Dalamud.Interface.Internal; using Dalamud.Plugin.Internal; +using Dalamud.Storage; using Dalamud.Utility; using PInvoke; using Serilog; @@ -40,14 +41,15 @@ internal sealed class Dalamud : IServiceType /// Initializes a new instance of the class. /// /// DalamudStartInfo instance. + /// ReliableFileStorage instance. /// The Dalamud configuration. /// Event used to signal the main thread to continue. - public Dalamud(DalamudStartInfo info, DalamudConfiguration configuration, IntPtr mainThreadContinueEvent) + public Dalamud(DalamudStartInfo info, ReliableFileStorage fs, DalamudConfiguration configuration, IntPtr mainThreadContinueEvent) { this.unloadSignal = new ManualResetEvent(false); this.unloadSignal.Reset(); - ServiceManager.InitializeProvidedServicesAndClientStructs(this, info, configuration); + ServiceManager.InitializeProvidedServicesAndClientStructs(this, info, fs, configuration); if (!configuration.IsResumeGameAfterPluginLoad) { diff --git a/Dalamud/Dalamud.csproj b/Dalamud/Dalamud.csproj index 5093fbfe9..7ae97e1a6 100644 --- a/Dalamud/Dalamud.csproj +++ b/Dalamud/Dalamud.csproj @@ -77,6 +77,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Dalamud/EntryPoint.cs b/Dalamud/EntryPoint.cs index 7ad794e42..6b53ee3a6 100644 --- a/Dalamud/EntryPoint.cs +++ b/Dalamud/EntryPoint.cs @@ -10,6 +10,7 @@ using Dalamud.Configuration.Internal; using Dalamud.Logging.Internal; using Dalamud.Logging.Retention; using Dalamud.Plugin.Internal; +using Dalamud.Storage; using Dalamud.Support; using Dalamud.Utility; using Newtonsoft.Json; @@ -137,7 +138,8 @@ public sealed class EntryPoint SerilogEventSink.Instance.LogLine += SerilogOnLogLine; // Load configuration first to get some early persistent state, like log level - var configuration = DalamudConfiguration.Load(info.ConfigurationPath!); + var fs = new ReliableFileStorage(Path.GetDirectoryName(info.ConfigurationPath)!); + var configuration = DalamudConfiguration.Load(info.ConfigurationPath!, fs); // Set the appropriate logging level from the configuration if (!configuration.LogSynchronously) @@ -169,7 +171,7 @@ public sealed class EntryPoint if (!Util.IsWine()) InitSymbolHandler(info); - var dalamud = new Dalamud(info, configuration, mainThreadContinueEvent); + var dalamud = new Dalamud(info, fs, configuration, mainThreadContinueEvent); Log.Information("This is Dalamud - Core: {GitHash}, CS: {CsGitHash} [{CsVersion}]", Util.GetGitHash(), Util.GetGitHashClientStructs(), FFXIVClientStructs.Interop.Resolver.Version); dalamud.WaitForUnload(); diff --git a/Dalamud/Plugin/DalamudPluginInterface.cs b/Dalamud/Plugin/DalamudPluginInterface.cs index 6fdf875e5..004b7196c 100644 --- a/Dalamud/Plugin/DalamudPluginInterface.cs +++ b/Dalamud/Plugin/DalamudPluginInterface.cs @@ -343,7 +343,7 @@ public sealed class DalamudPluginInterface : IDisposable if (currentConfig == null) return; - this.configs.Save(currentConfig, this.plugin.InternalName); + this.configs.Save(currentConfig, this.plugin.InternalName, this.plugin.Manifest.WorkingPluginId); } /// @@ -370,7 +370,7 @@ public sealed class DalamudPluginInterface : IDisposable } // this shouldn't be a thing, I think, but just in case - return this.configs.Load(this.plugin.InternalName); + return this.configs.Load(this.plugin.InternalName, this.plugin.Manifest.WorkingPluginId); } /// diff --git a/Dalamud/Plugin/Internal/PluginManager.cs b/Dalamud/Plugin/Internal/PluginManager.cs index f91d4cd56..49608ac9b 100644 --- a/Dalamud/Plugin/Internal/PluginManager.cs +++ b/Dalamud/Plugin/Internal/PluginManager.cs @@ -836,7 +836,7 @@ internal partial class PluginManager : IDisposable, IServiceType var manifestFile = LocalPluginManifest.GetManifestFile(dllFile); // We need to save the repoManifest due to how the repo fills in some fields that authors are not expected to use. - File.WriteAllText(manifestFile.FullName, JsonConvert.SerializeObject(repoManifest, Formatting.Indented)); + Util.WriteAllTextSafe(manifestFile.FullName, JsonConvert.SerializeObject(repoManifest, Formatting.Indented)); // Reload as a local manifest, add some attributes, and save again. var manifest = LocalPluginManifest.Load(manifestFile); diff --git a/Dalamud/ServiceManager.cs b/Dalamud/ServiceManager.cs index ecb58d48b..f2ff864c3 100644 --- a/Dalamud/ServiceManager.cs +++ b/Dalamud/ServiceManager.cs @@ -11,6 +11,7 @@ using Dalamud.Configuration.Internal; using Dalamud.Game; using Dalamud.IoC.Internal; using Dalamud.Logging.Internal; +using Dalamud.Storage; using Dalamud.Utility.Timing; using JetBrains.Annotations; @@ -83,8 +84,9 @@ internal static class ServiceManager /// /// Instance of . /// Instance of . + /// Instance of . /// Instance of . - public static void InitializeProvidedServicesAndClientStructs(Dalamud dalamud, DalamudStartInfo startInfo, DalamudConfiguration configuration) + public static void InitializeProvidedServicesAndClientStructs(Dalamud dalamud, DalamudStartInfo startInfo, ReliableFileStorage fs, DalamudConfiguration configuration) { // Initialize the process information. var cacheDir = new DirectoryInfo(Path.Combine(startInfo.WorkingDirectory!, "cachedSigs")); @@ -98,6 +100,9 @@ internal static class ServiceManager Service.Provide(startInfo); LoadedServices.Add(typeof(DalamudStartInfo)); + + Service.Provide(fs); + LoadedServices.Add(typeof(ReliableFileStorage)); Service.Provide(configuration); LoadedServices.Add(typeof(DalamudConfiguration)); diff --git a/Dalamud/Storage/FileReadException.cs b/Dalamud/Storage/FileReadException.cs new file mode 100644 index 000000000..09f7ff4fb --- /dev/null +++ b/Dalamud/Storage/FileReadException.cs @@ -0,0 +1,16 @@ +namespace Dalamud.Storage; + +/// +/// Thrown if all read operations fail. +/// +public class FileReadException : Exception +{ + /// + /// Initializes a new instance of the class. + /// + /// Inner error that caused this exception. + internal FileReadException(Exception inner) + : base("Failed to read file", inner) + { + } +} diff --git a/Dalamud/Storage/ReliableFileStorage.cs b/Dalamud/Storage/ReliableFileStorage.cs new file mode 100644 index 000000000..fec461cc3 --- /dev/null +++ b/Dalamud/Storage/ReliableFileStorage.cs @@ -0,0 +1,324 @@ +using System.IO; +using System.Runtime.InteropServices; +using System.Text; + +using Dalamud.Logging.Internal; +using Dalamud.Utility; +using PInvoke; +using SQLite; + +namespace Dalamud.Storage; + +/* + * TODO: A file that is read frequently, but written very rarely, might not have offline changes by users persisted + * into the backup database, since it is only written to the backup database when it is written to the filesystem. + */ + +/// +/// A service that provides a reliable file storage. +/// Implements a VFS that writes files to the disk, and additionally keeps files in a SQLite database +/// for journaling/backup purposes. +/// Consumers can choose to receive a backup if they think that the file is corrupt. +/// +/// +/// This is not an early-loaded service, as it is needed before they are initialized. +/// +public class ReliableFileStorage : IServiceType, IDisposable +{ + private static readonly ModuleLog Log = new("VFS"); + + private SQLiteConnection? db; + + /// + /// Initializes a new instance of the class. + /// + /// Path to the VFS. + public ReliableFileStorage(string vfsDbPath) + { + var databasePath = Path.Combine(vfsDbPath, "dalamudVfs.db"); + + Log.Verbose("Initializing VFS database at {Path}", databasePath); + + try + { + this.SetupDb(vfsDbPath); + } + catch (Exception ex) + { + Log.Error(ex, "Failed to load VFS database, starting fresh"); + + try + { + if (File.Exists(vfsDbPath)) + File.Delete(vfsDbPath); + } + catch (Exception) + { + // ignored + } + + this.SetupDb(vfsDbPath); + } + } + + /// + /// Check if a file exists. + /// This will return true if the file does not exist on the filesystem, but in the transparent backup. + /// You must then use this instance to read the file to ensure consistency. + /// + /// The path to check. + /// The container to check in. + /// True if the file exists. + public bool Exists(string path, Guid containerId = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + if (File.Exists(path)) + return true; + + if (this.db == null) + return false; + + // If the file doesn't actually exist on the FS, but it does in the DB, we can say YES and read operations will read from the DB instead + var normalizedPath = NormalizePath(path); + var file = this.db.Table().FirstOrDefault(f => f.Path == normalizedPath && f.ContainerId == containerId); + return file != null; + } + + /// + /// Write all text to a file. + /// + /// Path to write to. + /// The contents of the file. + /// Container to write to. + public void WriteAllText(string path, string? contents, Guid containerId = default) + => this.WriteAllText(path, contents, Encoding.UTF8, containerId); + + /// + /// Write all text to a file. + /// + /// Path to write to. + /// The contents of the file. + /// The encoding to write with. + /// Container to write to. + public void WriteAllText(string path, string? contents, Encoding encoding, Guid containerId = default) + { + var bytes = encoding.GetBytes(contents ?? string.Empty); + this.WriteAllBytes(path, bytes, containerId); + } + + /// + /// Write all bytes to a file. + /// + /// Path to write to. + /// The contents of the file. + /// Container to write to. + public void WriteAllBytes(string path, byte[] bytes, Guid containerId = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + if (this.db == null) + { + Util.WriteAllBytesSafe(path, bytes); + return; + } + + this.db.RunInTransaction(() => + { + var normalizedPath = NormalizePath(path); + var file = this.db.Table().FirstOrDefault(f => f.Path == normalizedPath && f.ContainerId == containerId); + if (file == null) + { + file = new DbFile + { + ContainerId = containerId, + Path = normalizedPath, + Data = bytes, + }; + this.db.Insert(file); + } + else + { + file.Data = bytes; + this.db.Update(file); + } + + Util.WriteAllBytesSafe(path, bytes); + }); + } + + /// + /// Read all text from a file. + /// If the file does not exist on the filesystem, a read is attempted from the backup. The backup is not + /// automatically written back to disk, however. + /// + /// The path to read from. + /// Whether or not the backup of the file should take priority. + /// The container to read from. + /// All text stored in this file. + /// Thrown if the file does not exist on the filesystem or in the backup. + public string ReadAllText(string path, bool forceBackup = false, Guid containerId = default) + => this.ReadAllText(path, Encoding.UTF8, forceBackup, containerId); + + /// + /// Read all text from a file. + /// If the file does not exist on the filesystem, a read is attempted from the backup. The backup is not + /// automatically written back to disk, however. + /// + /// The path to read from. + /// The encoding to read with. + /// Whether or not the backup of the file should take priority. + /// The container to read from. + /// All text stored in this file. + /// Thrown if the file does not exist on the filesystem or in the backup. + public string ReadAllText(string path, Encoding encoding, bool forceBackup = false, Guid containerId = default) + { + var bytes = this.ReadAllBytes(path, forceBackup, containerId); + return encoding.GetString(bytes); + } + + /// + /// Read all text from a file, and automatically try again with the backup if the file does not exist or + /// the function throws an exception. If the backup read also throws an exception, + /// or the file does not exist in the backup, a is thrown. + /// + /// The path to read from. + /// Lambda that reads the file. Throw here to automatically attempt a read from the backup. + /// The container to read from. + /// Thrown if the file does not exist on the filesystem or in the backup. + /// Thrown here if the file and the backup fail their read. + public void ReadAllText(string path, Action reader, Guid containerId = default) + => this.ReadAllText(path, Encoding.UTF8, reader, containerId); + + /// + /// Read all text from a file, and automatically try again with the backup if the file does not exist or + /// the function throws an exception. If the backup read also throws an exception, + /// or the file does not exist in the backup, a is thrown. + /// + /// The path to read from. + /// The encoding to read with. + /// Lambda that reads the file. Throw here to automatically attempt a read from the backup. + /// The container to read from. + /// Thrown if the file does not exist on the filesystem or in the backup. + /// Thrown here if the file and the backup fail their read. + public void ReadAllText(string path, Encoding encoding, Action reader, Guid containerId = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + // TODO: We are technically reading one time too many here, if the file does not exist on the FS, ReadAllText + // fails over to the backup, and then the backup fails to read in the lambda. We should do something about that, + // but it's not a big deal. Would be nice if ReadAllText could indicate if it did fail over. + + // 1.) Try without using the backup + try + { + var text = this.ReadAllText(path, encoding, false, containerId); + reader(text); + return; + } + catch (FileNotFoundException) + { + // We can't do anything about this. + throw; + } + catch (Exception ex) + { + Log.Verbose(ex, "First chance read from {Path} failed, trying backup", path); + } + + // 2.) Try using the backup + try + { + var text = this.ReadAllText(path, encoding, true, containerId); + reader(text); + } + catch (Exception ex) + { + Log.Error(ex, "Second chance read from {Path} failed, giving up", path); + throw new FileReadException(ex); + } + } + + /// + /// Read all bytes from a file. + /// If the file does not exist on the filesystem, a read is attempted from the backup. The backup is not + /// automatically written back to disk, however. + /// + /// The path to read from. + /// Whether or not the backup of the file should take priority. + /// The container to read from. + /// All bytes stored in this file. + /// Thrown if the file does not exist on the filesystem or in the backup. + public byte[] ReadAllBytes(string path, bool forceBackup = false, Guid containerId = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + if (forceBackup) + { + // If the db failed to load, act as if the file does not exist + if (this.db == null) + throw new FileNotFoundException("Backup database was not available"); + + var normalizedPath = NormalizePath(path); + var file = this.db.Table().FirstOrDefault(f => f.Path == normalizedPath && f.ContainerId == containerId); + if (file == null) + throw new FileNotFoundException(); + + return file.Data; + } + + // If the file doesn't exist, immediately check the backup db + if (!File.Exists(path)) + return this.ReadAllBytes(path, true, containerId); + + try + { + return File.ReadAllBytes(path); + } + catch (Exception e) + { + Log.Error(e, "Failed to read file from disk, falling back to database"); + return this.ReadAllBytes(path, true, containerId); + } + } + + /// + public void Dispose() + { + this.db?.Dispose(); + } + + /// + /// Replace possible non-portable parts of a path with portable versions. + /// + /// The path to normalize. + /// The normalized path. + private static string NormalizePath(string path) + { + // Replace users folder + var usersFolder = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + path = path.Replace(usersFolder, "%USERPROFILE%"); + + return path; + } + + private void SetupDb(string path) + { + this.db = new SQLiteConnection(path, + SQLiteOpenFlags.ReadWrite | SQLiteOpenFlags.Create | SQLiteOpenFlags.FullMutex); + this.db.CreateTable(); + } + + private class DbFile + { + [PrimaryKey] + [AutoIncrement] + public int Id { get; set; } + + public Guid ContainerId { get; set; } + + public string Path { get; set; } = null!; + + public byte[] Data { get; set; } = null!; + } +} diff --git a/Dalamud/Utility/Util.cs b/Dalamud/Utility/Util.cs index 8ca87b691..36918abd2 100644 --- a/Dalamud/Utility/Util.cs +++ b/Dalamud/Utility/Util.cs @@ -20,6 +20,7 @@ using Dalamud.Logging.Internal; using Dalamud.Memory; using ImGuiNET; using Lumina.Excel.GeneratedSheets; +using PInvoke; using Serilog; namespace Dalamud.Utility; @@ -609,7 +610,7 @@ public static class Util } } } - + /// /// Overwrite text in a file by first writing it to a temporary file, and then /// moving that file to the path specified. @@ -618,12 +619,58 @@ public static class Util /// The text to write. public static void WriteAllTextSafe(string path, string text) { - var tmpPath = path + ".tmp"; - if (File.Exists(tmpPath)) - File.Delete(tmpPath); + WriteAllTextSafe(path, text, Encoding.UTF8); + } + + /// + /// Overwrite text in a file by first writing it to a temporary file, and then + /// moving that file to the path specified. + /// + /// The path of the file to write to. + /// The text to write. + /// Encoding to use. + public static void WriteAllTextSafe(string path, string text, Encoding encoding) + { + WriteAllBytesSafe(path, encoding.GetBytes(text)); + } + + /// + /// Overwrite data in a file by first writing it to a temporary file, and then + /// moving that file to the path specified. + /// + /// The path of the file to write to. + /// The data to write. + public static void WriteAllBytesSafe(string path, byte[] bytes) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + // Open the temp file + var tempPath = path + ".tmp"; - File.WriteAllText(tmpPath, text); - File.Move(tmpPath, path, true); + using var tempFile = Kernel32 + .CreateFile(tempPath.AsSpan(), + new Kernel32.ACCESS_MASK(Kernel32.FileAccess.FILE_GENERIC_READ | Kernel32.FileAccess.FILE_GENERIC_WRITE), + Kernel32.FileShare.None, + null, + Kernel32.CreationDisposition.CREATE_ALWAYS, + Kernel32.CreateFileFlags.FILE_ATTRIBUTE_NORMAL, + Kernel32.SafeObjectHandle.Null); + + if (tempFile.IsInvalid) + throw new Win32Exception(); + + // Write the data + var bytesWritten = Kernel32.WriteFile(tempFile, new ArraySegment(bytes)); + if (bytesWritten != bytes.Length) + throw new Exception($"Could not write all bytes to temp file ({bytesWritten} of {bytes.Length})"); + + if (!Kernel32.FlushFileBuffers(tempFile)) + throw new Win32Exception(); + + tempFile.Close(); + + if (!MoveFileEx(tempPath, path, MoveFileFlags.MovefileReplaceExisting | MoveFileFlags.MovefileWriteThrough)) + throw new Win32Exception(); } /// @@ -762,4 +809,18 @@ public static class Util } } } + + [Flags] +#pragma warning disable SA1201 + private enum MoveFileFlags +#pragma warning restore SA1201 + { + MovefileReplaceExisting = 0x00000001, + MovefileWriteThrough = 0x00000008, + } + + [return: MarshalAs(UnmanagedType.Bool)] + [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Unicode)] + private static extern bool MoveFileEx(string lpExistingFileName, string lpNewFileName, + MoveFileFlags dwFlags); }