diff --git a/Dalamud/Configuration/Internal/DalamudConfiguration.cs b/Dalamud/Configuration/Internal/DalamudConfiguration.cs index 2d0a08942..55bf82496 100644 --- a/Dalamud/Configuration/Internal/DalamudConfiguration.cs +++ b/Dalamud/Configuration/Internal/DalamudConfiguration.cs @@ -7,6 +7,7 @@ using System.Linq; using Dalamud.Game.Text; using Dalamud.Interface.Style; using Dalamud.Plugin.Internal.Profiles; +using Dalamud.Storage; using Dalamud.Utility; using Newtonsoft.Json; using Serilog; @@ -18,7 +19,7 @@ namespace Dalamud.Configuration.Internal; /// Class containing Dalamud settings. /// [Serializable] -internal sealed class DalamudConfiguration : IServiceType +internal sealed class DalamudConfiguration : IServiceType, IDisposable { private static readonly JsonSerializerSettings SerializerSettings = new() { @@ -422,23 +423,33 @@ internal sealed class DalamudConfiguration : IServiceType /// /// Load a configuration from the provided path. /// - /// The path to load the configuration file from. + /// Path to read from. + /// File storage. /// The deserialized configuration file. - public static DalamudConfiguration Load(string path) + public static DalamudConfiguration Load(string path, ReliableFileStorage fs) { DalamudConfiguration deserialized = null; + try { - deserialized = JsonConvert.DeserializeObject(File.ReadAllText(path), SerializerSettings); + fs.ReadAllText(path, text => + { + deserialized = + JsonConvert.DeserializeObject(text, SerializerSettings); + }); } - catch (Exception ex) + catch (FileNotFoundException) { - Log.Warning(ex, "Failed to load DalamudConfiguration at {0}", path); + // ignored + } + catch (Exception e) + { + Log.Error(e, "Could not load DalamudConfiguration at {Path}, creating new", path); } deserialized ??= new DalamudConfiguration(); deserialized.configPath = path; - + return deserialized; } @@ -457,6 +468,13 @@ internal sealed class DalamudConfiguration : IServiceType { this.Save(); } + + /// + public void Dispose() + { + // Make sure that we save, if a save is queued while we are shutting down + this.Update(); + } /// /// Save the file, if needed. Only needs to be done once a frame. @@ -476,7 +494,8 @@ internal sealed class DalamudConfiguration : IServiceType { ThreadSafety.AssertMainThread(); - Util.WriteAllTextSafe(this.configPath, JsonConvert.SerializeObject(this, SerializerSettings)); + Service.Get().WriteAllText( + this.configPath, JsonConvert.SerializeObject(this, SerializerSettings)); this.DalamudConfigurationSaved?.Invoke(this); } } diff --git a/Dalamud/Dalamud.cs b/Dalamud/Dalamud.cs index a9d822f55..2187f0da2 100644 --- a/Dalamud/Dalamud.cs +++ b/Dalamud/Dalamud.cs @@ -12,6 +12,7 @@ using Dalamud.Game; using Dalamud.Game.Gui.Internal; using Dalamud.Interface.Internal; using Dalamud.Plugin.Internal; +using Dalamud.Storage; using Dalamud.Utility; using PInvoke; using Serilog; @@ -40,14 +41,15 @@ internal sealed class Dalamud : IServiceType /// Initializes a new instance of the class. /// /// DalamudStartInfo instance. + /// ReliableFileStorage instance. /// The Dalamud configuration. /// Event used to signal the main thread to continue. - public Dalamud(DalamudStartInfo info, DalamudConfiguration configuration, IntPtr mainThreadContinueEvent) + public Dalamud(DalamudStartInfo info, ReliableFileStorage fs, DalamudConfiguration configuration, IntPtr mainThreadContinueEvent) { this.unloadSignal = new ManualResetEvent(false); this.unloadSignal.Reset(); - ServiceManager.InitializeProvidedServicesAndClientStructs(this, info, configuration); + ServiceManager.InitializeProvidedServicesAndClientStructs(this, info, fs, configuration); if (!configuration.IsResumeGameAfterPluginLoad) { diff --git a/Dalamud/Dalamud.csproj b/Dalamud/Dalamud.csproj index 5093fbfe9..7ae97e1a6 100644 --- a/Dalamud/Dalamud.csproj +++ b/Dalamud/Dalamud.csproj @@ -77,6 +77,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Dalamud/EntryPoint.cs b/Dalamud/EntryPoint.cs index 7ad794e42..6b53ee3a6 100644 --- a/Dalamud/EntryPoint.cs +++ b/Dalamud/EntryPoint.cs @@ -10,6 +10,7 @@ using Dalamud.Configuration.Internal; using Dalamud.Logging.Internal; using Dalamud.Logging.Retention; using Dalamud.Plugin.Internal; +using Dalamud.Storage; using Dalamud.Support; using Dalamud.Utility; using Newtonsoft.Json; @@ -137,7 +138,8 @@ public sealed class EntryPoint SerilogEventSink.Instance.LogLine += SerilogOnLogLine; // Load configuration first to get some early persistent state, like log level - var configuration = DalamudConfiguration.Load(info.ConfigurationPath!); + var fs = new ReliableFileStorage(Path.GetDirectoryName(info.ConfigurationPath)!); + var configuration = DalamudConfiguration.Load(info.ConfigurationPath!, fs); // Set the appropriate logging level from the configuration if (!configuration.LogSynchronously) @@ -169,7 +171,7 @@ public sealed class EntryPoint if (!Util.IsWine()) InitSymbolHandler(info); - var dalamud = new Dalamud(info, configuration, mainThreadContinueEvent); + var dalamud = new Dalamud(info, fs, configuration, mainThreadContinueEvent); Log.Information("This is Dalamud - Core: {GitHash}, CS: {CsGitHash} [{CsVersion}]", Util.GetGitHash(), Util.GetGitHashClientStructs(), FFXIVClientStructs.Interop.Resolver.Version); dalamud.WaitForUnload(); diff --git a/Dalamud/ServiceManager.cs b/Dalamud/ServiceManager.cs index ecb58d48b..38dc7534f 100644 --- a/Dalamud/ServiceManager.cs +++ b/Dalamud/ServiceManager.cs @@ -11,6 +11,7 @@ using Dalamud.Configuration.Internal; using Dalamud.Game; using Dalamud.IoC.Internal; using Dalamud.Logging.Internal; +using Dalamud.Storage; using Dalamud.Utility.Timing; using JetBrains.Annotations; @@ -83,8 +84,9 @@ internal static class ServiceManager /// /// Instance of . /// Instance of . + /// Instance of /// Instance of . - public static void InitializeProvidedServicesAndClientStructs(Dalamud dalamud, DalamudStartInfo startInfo, DalamudConfiguration configuration) + public static void InitializeProvidedServicesAndClientStructs(Dalamud dalamud, DalamudStartInfo startInfo, ReliableFileStorage fs, DalamudConfiguration configuration) { // Initialize the process information. var cacheDir = new DirectoryInfo(Path.Combine(startInfo.WorkingDirectory!, "cachedSigs")); @@ -98,6 +100,9 @@ internal static class ServiceManager Service.Provide(startInfo); LoadedServices.Add(typeof(DalamudStartInfo)); + + Service.Provide(fs); + LoadedServices.Add(typeof(ReliableFileStorage)); Service.Provide(configuration); LoadedServices.Add(typeof(DalamudConfiguration)); diff --git a/Dalamud/Storage/ReliableFileStorage.cs b/Dalamud/Storage/ReliableFileStorage.cs new file mode 100644 index 000000000..14ab59143 --- /dev/null +++ b/Dalamud/Storage/ReliableFileStorage.cs @@ -0,0 +1,337 @@ +using System.IO; +using System.Runtime.InteropServices; +using System.Text; + +using Dalamud.Logging.Internal; +using PInvoke; +using SQLite; + +namespace Dalamud.Storage; + +/* + * TODO: A file that is read frequently, but written very rarely, might not have offline changes by users persisted + * into the backup database, since it is only written to the backup database when it is written to the filesystem. + */ + +/// +/// A service that provides a reliable file storage. +/// Implements a VFS that writes files to the disk, and additionally keeps files in a SQLite database +/// for journaling/backup purposes. +/// Consumers can choose to receive a backup if they think that the file is corrupt. +/// +/// +/// This is not an early-loaded service, as it is needed before they are initialized. +/// +public class ReliableFileStorage : IServiceType, IDisposable +{ + private static readonly ModuleLog Log = new("VFS"); + + private SQLiteConnection db; + + /// + /// Initializes a new instance of the class. + /// + /// Path to the VFS. + [ServiceManager.ServiceConstructor] + public ReliableFileStorage(string vfsDbPath) + { + var databasePath = Path.Combine(vfsDbPath, "dalamudVfs.db"); + + Log.Verbose("Initializing VFS database at {Path}", databasePath); + this.db = new SQLiteConnection(databasePath, SQLiteOpenFlags.ReadWrite | SQLiteOpenFlags.Create | SQLiteOpenFlags.FullMutex); + this.db.CreateTable(); + } + + /// + /// Check if a file exists. + /// This will return true if the file does not exist on the filesystem, but in the transparent backup. + /// You must then use this instance to read the file to ensure consistency. + /// + /// The path to check. + /// The container to check in. + /// True if the file exists. + public bool Exists(string path, Guid containerId = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + if (File.Exists(path)) + return true; + + // If the file doesn't actually exist on the FS, but it does in the DB, we can say YES and read operations will read from the DB instead + var normalizedPath = NormalizePath(path); + var file = this.db.Table().FirstOrDefault(f => f.Path == normalizedPath && f.ContainerId == containerId); + return file != null; + } + + /// + /// Write all text to a file. + /// + /// Path to write to. + /// The contents of the file. + /// Container to write to. + public void WriteAllText(string path, string? contents, Guid containerId = default) + => this.WriteAllText(path, contents, Encoding.UTF8, containerId); + + /// + /// Write all text to a file. + /// + /// Path to write to. + /// The contents of the file. + /// The encoding to write with. + /// Container to write to. + public void WriteAllText(string path, string? contents, Encoding encoding, Guid containerId = default) + { + var bytes = encoding.GetBytes(contents ?? string.Empty); + this.WriteAllBytes(path, bytes, containerId); + } + + /// + /// Write all bytes to a file. + /// + /// Path to write to. + /// The contents of the file. + /// Container to write to. + public void WriteAllBytes(string path, byte[] bytes, Guid containerId = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + var normalizedPath = NormalizePath(path); + var file = this.db.Table().FirstOrDefault(f => f.Path == normalizedPath && f.ContainerId == containerId); + if (file == null) + { + file = new DbFile + { + ContainerId = containerId, + Path = normalizedPath, + Data = bytes, + }; + this.db.Insert(file); + } + else + { + file.Data = bytes; + this.db.Update(file); + } + + WriteFileReliably(path, bytes); + } + + /// + /// Read all text from a file. + /// If the file does not exist on the filesystem, a read is attempted from the backup. The backup is not + /// automatically written back to disk, however. + /// + /// The path to read from. + /// Whether or not the backup of the file should take priority. + /// The container to read from. + /// All text stored in this file. + /// Thrown if the file does not exist on the filesystem or in the backup. + public string ReadAllText(string path, bool forceBackup = false, Guid containerId = default) + => this.ReadAllText(path, Encoding.UTF8, forceBackup, containerId); + + /// + /// Read all text from a file. + /// If the file does not exist on the filesystem, a read is attempted from the backup. The backup is not + /// automatically written back to disk, however. + /// + /// The path to read from. + /// The encoding to read with. + /// Whether or not the backup of the file should take priority. + /// The container to read from. + /// All text stored in this file. + /// Thrown if the file does not exist on the filesystem or in the backup. + public string ReadAllText(string path, Encoding encoding, bool forceBackup = false, Guid containerId = default) + { + var bytes = this.ReadAllBytes(path, forceBackup, containerId); + return encoding.GetString(bytes); + } + + /// + /// Read all text from a file, and automatically try again with the backup if the file does not exist or + /// the function throws an exception. If the backup read also throws an exception, + /// or the file does not exist in the backup, a is thrown. + /// + /// The path to read from. + /// Lambda that reads the file. Throw here to automatically attempt a read from the backup. + /// The container to read from. + /// Thrown if the file does not exist on the filesystem or in the backup. + /// Thrown here if the file and the backup fail their read. + public void ReadAllText(string path, Action reader, Guid containerId = default) + => this.ReadAllText(path, Encoding.UTF8, reader, containerId); + + /// + /// Read all text from a file, and automatically try again with the backup if the file does not exist or + /// the function throws an exception. If the backup read also throws an exception, + /// or the file does not exist in the backup, a is thrown. + /// + /// The path to read from. + /// The encoding to read with. + /// Lambda that reads the file. Throw here to automatically attempt a read from the backup. + /// The container to read from. + /// Thrown if the file does not exist on the filesystem or in the backup. + /// Thrown here if the file and the backup fail their read. + public void ReadAllText(string path, Encoding encoding, Action reader, Guid containerId = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + // TODO: We are technically reading one time too many here, if the file does not exist on the FS, ReadAllText + // fails over to the backup, and then the backup fails to read in the lambda. We should do something about that, + // but it's not a big deal. Would be nice if ReadAllText could indicate if it did fail over. + + // 1.) Try without using the backup + try + { + var text = this.ReadAllText(path, encoding, false, containerId); + reader(text); + return; + } + catch (FileNotFoundException) + { + // We can't do anything about this. + throw; + } + catch (Exception ex) + { + Log.Verbose(ex, "First chance read from {Path} failed, trying backup", path); + } + + // 2.) Try using the backup + try + { + var text = this.ReadAllText(path, encoding, true, containerId); + reader(text); + } + catch (Exception ex) + { + Log.Error(ex, "Second chance read from {Path} failed, giving up", path); + throw new FileReadException(ex); + } + } + + /// + /// Read all bytes from a file. + /// If the file does not exist on the filesystem, a read is attempted from the backup. The backup is not + /// automatically written back to disk, however. + /// + /// The path to read from. + /// Whether or not the backup of the file should take priority. + /// The container to read from. + /// All bytes stored in this file. + /// Thrown if the file does not exist on the filesystem or in the backup. + public byte[] ReadAllBytes(string path, bool forceBackup = false, Guid containerId = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + if (forceBackup) + { + var normalizedPath = NormalizePath(path); + var file = this.db.Table().FirstOrDefault(f => f.Path == normalizedPath && f.ContainerId == containerId); + if (file == null) + throw new FileNotFoundException(); + + return file.Data; + } + + // If the file doesn't exist, immediately check the backup db + if (!File.Exists(path)) + return this.ReadAllBytes(path, true, containerId); + + try + { + return File.ReadAllBytes(path); + } + catch (Exception e) + { + Log.Error(e, "Failed to read file from disk, falling back to database"); + return this.ReadAllBytes(path, true, containerId); + } + } + + /// + public void Dispose() + { + this.db.Dispose(); + } + + private static void WriteFileReliably(string path, byte[] bytes) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + // Open the temp file + var tempPath = path + ".tmp"; + + using var tempFile = Kernel32 + .CreateFile(tempPath.AsSpan(), + new Kernel32.ACCESS_MASK(Kernel32.FileAccess.FILE_GENERIC_READ | Kernel32.FileAccess.FILE_GENERIC_WRITE), + Kernel32.FileShare.None, + null, + Kernel32.CreationDisposition.CREATE_ALWAYS, + Kernel32.CreateFileFlags.FILE_ATTRIBUTE_NORMAL, + Kernel32.SafeObjectHandle.Null); + + if (tempFile.IsInvalid) + throw new Win32Exception(); + + // Write the data + var bytesWritten = Kernel32.WriteFile(tempFile, new ArraySegment(bytes)); + if (bytesWritten != bytes.Length) + throw new Exception($"Could not write all bytes to temp file ({bytesWritten} of {bytes.Length})"); + + if (!Kernel32.FlushFileBuffers(tempFile)) + throw new Win32Exception(); + + tempFile.Close(); + + if (!MoveFileEx(tempPath, path, MoveFileFlags.MovefileReplaceExisting | MoveFileFlags.MovefileWriteThrough)) + throw new Win32Exception(); + } + + /// + /// Replace possible non-portable parts of a path with portable versions. + /// + /// The path to normalize. + /// The normalized path. + private static string NormalizePath(string path) + { + // Replace users folder + var usersFolder = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + path = path.Replace(usersFolder, "%USERPROFILE%"); + + return path; + } + + [Flags] +#pragma warning disable SA1201 + private enum MoveFileFlags +#pragma warning restore SA1201 + { + MovefileReplaceExisting = 0x00000001, + MovefileWriteThrough = 0x00000008, + } + + [return: MarshalAs(UnmanagedType.Bool)] + [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Unicode)] + private static extern bool MoveFileEx(string lpExistingFileName, string lpNewFileName, + MoveFileFlags dwFlags); + + private class DbFile + { + [PrimaryKey] + [AutoIncrement] + public int Id { get; set; } + + public Guid ContainerId { get; set; } + + public string Path { get; set; } = null!; + + public byte[] Data { get; set; } = null!; + } +} + +public class FileReadException : Exception +{ + public FileReadException(Exception inner) + : base("Failed to read file", inner) + { + } +}