Fix log culling

Dalamud.Boot/logging.cpp

@@ -10,32 +10,102 @@ static bool s_bLoaded = false;
 static bool s_bSkipLogFileWrite = false;
 static std::shared_ptr<void> s_hLogFile;
 
-void logging::start_file_logging(const std::filesystem::path& path, bool redirect_stderrout) {
+void logging::start_file_logging(const std::filesystem::path& logPath, bool redirect_stderrout) {
+    constexpr auto MaxLogFileSize = 1 * 1024 * 1024;
+    constexpr auto MaxOldFileSize = 10 * 1024 * 1024;
+    char buf[4096];
+
     if (s_hLogFile)
         return;
 
+    const auto oldPath = std::filesystem::path(logPath).replace_extension(".old.log");
+    
     try {
-        if (exists(path) && file_size(path) > 1048576) {
+        const auto oldPathOld = std::filesystem::path(logPath).replace_extension(".log.old");
-            auto oldPath = std::filesystem::path(path);
+        if (exists(oldPathOld)) {
-            oldPath.replace_extension(".log.old");
             if (exists(oldPath))
-                remove(oldPath);
+                remove(oldPathOld);
-            rename(path, oldPath);
+            else
+                rename(oldPathOld, oldPath);
         }
     } catch (...) {
         // whatever
     }
 
-    const auto h = CreateFile(path.wstring().c_str(),
+    const auto h = CreateFileW(logPath.wstring().c_str(),
-                              GENERIC_WRITE,
+                               GENERIC_READ | GENERIC_WRITE,
                                FILE_SHARE_READ | FILE_SHARE_WRITE,
                                nullptr, OPEN_ALWAYS, 0, nullptr);
     if (h == INVALID_HANDLE_VALUE)
         throw std::runtime_error(std::format("Win32 error {}(0x{:x})", GetLastError(), GetLastError()));
 
-    SetFilePointer(h, 0, 0, FILE_END);
     s_hLogFile = {h, &CloseHandle};
 
+    // 1. Move excess data from logPath to oldPath
+    if (LARGE_INTEGER fsize; SetFilePointerEx(h, {}, &fsize, FILE_END) && fsize.QuadPart > MaxLogFileSize) {
+        const auto amountToMove = (std::min<int64_t>)(fsize.QuadPart - MaxLogFileSize, MaxOldFileSize);
+        SetFilePointerEx(h, LARGE_INTEGER{.QuadPart = -(MaxLogFileSize + amountToMove)}, nullptr, FILE_END);
+
+        const auto hOld = CreateFileW(oldPath.c_str(),
+                                      GENERIC_READ | GENERIC_WRITE,
+                                      FILE_SHARE_READ | FILE_SHARE_WRITE,
+                                      nullptr, OPEN_ALWAYS, 0, nullptr);
+        if (hOld != INVALID_HANDLE_VALUE) {
+            const auto hOldCloser = std::unique_ptr<std::remove_pointer_t<HANDLE>, decltype(&CloseHandle)>(hOld, &CloseHandle);
+            SetFilePointerEx(hOld, {}, nullptr, FILE_END);
+
+            DWORD read = 0, written = 0;
+            for (int64_t i = 0; i < amountToMove; i += sizeof buf) {
+                const auto chunkSize = static_cast<DWORD>((std::min<int64_t>)(sizeof buf, amountToMove - i));
+                if (!ReadFile(h, buf, chunkSize, &read, nullptr) || read != chunkSize)
+                    break;
+                if (!WriteFile(hOld, buf, read, &written, nullptr) || read != written)
+                    break;
+            }
+        }
+    }
+
+    // 2. Cull each of .log and .old files
+    for (const auto& [path, maxSize] : std::initializer_list<std::pair<std::filesystem::path, int64_t>>{
+        {oldPath, MaxOldFileSize},
+        {logPath, MaxLogFileSize},
+    }) {
+        try {
+            const auto hFileRead = CreateFileW(path.c_str(),
+                GENERIC_READ,
+                FILE_SHARE_READ | FILE_SHARE_WRITE,
+                nullptr, OPEN_EXISTING, 0, nullptr);
+            if (hFileRead == INVALID_HANDLE_VALUE)
+                continue;
+            const auto closeRead = std::unique_ptr<std::remove_pointer_t<HANDLE>, decltype(&CloseHandle)>(hFileRead, &CloseHandle);
+
+            if (LARGE_INTEGER ptr; !SetFilePointerEx(hFileRead, { .QuadPart = -maxSize }, &ptr, FILE_END) || ptr.QuadPart <= 0)
+                continue;
+
+            const auto hFileWrite = CreateFileW(path.c_str(),
+                GENERIC_WRITE,
+                FILE_SHARE_READ | FILE_SHARE_WRITE,
+                nullptr, OPEN_EXISTING, 0, nullptr);
+            if (hFileWrite == INVALID_HANDLE_VALUE)
+                continue;
+            const auto closeWrite = std::unique_ptr<std::remove_pointer_t<HANDLE>, decltype(&CloseHandle)>(hFileWrite, &CloseHandle);
+
+            DWORD read = 0, written = 0;
+            for (int64_t i = 0; i < maxSize; i += sizeof buf) {
+                const auto chunkSize = static_cast<DWORD>((std::min<int64_t>)(sizeof buf, maxSize - i));
+                if (!ReadFile(hFileRead, buf, chunkSize, &read, nullptr) || read != chunkSize)
+                    break;
+                if (!WriteFile(hFileWrite, buf, read, &written, nullptr) || read != written)
+                    break;
+            }
+
+            SetEndOfFile(hFileWrite);
+        } catch (...) {
+            // ignore
+        }
+    }
+
+    SetFilePointerEx(h, {}, nullptr, FILE_END);
     if (redirect_stderrout) {
         SetStdHandle(STD_ERROR_HANDLE, h);
         SetStdHandle(STD_OUTPUT_HANDLE, h);
@@ -81,11 +151,13 @@ void logging::print<char>(Level level, const char* s) {
 
     // Handle accesses should not be done during DllMain process attach/detach calls
     if (s_bLoaded) {
+        if (s_hLogFile)
+            SetFilePointerEx(s_hLogFile.get(), {}, nullptr, FILE_END);
+
         DWORD wr{};
         WriteFile(GetStdHandle(STD_ERROR_HANDLE), &estr[0], static_cast<DWORD>(estr.size()), &wr, nullptr);
 
-        if (s_hLogFile && !s_bSkipLogFileWrite) {
+        if (s_hLogFile && !s_bSkipLogFileWrite)
             WriteFile(s_hLogFile.get(), &estr[0], static_cast<DWORD>(estr.size()), &wr, nullptr);
     }
-    }
 }

Dalamud.Boot/logging.h

@@ -24,7 +24,7 @@ namespace logging {
     /**
      * @brief Starts writing log to specified file.
      */
-    void start_file_logging(const std::filesystem::path& path, bool redirect_stderrout = false);
+    void start_file_logging(const std::filesystem::path& logPath, bool redirect_stderrout = false);
 
     /**
      * @brief Marks this DLL either as loaded or unloaded, top prevent accessing handles when the DLL is not loaded.

Dalamud/EntryPoint.cs

@@ -85,15 +85,26 @@ namespace Dalamud
         {
 #if DEBUG
             var logPath = Path.Combine(baseDirectory, "dalamud.log");
-            var oldPath = Path.Combine(baseDirectory, "dalamud.log.old");
+            var oldPath = Path.Combine(baseDirectory, "dalamud.old.log");
+            var oldPathOld = Path.Combine(baseDirectory, "dalamud.log.old");
 #else
             var logPath = Path.Combine(baseDirectory, "..", "..", "..", "dalamud.log");
-            var oldPath = Path.Combine(baseDirectory, "..", "..", "..", "dalamud.log.old");
+            var oldPath = Path.Combine(baseDirectory, "..", "..", "..", "dalamud.old.log");
+            var oldPathOld = Path.Combine(baseDirectory, "..", "..", "..", "dalamud.log.old");
 #endif
             Log.CloseAndFlush();
 
-            CullLogFile(logPath, oldPath, 1 * 1024 * 1024);
+            var oldFileOld = new FileInfo(oldPathOld);
-            CullLogFile(oldPath, null, 10 * 1024 * 1024);
+            if (oldFileOld.Exists)
+            {
+                var oldFile = new FileInfo(oldPath);
+                if (oldFile.Exists)
+                    oldFileOld.Delete();
+                else
+                    oldFileOld.MoveTo(oldPath);
+            }
+
+            CullLogFile(logPath, 1 * 1024 * 1024, oldPath, 10 * 1024 * 1024);
 
             var config = new LoggerConfiguration()
                          .WriteTo.Sink(SerilogEventSink.Instance)
@@ -217,67 +228,76 @@ namespace Dalamud
             }
         }
 
-        private static void CullLogFile(string logPath, string? oldPath, int cullingFileSize)
+        /// <summary>
+        /// Trim existing log file to a specified length, and optionally move the excess data to another file.
+        /// </summary>
+        /// <param name="logPath">Target log file to trim.</param>
+        /// <param name="logMaxSize">Maximum size of target log file.</param>
+        /// <param name="oldPath">.old file to move excess data to.</param>
+        /// <param name="oldMaxSize">Maximum size of .old file.</param>
+        private static void CullLogFile(string logPath, int logMaxSize, string oldPath, int oldMaxSize)
         {
+            var logFile = new FileInfo(logPath);
+            var oldFile = new FileInfo(oldPath);
+            var targetFiles = new[]
+            {
+                (logFile, logMaxSize),
+                (oldFile, oldMaxSize),
+            };
+            var buffer = new byte[4096];
+
             try
             {
-                var bufferSize = 4096;
-
-                var logFile = new FileInfo(logPath);
-
                 if (!logFile.Exists)
-                    logFile.Create();
+                    logFile.Create().Close();
 
-                if (logFile.Length <= cullingFileSize)
+                // 1. Move excess data from logFile to oldFile
-                    return;
+                if (logFile.Length > logMaxSize)
-
-                var amountToCull = logFile.Length - cullingFileSize;
-
-                if (amountToCull < bufferSize)
-                    return;
-
-                if (oldPath != null)
                 {
-                    var oldFile = new FileInfo(oldPath);
+                    using var reader = logFile.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+                    using var writer = oldFile.Open(FileMode.Append, FileAccess.Write, FileShare.ReadWrite);
 
-                    if (!oldFile.Exists)
+                    var amountToMove = (int)Math.Min(logFile.Length - logMaxSize, oldMaxSize);
-                        oldFile.Create().Close();
+                    reader.Seek(-(logMaxSize + amountToMove), SeekOrigin.End);
 
-                    using var reader = new BinaryReader(logFile.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
+                    for (var i = 0; i < amountToMove; i += buffer.Length)
-                    using var writer = new BinaryWriter(oldFile.Open(FileMode.Append, FileAccess.Write, FileShare.ReadWrite));
+                        writer.Write(buffer, 0, reader.Read(buffer, 0, Math.Min(buffer.Length, amountToMove - i)));
+                }
 
-                    var read = -1;
+                // 2. Cull each of .log and .old files
-                    var total = 0;
+                foreach (var (file, maxSize) in targetFiles)
-                    var buffer = new byte[bufferSize];
-                    while (read != 0 && total < amountToCull)
                 {
-                        read = reader.Read(buffer, 0, buffer.Length);
+                    if (!file.Exists || file.Length <= maxSize)
+                        continue;
+
+                    using var reader = file.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+                    using var writer = file.Open(FileMode.Open, FileAccess.Write, FileShare.ReadWrite);
+
+                    reader.Seek(file.Length - maxSize, SeekOrigin.Begin);
+                    for (int read; (read = reader.Read(buffer, 0, buffer.Length)) > 0;)
                         writer.Write(buffer, 0, read);
-                        total += read;
-                    }
-                }
 
-                {
+                    writer.SetLength(maxSize);
-                    using var reader = new BinaryReader(logFile.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
-                    using var writer = new BinaryWriter(logFile.Open(FileMode.Open, FileAccess.Write, FileShare.ReadWrite));
-
-                    reader.BaseStream.Seek(amountToCull, SeekOrigin.Begin);
-
-                    var read = -1;
-                    var total = 0;
-                    var buffer = new byte[bufferSize];
-                    while (read != 0)
-                    {
-                        read = reader.Read(buffer, 0, buffer.Length);
-                        writer.Write(buffer, 0, read);
-                        total += read;
-                    }
-
-                    writer.BaseStream.SetLength(total);
                 }
             }
             catch (Exception ex)
             {
+                if (ex is IOException)
+                {
+                    foreach (var (file, _) in targetFiles)
+                    {
+                        try
+                        {
+                            if (file.Exists)
+                                file.Delete();
+                        }
+                        catch (Exception ex2)
+                        {
+                            Log.Error(ex2, "Failed to delete {file}", file.FullName);
+                        }
+                    }
+                }
+
                 Log.Error(ex, "Log cull failed");
 
                 /*