Limit console log lines held in memory (#1683)

* Add AG.Collections.RollingList

* Use RollingList for logs + Adaption changes

* Create Dalamud.Utility.ThrowHelper

* Create Dalamud.Utility.RollingList

* ConsoleWindow: Remove dependency

* Remove NuGet Dependency

* Add Log Lines Limit configuration

* Use Log Lines Limit configuration and handle changes

* Make log lines limit configurable

Dalamud/Configuration/Internal/DalamudConfiguration.cs

@@ -215,6 +215,11 @@ internal sealed class DalamudConfiguration : IServiceType, IDisposable
     /// </summary>
     public bool LogOpenAtStartup { get; set; }
 
+    /// <summary>
+    /// Gets or sets the number of lines to keep for the Dalamud Console window.
+    /// </summary>
+    public int LogLinesLimit { get; set; } = 10000;
+
     /// <summary>
     /// Gets or sets a value indicating whether or not the dev bar should open at startup.
     /// </summary>

Dalamud/Interface/Internal/Windows/ConsoleWindow.cs

@@ -6,6 +6,7 @@ using System.Numerics;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Text.RegularExpressions;
+using System.Threading;
 
 using Dalamud.Configuration.Internal;
 using Dalamud.Game.Command;
@@ -28,7 +29,11 @@ namespace Dalamud.Interface.Internal.Windows;
 /// </summary>
 internal class ConsoleWindow : Window, IDisposable
 {
-    private readonly List<LogEntry> logText = new();
+    private const int LogLinesMinimum = 100;
+    private const int LogLinesMaximum = 1000000;
+    
+    private readonly RollingList<LogEntry> logText;
+    private volatile int newRolledLines;
     private readonly object renderLock = new();
 
     private readonly List<string> history = new();
@@ -42,12 +47,14 @@ internal class ConsoleWindow : Window, IDisposable
     private string pluginFilter = string.Empty;
 
     private bool filterShowUncaughtExceptions;
+    private bool settingsPopupWasOpen;
     private bool showFilterToolbar;
     private bool clearLog;
     private bool copyLog;
     private bool copyMode;
     private bool killGameArmed;
     private bool autoScroll;
+    private int logLinesLimit;
     private bool autoOpen;
     private bool regexError;
 
@@ -74,9 +81,17 @@ internal class ConsoleWindow : Window, IDisposable
         };
 
         this.RespectCloseHotkey = false;
+
+        this.logLinesLimit = configuration.LogLinesLimit;
+
+        var limit = Math.Max(LogLinesMinimum, this.logLinesLimit);
+        this.logText = new(limit);
+        this.FilteredLogEntries = new(limit);
+
+        configuration.DalamudConfigurationSaved += this.OnDalamudConfigurationSaved;
     }
 
-    private List<LogEntry> FilteredLogEntries { get; set; } = new();
+    private RollingList<LogEntry> FilteredLogEntries { get; set; }
 
     /// <inheritdoc/>
     public override void OnOpen()
@@ -91,6 +106,7 @@ internal class ConsoleWindow : Window, IDisposable
     public void Dispose()
     {
         SerilogEventSink.Instance.LogLine -= this.OnLogLine;
+        Service<DalamudConfiguration>.Get().DalamudConfigurationSaved -= this.OnDalamudConfigurationSaved;
     }
 
     /// <summary>
@@ -180,6 +196,9 @@ internal class ConsoleWindow : Window, IDisposable
         var dividerOffset = ImGui.CalcTextSize("00:00:00.000 | AAA ").X + (ImGui.CalcTextSize(" ").X / 2);
         var cursorLogLine = ImGui.CalcTextSize("00:00:00.000 | AAA | ").X;
 
+        var lastLinePosY = 0.0f;
+        var logLineHeight = 0.0f;
+
         lock (this.renderLock)
         {
             clipper.Begin(this.FilteredLogEntries.Count);
@@ -187,7 +206,8 @@ internal class ConsoleWindow : Window, IDisposable
             {
                 for (var i = clipper.DisplayStart; i < clipper.DisplayEnd; i++)
                 {
-                    var line = this.FilteredLogEntries[i];
+                    var index = Math.Max(i - this.newRolledLines, 0); // Prevents flicker effect. Also workaround to avoid negative indexes.
+                    var line = this.FilteredLogEntries[index];
 
                     if (!line.IsMultiline && !this.copyLog)
                         ImGui.Separator();
@@ -228,6 +248,10 @@ internal class ConsoleWindow : Window, IDisposable
 
                     ImGui.SetCursorPosX(cursorLogLine);
                     ImGui.TextUnformatted(line.Line);
+
+                    var currentLinePosY = ImGui.GetCursorPosY();
+                    logLineHeight = currentLinePosY - lastLinePosY;
+                    lastLinePosY = currentLinePosY;
                 }
             }
 
@@ -239,6 +263,12 @@ internal class ConsoleWindow : Window, IDisposable
 
         ImGui.PopStyleVar();
 
+        var newRolledLinesCount = Interlocked.Exchange(ref this.newRolledLines, 0);
+        if (!this.autoScroll || ImGui.GetScrollY() < ImGui.GetScrollMaxY())
+        {
+            ImGui.SetScrollY(ImGui.GetScrollY() - (logLineHeight * newRolledLinesCount));
+        }
+
         if (this.autoScroll && ImGui.GetScrollY() >= ImGui.GetScrollMaxY())
         {
             ImGui.SetScrollHereY(1.0f);
@@ -363,21 +393,21 @@ internal class ConsoleWindow : Window, IDisposable
 
         ImGui.SameLine();
 
-        this.autoScroll = configuration.LogAutoScroll;
-        if (this.DrawToggleButtonWithTooltip("auto_scroll", "Auto-scroll", FontAwesomeIcon.Sync, ref this.autoScroll))
+        var settingsPopup = ImGui.BeginPopup("##console_settings");
+        if (settingsPopup)
         {
-            configuration.LogAutoScroll = !configuration.LogAutoScroll;
-            configuration.QueueSave();
+            this.DrawSettingsPopup(configuration);
+            ImGui.EndPopup();
+        }
+        else if (this.settingsPopupWasOpen)
+        {
+            // Prevent side effects in case Apply wasn't clicked
+            this.logLinesLimit = configuration.LogLinesLimit;
         }
 
-        ImGui.SameLine();
+        this.settingsPopupWasOpen = settingsPopup;
 
-        this.autoOpen = configuration.LogOpenAtStartup;
-        if (this.DrawToggleButtonWithTooltip("auto_open", "Open at startup", FontAwesomeIcon.WindowRestore, ref this.autoOpen))
-        {
-            configuration.LogOpenAtStartup = !configuration.LogOpenAtStartup;
-            configuration.QueueSave();
-        }
+        if (this.DrawToggleButtonWithTooltip("show_settings", "Show settings", FontAwesomeIcon.List, ref settingsPopup)) ImGui.OpenPopup("##console_settings");
 
         ImGui.SameLine();
 
@@ -447,6 +477,33 @@ internal class ConsoleWindow : Window, IDisposable
         }
     }
 
+    private void DrawSettingsPopup(DalamudConfiguration configuration)
+    {
+        if (ImGui.Checkbox("Open at startup", ref this.autoOpen))
+        {
+            configuration.LogOpenAtStartup = this.autoOpen;
+            configuration.QueueSave();
+        }
+
+        if (ImGui.Checkbox("Auto-scroll", ref this.autoScroll))
+        {
+            configuration.LogAutoScroll = this.autoScroll;
+            configuration.QueueSave();
+        }
+
+        ImGui.TextUnformatted("Logs buffer");
+        ImGui.SliderInt("lines", ref this.logLinesLimit, LogLinesMinimum, LogLinesMaximum);
+        if (ImGui.Button("Apply"))
+        {
+            this.logLinesLimit = Math.Max(LogLinesMinimum, this.logLinesLimit);
+
+            configuration.LogLinesLimit = this.logLinesLimit;
+            configuration.QueueSave();
+
+            ImGui.CloseCurrentPopup();
+        }
+    }
+
     private void DrawFilterToolbar()
     {
         if (!this.showFilterToolbar) return;
@@ -686,8 +743,12 @@ internal class ConsoleWindow : Window, IDisposable
 
         this.logText.Add(entry);
 
+        var avoidScroll = this.FilteredLogEntries.Count == this.FilteredLogEntries.Size;
         if (this.IsFilterApplicable(entry))
+        {
             this.FilteredLogEntries.Add(entry);
+            if (avoidScroll) Interlocked.Increment(ref this.newRolledLines);
+        }
     }
 
     private bool IsFilterApplicable(LogEntry entry)
@@ -740,7 +801,7 @@ internal class ConsoleWindow : Window, IDisposable
         lock (this.renderLock)
         {
             this.regexError = false;
-            this.FilteredLogEntries = this.logText.Where(this.IsFilterApplicable).ToList();
+            this.FilteredLogEntries = new RollingList<LogEntry>(this.logText.Where(this.IsFilterApplicable), Math.Max(LogLinesMinimum, this.logLinesLimit));
         }
     }
 
@@ -789,6 +850,14 @@ internal class ConsoleWindow : Window, IDisposable
         return result;
     }
 
+    private void OnDalamudConfigurationSaved(DalamudConfiguration dalamudConfiguration)
+    {
+        this.logLinesLimit = dalamudConfiguration.LogLinesLimit;
+        var limit = Math.Max(LogLinesMinimum, this.logLinesLimit);
+        this.logText.Size = limit;
+        this.FilteredLogEntries.Size = limit;
+    }
+
     private class LogEntry
     {
         public string Line { get; init; } = string.Empty;

Dalamud/Utility/RollingList.cs

@@ -0,0 +1,234 @@
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Runtime.CompilerServices;
+
+namespace Dalamud.Utility
+{
+    /// <summary>
+    /// A list with limited capacity holding items of type <typeparamref name="T"/>.
+    /// Adding further items will result in the list rolling over.
+    /// </summary>
+    /// <typeparam name="T">Item type.</typeparam>
+    /// <remarks>
+    /// Implemented as a circular list using a <see cref="List{T}"/> internally.
+    /// Insertions and Removals are not supported.
+    /// Not thread-safe.
+    /// </remarks>
+    internal class RollingList<T> : IList<T>
+    {
+        private List<T> items;
+        private int size;
+        private int firstIndex;
+
+        /// <summary>Initializes a new instance of the <see cref="RollingList{T}"/> class.</summary>
+        /// <param name="size"><see cref="RollingList{T}"/> size.</param>
+        /// <param name="capacity">Internal <see cref="List{T}"/> initial capacity.</param>
+        public RollingList(int size, int capacity)
+        {
+            ThrowHelper.ThrowArgumentOutOfRangeExceptionIfLessThan(nameof(size), size, 0);
+            capacity = Math.Min(capacity, size);
+            this.size = size;
+            this.items = new List<T>(capacity);
+        }
+
+        /// <summary>Initializes a new instance of the <see cref="RollingList{T}"/> class.</summary>
+        /// <param name="size"><see cref="RollingList{T}"/> size.</param>
+        public RollingList(int size)
+        {
+            ThrowHelper.ThrowArgumentOutOfRangeExceptionIfLessThan(nameof(size), size, 0);
+            this.size = size;
+            this.items = new();
+        }
+
+        /// <summary>Initializes a new instance of the <see cref="RollingList{T}"/> class.</summary>
+        /// <param name="items">Collection where elements are copied from.</param>
+        /// <param name="size"><see cref="RollingList{T}"/> size.</param>
+        public RollingList(IEnumerable<T> items, int size)
+        {
+            if (!items.TryGetNonEnumeratedCount(out var capacity)) capacity = 4;
+            capacity = Math.Min(capacity, size);
+            this.size = size;
+            this.items = new List<T>(capacity);
+            this.AddRange(items);
+        }
+
+        /// <summary>Initializes a new instance of the <see cref="RollingList{T}"/> class.</summary>
+        /// <param name="items">Collection where elements are copied from.</param>
+        /// <param name="size"><see cref="RollingList{T}"/> size.</param>
+        /// <param name="capacity">Internal <see cref="List{T}"/> initial capacity.</param>
+        public RollingList(IEnumerable<T> items, int size, int capacity)
+        {
+            if (items.TryGetNonEnumeratedCount(out var count) && count > capacity) capacity = count;
+            capacity = Math.Min(capacity, size);
+            this.size = size;
+            this.items = new List<T>(capacity);
+            this.AddRange(items);
+        }
+
+        /// <summary>Gets item count.</summary>
+        public int Count => this.items.Count;
+
+        /// <summary>Gets or sets the internal list capacity.</summary>
+        public int Capacity
+        {
+            get => this.items.Capacity;
+            set => this.items.Capacity = Math.Min(value, this.size);
+        }
+
+        /// <summary>Gets or sets rolling list size.</summary>
+        public int Size
+        {
+            get => this.size;
+            set
+            {
+                if (value == this.size) return;
+                if (value > this.size)
+                {
+                    if (this.firstIndex > 0)
+                    {
+                        this.items = new List<T>(this);
+                        this.firstIndex = 0;
+                    }
+                }
+                else // value < this._size
+                {
+                    ThrowHelper.ThrowArgumentOutOfRangeExceptionIfLessThan(nameof(value), value, 0);
+                    if (value < this.Count)
+                    {
+                        this.items = new List<T>(this.TakeLast(value));
+                        this.firstIndex = 0;
+                    }
+                }
+
+                this.size = value;
+            }
+        }
+
+        /// <summary>Gets a value indicating whether the item is read only.</summary>
+        public bool IsReadOnly => false;
+
+        /// <summary>Gets or sets an item by index.</summary>
+        /// <param name="index">Item index.</param>
+        /// <returns>Item at specified index.</returns>
+        public T this[int index]
+        {
+            get
+            {
+                ThrowHelper.ThrowArgumentOutOfRangeExceptionIfGreaterThanOrEqual(nameof(index), index, this.Count);
+                ThrowHelper.ThrowArgumentOutOfRangeExceptionIfLessThan(nameof(index), index, 0);
+                return this.items[this.GetRealIndex(index)];
+            }
+
+            set
+            {
+                ThrowHelper.ThrowArgumentOutOfRangeExceptionIfGreaterThanOrEqual(nameof(index), index, this.Count);
+                ThrowHelper.ThrowArgumentOutOfRangeExceptionIfLessThan(nameof(index), index, 0);
+                this.items[this.GetRealIndex(index)] = value;
+            }
+        }
+
+        /// <summary>Adds an item to this <see cref="RollingList{T}"/>.</summary>
+        /// <param name="item">Item to add.</param>
+        public void Add(T item)
+        {
+            if (this.size == 0) return;
+            if (this.items.Count >= this.size)
+            {
+                this.items[this.firstIndex] = item;
+                this.firstIndex = (this.firstIndex + 1) % this.size;
+            }
+            else
+            {
+                if (this.items.Count == this.items.Capacity)
+                {
+                    // Manual list capacity resize
+                    var newCapacity = Math.Max(Math.Min(this.size, this.items.Capacity * 2), this.items.Capacity);
+                    this.items.Capacity = newCapacity;
+                }
+
+                this.items.Add(item);
+            }
+
+            Debug.Assert(this.items.Count <= this.size, "Item count should be less than Size");
+        }
+
+        /// <summary>Add items to this <see cref="RollingList{T}"/>.</summary>
+        /// <param name="items">Items to add.</param>
+        public void AddRange(IEnumerable<T> items)
+        {
+            if (this.size == 0) return;
+            foreach (var item in items) this.Add(item);
+        }
+
+        /// <summary>Removes all elements from the <see cref="RollingList{T}"/></summary>
+        public void Clear()
+        {
+            this.items.Clear();
+            this.firstIndex = 0;
+        }
+
+        /// <summary>Find the index of a specific item.</summary>
+        /// <param name="item">item to find.</param>
+        /// <returns>Index where <paramref name="item"/> is found. -1 if not found.</returns>
+        public int IndexOf(T item)
+        {
+            var index = this.items.IndexOf(item);
+            if (index == -1) return -1;
+            return this.GetVirtualIndex(index);
+        }
+
+        /// <summary>Not supported.</summary>
+        [SuppressMessage("Documentation Rules", "SA1611", Justification = "Not supported")]
+        void IList<T>.Insert(int index, T item) => throw new NotSupportedException();
+
+        /// <summary>Not supported.</summary>
+        [SuppressMessage("Documentation Rules", "SA1611", Justification = "Not supported")]
+        void IList<T>.RemoveAt(int index) => throw new NotSupportedException();
+
+        /// <summary>Find wether an item exists.</summary>
+        /// <param name="item">item to find.</param>
+        /// <returns>Wether <paramref name="item"/> is found.</returns>
+        public bool Contains(T item) => this.items.Contains(item);
+
+        /// <summary>Copies the content of this list into an array.</summary>
+        /// <param name="array">Array to copy into.</param>
+        /// <param name="arrayIndex"><paramref name="array"/> index to start coping into.</param>
+        public void CopyTo(T[] array, int arrayIndex)
+        {
+            ThrowHelper.ThrowArgumentOutOfRangeExceptionIfLessThan(nameof(arrayIndex), arrayIndex, 0);
+            if (array.Length - arrayIndex < this.Count) ThrowHelper.ThrowArgumentException("Not enough space");
+            for (var index = 0; index < this.Count; index++)
+            {
+                array[arrayIndex++] = this[index];
+            }
+        }
+
+        /// <summary>Not supported.</summary>
+        [SuppressMessage("Documentation Rules", "SA1611", Justification = "Not supported")]
+        [SuppressMessage("Documentation Rules", "SA1615", Justification = "Not supported")]
+        bool ICollection<T>.Remove(T item) => throw new NotSupportedException();
+
+        /// <summary>Gets an enumerator for this <see cref="RollingList{T}"/>.</summary>
+        /// <returns><see cref="RollingList{T}"/> enumerator.</returns>
+        public IEnumerator<T> GetEnumerator()
+        {
+            for (var index = 0; index < this.items.Count; index++)
+            {
+                yield return this.items[this.GetRealIndex(index)];
+            }
+        }
+
+        /// <summary>Gets an enumerator for this <see cref="RollingList{T}"/>.</summary>
+        /// <returns><see cref="RollingList{T}"/> enumerator.</returns>
+        IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator();
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private int GetRealIndex(int index) => this.size > 0 ? (index + this.firstIndex) % this.size : 0;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private int GetVirtualIndex(int index) => this.size > 0 ? (this.size + index - this.firstIndex) % this.size : 0;
+    }
+}

Dalamud/Utility/ThrowHelper.cs

@@ -0,0 +1,52 @@
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Dalamud.Utility
+{
+    /// <summary>Helper methods for throwing exceptions.</summary>
+    internal static class ThrowHelper
+    {
+        /// <summary>Throws a <see cref="ArgumentException"/> with a specified <paramref name="message"/>.</summary>
+        /// <param name="message">Message for the exception.</param>
+        /// <exception cref="ArgumentException">Thrown by this method.</exception>
+        [DoesNotReturn]
+        public static void ThrowArgumentException(string message) => throw new ArgumentException(message);
+
+        /// <summary>Throws a <see cref="ArgumentOutOfRangeException"/> with a specified <paramref name="message"/> for a specified <paramref name="paramName"/>.</summary>
+        /// <param name="paramName">Parameter name.</param>
+        /// <param name="message">Message for the exception.</param>
+        /// <exception cref="ArgumentOutOfRangeException">Thrown by this method.</exception>
+        [DoesNotReturn]
+        public static void ThrowArgumentOutOfRangeException(string paramName, string message) => throw new ArgumentOutOfRangeException(paramName, message);
+
+        /// <summary>Throws a <see cref="ArgumentOutOfRangeException"/> if the specified <paramref name="value"/> is less than <paramref name="comparand"/>.</summary>
+        /// <typeparam name="T"><see cref="IComparable{T}"/> value type.</typeparam>
+        /// <param name="paramName">Parameter name.</param>
+        /// <param name="value">Value to compare from.</param>
+        /// <param name="comparand">Value to compare with.</param>
+        /// <exception cref="ArgumentOutOfRangeException">Thrown by this method if <paramref name="value"/> is less than <paramref name="comparand"/>.</exception>
+        public static void ThrowArgumentOutOfRangeExceptionIfLessThan<T>(string paramName, T value, T comparand) where T : IComparable<T>
+        {
+#if NET8_0_OR_GREATER
+            ArgumentOutOfRangeException.ThrowIfLessThan(value, comparand);
+#else
+            if (Comparer<T>.Default.Compare(value, comparand) <= -1) ThrowArgumentOutOfRangeException(paramName, $"{paramName} must be greater than or equal {comparand}");
+#endif
+        }
+
+        /// <summary>Throws a <see cref="ArgumentOutOfRangeException"/> if the specified <paramref name="value"/> is greater than or equal to <paramref name="comparand"/>.</summary>
+        /// <typeparam name="T"><see cref="IComparable{T}"/> value type.</typeparam>
+        /// <param name="paramName">Parameter name.</param>
+        /// <param name="value">Value to compare from.</param>
+        /// <param name="comparand">Value to compare with.</param>
+        /// <exception cref="ArgumentOutOfRangeException">Thrown by this method if <paramref name="value"/> is greater than or equal to<paramref name="comparand"/>.</exception>
+        public static void ThrowArgumentOutOfRangeExceptionIfGreaterThanOrEqual<T>(string paramName, T value, T comparand) where T : IComparable<T>
+        {
+#if NET8_0_OR_GREATER
+            ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(value, comparand);
+#else
+            if (Comparer<T>.Default.Compare(value, comparand) >= 0) ThrowArgumentOutOfRangeException(paramName, $"{paramName} must be less than {comparand}");
+#endif
+        }
+    }
+}