Merge pull request #283 from kalilistic/sanitizer

Dalamud.Test/Dalamud.Test.csproj

@@ -59,6 +59,7 @@
     </ItemGroup>
     <ItemGroup>
         <Compile Include="LocalizationTests.cs" />
+        <Compile Include="Plugin\Sanitizer\SanitizerTests.cs" />
         <Compile Include="Properties\AssemblyInfo.cs" />
     </ItemGroup>
     <ItemGroup>

Dalamud.Test/Plugin/Sanitizer/SanitizerTests.cs

@@ -0,0 +1,32 @@
+// ReSharper disable StringLiteralTypo
+
+using System.Collections.Generic;
+using System.Linq;
+using Xunit;
+
+namespace Dalamud.Test.Plugin.Sanitizer {
+
+    public class SanitizerTests {
+        private global::Dalamud.Plugin.Sanitizer.Sanitizer sanitizer;
+
+        [Theory]
+        [InlineData( ClientLanguage.English, "Pixie Cotton Hood of Healing", "Pixie Cotton Hood of Healing" )]
+        [InlineData( ClientLanguage.Japanese, "アラガントームストーン:真理", "アラガントームストーン:真理" )]
+        [InlineData( ClientLanguage.German, "Anemos-Pan\x02\x16\x01\x03zer\x02\x16\x01\x03hand\x02\x16\x01\x03schu\x02\x16\x01\x03he des Drachenbluts", "Anemos-Panzerhandschuhe des Drachenbluts" )]
+        [InlineData( ClientLanguage.German, "Bienen-Spatha †", "Bienen-Spatha" )]
+        [InlineData( ClientLanguage.French, "Le Diademe\x02\x1D\x01\x03: terrains de chasse|Le Diademe\x02\x1D\x01\x03: terrains de chasse", "Le Diademe: terrains de chasse|Le Diademe: terrains de chasse" )]
+        [InlineData( ClientLanguage.French, "Cuir de bœuf", "Cuir de boeuf" )]
+        public void StringsAreSanitizedCorrectly(
+            ClientLanguage clientLanguage, string unsanitizedString, string sanitizedString)
+        {
+            var sanitizedStrings = new List<string> {unsanitizedString};
+            sanitizer = new global::Dalamud.Plugin.Sanitizer.Sanitizer(clientLanguage);
+            Assert.Equal(sanitizedString, sanitizer.Sanitize(unsanitizedString));
+            Assert.Equal(sanitizedString, sanitizer.Sanitize(sanitizedStrings).First());
+            
+            sanitizer = new global::Dalamud.Plugin.Sanitizer.Sanitizer(ClientLanguage.English);
+            Assert.Equal(sanitizedString, sanitizer.Sanitize(unsanitizedString, clientLanguage));
+            Assert.Equal(sanitizedString, sanitizer.Sanitize(sanitizedStrings, clientLanguage).First());
+        }
+    }
+}

Dalamud.sln.DotSettings

@@ -47,4 +47,5 @@
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EMigrateBlankLinesAroundFieldToBlankLinesAroundProperty/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EMigrateThisQualifierSettings/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002EFormat_002ESettingsUpgrade_002EAlignmentTabFillStyleMigration/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/UserDictionary/Words/=Dalamud/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Dalamud/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=unsanitized/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

Dalamud/Data/DataManager.cs

@@ -19,6 +19,11 @@ namespace Dalamud.Data
     /// </summary>
     public class DataManager : IDisposable
     {
+        /// <summary>
+        /// The current game client language.
+        /// </summary>
+        internal ClientLanguage Language;
+
         private const string IconFileFormat = "ui/icon/{0:D3}000/{1}{2:D6}.tex";
 
         /// <summary>
@@ -26,8 +31,6 @@ namespace Dalamud.Data
         /// </summary>
         private Lumina.GameData gameData;
 
-        private ClientLanguage language;
-
         private Thread luminaResourceThread;
 
         /// <summary>
@@ -39,7 +42,7 @@ namespace Dalamud.Data
             // Set up default values so plugins do not null-reference when data is being loaded.
             this.ServerOpCodes = new ReadOnlyDictionary<string, ushort>(new Dictionary<string, ushort>());
 
-            this.language = language;
+            this.Language = language;
         }
 
         /// <summary>
@@ -94,14 +97,14 @@ namespace Dalamud.Data
                     PanicOnSheetChecksumMismatch = false,
 #endif
 
-                    DefaultExcelLanguage = this.language switch {
-                        ClientLanguage.Japanese => Language.Japanese,
-                        ClientLanguage.English => Language.English,
-                        ClientLanguage.German => Language.German,
-                        ClientLanguage.French => Language.French,
+                    DefaultExcelLanguage = this.Language switch {
+                        ClientLanguage.Japanese => Lumina.Data.Language.Japanese,
+                        ClientLanguage.English => Lumina.Data.Language.English,
+                        ClientLanguage.German => Lumina.Data.Language.German,
+                        ClientLanguage.French => Lumina.Data.Language.French,
                         _ => throw new ArgumentOutOfRangeException(
-                                 nameof(this.language),
-                                 "Unknown Language: " + this.language),
+                                 nameof(this.Language),
+                                 "Unknown Language: " + this.Language),
                     },
                 };
 
@@ -156,11 +159,11 @@ namespace Dalamud.Data
         public ExcelSheet<T> GetExcelSheet<T>(ClientLanguage language) where T : ExcelRow
         {
             var lang = language switch {
-                ClientLanguage.Japanese => Language.Japanese,
-                ClientLanguage.English => Language.English,
-                ClientLanguage.German => Language.German,
-                ClientLanguage.French => Language.French,
-                _ => throw new ArgumentOutOfRangeException(nameof(this.language), "Unknown Language: " + this.language)
+                ClientLanguage.Japanese => Lumina.Data.Language.Japanese,
+                ClientLanguage.English => Lumina.Data.Language.English,
+                ClientLanguage.German => Lumina.Data.Language.German,
+                ClientLanguage.French => Lumina.Data.Language.French,
+                _ => throw new ArgumentOutOfRangeException(nameof(this.Language), "Unknown Language: " + this.Language)
             };
             return this.Excel.GetSheet<T>(lang);
         }
@@ -207,7 +210,7 @@ namespace Dalamud.Data
         /// <returns>The <see cref="TexFile"/> containing the icon.</returns>
         public TexFile GetIcon(int iconId)
         {
-            return this.GetIcon(this.language, iconId);
+            return this.GetIcon(this.Language, iconId);
         }
 
         /// <summary>
@@ -223,7 +226,7 @@ namespace Dalamud.Data
                 ClientLanguage.English => "en/",
                 ClientLanguage.German => "de/",
                 ClientLanguage.French => "fr/",
-                _ => throw new ArgumentOutOfRangeException(nameof(this.language), "Unknown Language: " + this.language)
+                _ => throw new ArgumentOutOfRangeException(nameof(this.Language), "Unknown Language: " + this.Language)
             };
 
             return this.GetIcon(type, iconId);

Dalamud/Plugin/DalamudPluginInterface.cs

@@ -13,6 +13,7 @@ using Dalamud.Game.ClientState;
 using Dalamud.Game.Command;
 using Dalamud.Game.Internal;
 using Dalamud.Interface;
+using Dalamud.Plugin.Sanitizer;
 
 namespace Dalamud.Plugin
 {
@@ -48,6 +49,7 @@ namespace Dalamud.Plugin
             this.pluginName = pluginName;
             this.configs = configs;
 
+            this.Sanitizer = new Sanitizer.Sanitizer(this.Data.Language);
             this.UiLanguage = this.dalamud.Configuration.LanguageOverride;
             dalamud.LocalizationManager.OnLocalizationChanged += this.OnLocalizationChanged;
         }
@@ -132,6 +134,11 @@ namespace Dalamud.Plugin
         /// </summary>
         public string UiLanguage { get; private set; }
 
+        /// <summary>
+        /// Gets serializer class with functions to remove special characters from strings.
+        /// </summary>
+        public ISanitizer Sanitizer { get; }
+
         /// <summary>
         /// Gets the action that should be executed when any plugin sends a message.
         /// </summary>

Dalamud/Plugin/Sanitizer/ISanitizer.cs

@@ -0,0 +1,40 @@
+using System.Collections.Generic;
+
+namespace Dalamud.Plugin.Sanitizer
+{
+    /// <summary>
+    /// Sanitize strings to remove soft hyphens and other special characters.
+    /// </summary>
+    public interface ISanitizer
+    {
+        /// <summary>
+        /// Creates a sanitized string using current clientLanguage.
+        /// </summary>
+        /// <param name="unsanitizedString">An unsanitized string to sanitize.</param>
+        /// <returns>A sanitized string.</returns>
+        string Sanitize(string unsanitizedString);
+
+        /// <summary>
+        /// Creates a sanitized string using request clientLanguage.
+        /// </summary>
+        /// <param name="unsanitizedString">An unsanitized string to sanitize.</param>
+        /// <param name="clientLanguage">Target language for sanitized strings.</param>
+        /// <returns>A sanitized string.</returns>
+        string Sanitize(string unsanitizedString, ClientLanguage clientLanguage);
+
+        /// <summary>
+        /// Creates a list of sanitized strings using current clientLanguage.
+        /// </summary>
+        /// <param name="unsanitizedStrings">List of unsanitized string to sanitize.</param>
+        /// <returns>A list of sanitized strings.</returns>
+        IEnumerable<string> Sanitize(IEnumerable<string> unsanitizedStrings);
+
+        /// <summary>
+        /// Creates a list of sanitized strings using requested clientLanguage.
+        /// </summary>
+        /// <param name="unsanitizedStrings">List of unsanitized string to sanitize.</param>
+        /// <param name="clientLanguage">Target language for sanitized strings.</param>
+        /// <returns>A list of sanitized strings.</returns>
+        IEnumerable<string> Sanitize(IEnumerable<string> unsanitizedStrings, ClientLanguage clientLanguage);
+    }
+}

Dalamud/Plugin/Sanitizer/Sanitizer.cs

@@ -0,0 +1,131 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Dalamud.Plugin.Sanitizer
+{
+    /// <summary>
+    /// Sanitize strings to remove soft hyphens and other special characters.
+    /// </summary>
+    public class Sanitizer : ISanitizer
+    {
+        private static readonly Dictionary<string, string> DESanitizationDict = new Dictionary<string, string>
+        {
+            { "\u0020\u2020", string.Empty }, // dagger
+        };
+
+        private static readonly Dictionary<string, string> FRSanitizationDict = new Dictionary<string, string>
+        {
+            { "\u0153", "\u006F\u0065" }, // ligature oe
+        };
+
+        private readonly ClientLanguage defaultClientLanguage;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Sanitizer"/> class.
+        /// </summary>
+        /// <param name="defaultClientLanguage">Default clientLanguage for sanitizing strings.</param>
+        public Sanitizer(ClientLanguage defaultClientLanguage)
+        {
+            this.defaultClientLanguage = defaultClientLanguage;
+        }
+
+        /// <summary>
+        /// Creates a sanitized string using current clientLanguage.
+        /// </summary>
+        /// <param name="unsanitizedString">An unsanitized string to sanitize.</param>
+        /// <returns>A sanitized string.</returns>
+        public string Sanitize(string unsanitizedString)
+        {
+            return SanitizeByLanguage(unsanitizedString, this.defaultClientLanguage);
+        }
+
+        /// <summary>
+        /// Creates a sanitized string using request clientLanguage.
+        /// </summary>
+        /// <param name="unsanitizedString">An unsanitized string to sanitize.</param>
+        /// <param name="clientLanguage">Target language for sanitized strings.</param>
+        /// <returns>A sanitized string.</returns>
+        public string Sanitize(string unsanitizedString, ClientLanguage clientLanguage)
+        {
+            return SanitizeByLanguage(unsanitizedString, clientLanguage);
+        }
+
+        /// <summary>
+        /// Creates a list of sanitized strings using current clientLanguage.
+        /// </summary>
+        /// <param name="unsanitizedStrings">List of unsanitized string to sanitize.</param>
+        /// <returns>A list of sanitized strings.</returns>
+        public IEnumerable<string> Sanitize(IEnumerable<string> unsanitizedStrings)
+        {
+            return SanitizeByLanguage(unsanitizedStrings, this.defaultClientLanguage);
+        }
+
+        /// <summary>
+        /// Creates a list of sanitized strings using requested clientLanguage.
+        /// </summary>
+        /// <param name="unsanitizedStrings">List of unsanitized string to sanitize.</param>
+        /// <param name="clientLanguage">Target language for sanitized strings.</param>
+        /// <returns>A list of sanitized strings.</returns>
+        public IEnumerable<string> Sanitize(IEnumerable<string> unsanitizedStrings, ClientLanguage clientLanguage)
+        {
+            return SanitizeByLanguage(unsanitizedStrings, clientLanguage);
+        }
+
+        private static string SanitizeByLanguage(string unsanitizedString, ClientLanguage clientLanguage)
+        {
+            var sanitizedString = FilterUnprintableCharacters(unsanitizedString);
+            switch (clientLanguage)
+            {
+                case ClientLanguage.Japanese:
+                case ClientLanguage.English:
+                    return sanitizedString;
+                case ClientLanguage.German:
+                    return FilterByDict(sanitizedString, DESanitizationDict);
+                case ClientLanguage.French:
+                    return FilterByDict(sanitizedString, FRSanitizationDict);
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(clientLanguage), clientLanguage, null);
+            }
+        }
+
+        private static IEnumerable<string> SanitizeByLanguage(
+            IEnumerable<string> unsanitizedStrings, ClientLanguage clientLanguage)
+        {
+            var sanitizedStrings = new List<string>();
+            switch (clientLanguage)
+            {
+                case ClientLanguage.Japanese:
+                case ClientLanguage.English:
+                    sanitizedStrings.AddRange(unsanitizedStrings.Select(FilterUnprintableCharacters));
+                    return sanitizedStrings;
+                case ClientLanguage.German:
+                    sanitizedStrings.AddRange(
+                        unsanitizedStrings.Select(
+                            unsanitizedString =>
+                                FilterByDict(FilterUnprintableCharacters(unsanitizedString), DESanitizationDict)));
+                    return sanitizedStrings;
+                case ClientLanguage.French:
+                    sanitizedStrings.AddRange(
+                        unsanitizedStrings.Select(
+                            unsanitizedString =>
+                                FilterByDict(FilterUnprintableCharacters(unsanitizedString), FRSanitizationDict)));
+                    return sanitizedStrings;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(clientLanguage), clientLanguage, null);
+            }
+        }
+
+        private static string FilterUnprintableCharacters(string str)
+        {
+            return new string(str?.Where(ch => ch >= 0x20).ToArray());
+        }
+
+        private static string FilterByDict(string str, Dictionary<string, string> dict)
+        {
+            return dict.Aggregate(
+                str, (current, kvp) =>
+                    current.Replace(kvp.Key, kvp.Value));
+        }
+    }
+}