Allow Penumbra to use long and arbitrary UTF8 paths.

Penumbra.String

@@ -1 +1 @@
-Subproject commit 2f396444c80ef2d2dca30b32c8f0ef787a928534
+Subproject commit 3276f37974e668b1e4769c176c114da17611b734

Penumbra/Interop/Loader/CreateFileWHook.cs

@@ -0,0 +1,172 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading;
+using Dalamud.Hooking;
+using Penumbra.String;
+using Penumbra.String.Classes;
+using Penumbra.String.Functions;
+
+namespace Penumbra.Interop.Loader;
+
+/// <summary>
+/// To allow XIV to load files of arbitrary path length,
+/// we use the fixed size buffers of their formats to only store pointers to the actual path instead.
+/// Then we translate the stored pointer to the path in CreateFileW, if the prefix matches.
+/// </summary>
+public unsafe class CreateFileWHook : IDisposable
+{
+    public const int RequiredSize = 28;
+
+    // The prefix is not valid for any actual path, so should never run into false-positives.
+    private const char Prefix     = ( char )( ( byte )'P' | ( ( '?' & 0x00FF ) << 8 ) );
+    private const int  BufferSize = Utf8GamePath.MaxGamePathLength;
+
+    [DllImport( "kernel32.dll" )]
+    private static extern nint LoadLibrary( string dllName );
+
+    [DllImport( "kernel32.dll" )]
+    private static extern nint GetProcAddress( nint hModule, string procName );
+
+    private delegate nint CreateFileWDelegate( char* fileName, uint access, uint shareMode, nint security, uint creation, uint flags, nint template );
+
+    private readonly Hook< CreateFileWDelegate > _createFileWHook;
+
+    /// <summary> Some storage to skip repeated allocations. </summary>
+    private readonly ThreadLocal< nint > _fileNameStorage = new(SetupStorage, true);
+
+    public CreateFileWHook()
+    {
+        var userApi           = LoadLibrary( "kernel32.dll" );
+        var createFileAddress = GetProcAddress( userApi, "CreateFileW" );
+        _createFileWHook = Hook< CreateFileWDelegate >.FromAddress( createFileAddress, CreateFileWDetour );
+    }
+
+    /// <remarks> Long paths in windows need to start with "\\?\", so we keep this static in the pointers. </remarks>
+    private static nint SetupStorage()
+    {
+        var ptr = ( char* )Marshal.AllocHGlobal( 2 * BufferSize );
+        ptr[ 0 ] = '\\';
+        ptr[ 1 ] = '\\';
+        ptr[ 2 ] = '?';
+        ptr[ 3 ] = '\\';
+        ptr[ 4 ] = '\0';
+        return ( nint )ptr;
+    }
+
+    public void Enable()
+        => _createFileWHook.Enable();
+
+    public void Disable()
+        => _createFileWHook.Disable();
+
+    public void Dispose()
+    {
+        _createFileWHook.Dispose();
+        foreach( var ptr in _fileNameStorage.Values )
+        {
+            Marshal.FreeHGlobal( ptr );
+        }
+    }
+
+    private nint CreateFileWDetour( char* fileName, uint access, uint shareMode, nint security, uint creation, uint flags, nint template )
+    {
+        // Translate data if prefix fits.
+        if( CheckPtr( fileName, out var name ) )
+        {
+            // Use static storage.
+            var ptr = WriteFileName( name );
+            Penumbra.Log.Verbose( $"Calling CreateFileWDetour with {ByteString.FromSpanUnsafe( name, false )}." );
+            return _createFileWHook.Original( ptr, access, shareMode, security, creation, flags, template );
+        }
+
+        return _createFileWHook.Original( fileName, access, shareMode, security, creation, flags, template );
+    }
+
+
+    /// <remarks>Write the UTF8-encoded byte string as UTF16 into the static buffers,
+    /// replacing any forward-slashes with back-slashes and adding a terminating null-wchar_t.</remarks>
+    private char* WriteFileName( ReadOnlySpan< byte > actualName )
+    {
+        var span    = new Span< char >( ( char* )_fileNameStorage.Value + 4, BufferSize - 4 );
+        var written = Encoding.UTF8.GetChars( actualName, span );
+        for( var i = 0; i < written; ++i )
+        {
+            if( span[ i ] == '/' )
+            {
+                span[ i ] = '\\';
+            }
+        }
+
+        span[ written ] = '\0';
+
+        return ( char* )_fileNameStorage.Value;
+    }
+
+
+    public static void WritePtr( char* buffer, byte* address, int length )
+    {
+        // Set the prefix, which is not valid for any actual path.
+        buffer[ 0 ] = Prefix;
+
+        var ptr = ( byte* )buffer;
+        var v   = ( ulong )address;
+        var l   = ( uint )length;
+
+        // Since the game calls wstrcpy without a length, we need to ensure
+        // that there is no wchar_t (i.e. 2 bytes) of 0-values before the end.
+        // Fill everything with 0xFF and use every second byte.
+        MemoryUtility.MemSet( ptr + 2, 0xFF, 23 );
+
+        // Write the byte pointer.
+        ptr[ 2 ]  = ( byte )( v >> 0 );
+        ptr[ 4 ]  = ( byte )( v >> 8 );
+        ptr[ 6 ]  = ( byte )( v >> 16 );
+        ptr[ 8 ]  = ( byte )( v >> 24 );
+        ptr[ 10 ] = ( byte )( v >> 32 );
+        ptr[ 12 ] = ( byte )( v >> 40 );
+        ptr[ 14 ] = ( byte )( v >> 48 );
+        ptr[ 16 ] = ( byte )( v >> 56 );
+
+        // Write the length.
+        ptr[ 18 ] = ( byte )( l >> 0 );
+        ptr[ 20 ] = ( byte )( l >> 8 );
+        ptr[ 22 ] = ( byte )( l >> 16 );
+        ptr[ 24 ] = ( byte )( l >> 24 );
+
+        ptr[ RequiredSize - 2 ] = 0;
+        ptr[ RequiredSize - 1 ] = 0;
+    }
+
+    private static bool CheckPtr( char* buffer, out ReadOnlySpan< byte > fileName )
+    {
+        if( buffer[ 0 ] is not Prefix )
+        {
+            fileName = ReadOnlySpan< byte >.Empty;
+            return false;
+        }
+
+        var ptr = ( byte* )buffer;
+
+        // Read the byte pointer.
+        var address = 0ul;
+        address |= ( ulong )ptr[ 2 ]  << 0;
+        address |= ( ulong )ptr[ 4 ]  << 8;
+        address |= ( ulong )ptr[ 6 ]  << 16;
+        address |= ( ulong )ptr[ 8 ]  << 24;
+        address |= ( ulong )ptr[ 10 ] << 32;
+        address |= ( ulong )ptr[ 12 ] << 40;
+        address |= ( ulong )ptr[ 14 ] << 48;
+        address |= ( ulong )ptr[ 16 ] << 56;
+
+        // Read the length.
+        var length = 0u;
+        length |= ( uint )ptr[ 18 ] << 0;
+        length |= ( uint )ptr[ 20 ] << 8;
+        length |= ( uint )ptr[ 22 ] << 16;
+        length |= ( uint )ptr[ 24 ] << 24;
+
+        fileName = new ReadOnlySpan< byte >( ( void* )address, ( int )length );
+        return true;
+    }
+}

Penumbra/Interop/Loader/ResourceLoader.Debug.cs

@@ -209,6 +209,7 @@ public unsafe partial class ResourceLoader
         for( var i = 0; i < _debugList.Count; ++i )
         {
             var data = _debugList.Values[ i ];
+            // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
             if( data.OriginalPath.Path == null )
             {
                 _debugList.RemoveAt( i-- );

Penumbra/Interop/Loader/ResourceLoader.Replacement.cs

@@ -1,8 +1,3 @@
-using System;
-using System.Diagnostics;
-using System.Linq;
-using System.Runtime.InteropServices;
-using System.Threading;
 using Dalamud.Hooking;
 using Dalamud.Utility.Signatures;
 using FFXIVClientStructs.FFXIV.Client.System.Resource;
@@ -13,6 +8,11 @@ using Penumbra.Interop.Structs;
 using Penumbra.String;
 using Penumbra.String.Classes;
 using Penumbra.Util;
+using System;
+using System.Diagnostics;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Threading;
 using FileMode = Penumbra.Interop.Structs.FileMode;
 using ResourceHandle = FFXIVClientStructs.FFXIV.Client.System.Resource.Handle.ResourceHandle;
 
@@ -20,6 +20,8 @@ namespace Penumbra.Interop.Loader;
 
 public unsafe partial class ResourceLoader
 {
+    private readonly CreateFileWHook _createFileWHook = new();
+
     // Resources can be obtained synchronously and asynchronously. We need to change behaviour in both cases.
     // Both work basically the same, so we can reduce the main work to one function used by both hooks.
 
@@ -173,6 +175,7 @@ public unsafe partial class ResourceLoader
             default:
                 break;
         }
+
         return DefaultResolver( path );
     }
 
@@ -249,27 +252,20 @@ public unsafe partial class ResourceLoader
         return ret;
     }
 
-    // Load the resource from a path on the users hard drives.
+    /// <summary> Load the resource from a path on the users hard drives. </summary>
+    /// <remarks> <see cref="CreateFileWHook" /> </remarks>
     private byte DefaultRootedResourceLoad( ByteString gamePath, ResourceManager* resourceManager,
         SeFileDescriptor* fileDescriptor, int priority, bool isSync )
     {
         // Specify that we are loading unpacked files from the drive.
-        // We need to copy the actual file path in UTF16 (Windows-Unicode) on two locations,
-        // but since we only allow ASCII in the game paths, this is just a matter of upcasting.
+        // We need to copy the actual file path in UTF16 (Windows-Unicode) on two locations.
         fileDescriptor->FileMode = FileMode.LoadUnpackedResource;
 
-        var fd = stackalloc byte[0x20 + 2 * gamePath.Length + 0x16];
-        fileDescriptor->FileDescriptor = fd;
-        var fdPtr = ( char* )( fd + 0x21 );
-        for( var i = 0; i < gamePath.Length; ++i )
-        {
-            var c = ( char )gamePath.Path[ i ];
-            ( &fileDescriptor->Utf16FileName )[ i ] = c;
-            fdPtr[ i ]                              = c;
-        }
-
-        ( &fileDescriptor->Utf16FileName )[ gamePath.Length ] = '\0';
-        fdPtr[ gamePath.Length ]                              = '\0';
+        // Ensure that the file descriptor has its wchar_t array on aligned boundary even if it has to be odd.
+        var fd = stackalloc char[0x11 + 0x0B + 14];
+        fileDescriptor->FileDescriptor = (byte*) fd + 1;
+        CreateFileWHook.WritePtr( fd + 0x11, gamePath.Path, gamePath.Length );
+        CreateFileWHook.WritePtr( &fileDescriptor->Utf16FileName, gamePath.Path, gamePath.Length );
 
         // Use the SE ReadFile function.
         var ret = ReadFile( resourceManager, fileDescriptor, priority, isSync );
@@ -287,6 +283,7 @@ public unsafe partial class ResourceLoader
     private void DisposeHooks()
     {
         DisableHooks();
+        _createFileWHook.Dispose();
         ReadSqPackHook.Dispose();
         GetResourceSyncHook.Dispose();
         GetResourceAsyncHook.Dispose();

Penumbra/Interop/Loader/ResourceLoader.cs

@@ -82,6 +82,7 @@ public unsafe partial class ResourceLoader : IDisposable
         }
 
         HooksEnabled = true;
+        _createFileWHook.Enable();
         ReadSqPackHook.Enable();
         GetResourceSyncHook.Enable();
         GetResourceAsyncHook.Enable();
@@ -96,6 +97,7 @@ public unsafe partial class ResourceLoader : IDisposable
         }
 
         HooksEnabled = false;
+        _createFileWHook.Disable();
         ReadSqPackHook.Disable();
         GetResourceSyncHook.Disable();
         GetResourceAsyncHook.Disable();

Penumbra/Mods/Mod.Creation.cs

@@ -156,9 +156,7 @@ public partial class Mod
     private static DirectoryInfo NewOptionDirectory( DirectoryInfo baseDir, string optionName )
         => new(Path.Combine( baseDir.FullName, ReplaceBadXivSymbols( optionName ) ));
 
-
-    // XIV can not deal with non-ascii symbols in a path,
-    // and the path must obviously be valid itself.
+    // Normalize for nicer names, and remove invalid symbols or invalid paths.
     public static string ReplaceBadXivSymbols( string s, string replacement = "_" )
     {
         if( s == "." )
@@ -174,7 +172,7 @@ public partial class Mod
         StringBuilder sb = new(s.Length);
         foreach( var c in s.Normalize( NormalizationForm.FormKC ) )
         {
-            if( c.IsInvalidAscii() || c.IsInvalidInPath() )
+            if( c.IsInvalidInPath() )
             {
                 sb.Append( replacement );
             }

Penumbra/UI/ConfigWindow.SettingsTab.cs

@@ -104,12 +104,6 @@ public partial class ConfigWindow
                 return ( "Path is not allowed to be a drive root. Please add a directory.", false );
             }
 
-            var symbol = '\0';
-            if( newName.Any( c => ( symbol = c ) > ( char )0x7F ) )
-            {
-                return ( $"Path contains invalid symbol {symbol}. Only ASCII is allowed.", false );
-            }
-
             var desktop = Environment.GetFolderPath( Environment.SpecialFolder.Desktop );
             if( IsSubPathOf( desktop, newName ) )
             {