From a2b62a8b6a65e92424af78afe9baac752c436635 Mon Sep 17 00:00:00 2001 From: Ottermandias Date: Thu, 23 Feb 2023 14:48:46 +0100 Subject: [PATCH] Some formatting and naming changes, splitting files and some minor improvements. --- Penumbra.GameData/Data/DisassembledShader.cs | 229 +++--- .../Files/MtrlFile.ColorDyeSet.cs | 90 +++ Penumbra.GameData/Files/MtrlFile.ColorSet.cs | 135 ++++ Penumbra.GameData/Files/MtrlFile.Write.cs | 1 - Penumbra.GameData/Files/MtrlFile.cs | 321 ++------ Penumbra.GameData/Files/ShpkFile.Shader.cs | 220 +++++ .../Files/ShpkFile.StringPool.cs | 79 ++ Penumbra.GameData/Files/ShpkFile.Write.cs | 57 +- Penumbra.GameData/Files/ShpkFile.cs | 763 +++++------------- Penumbra.GameData/UtilityFunctions.cs | 13 + .../UI/Classes/ModEditWindow.FileEditor.cs | 7 +- .../UI/Classes/ModEditWindow.Materials.cs | 1 + .../Classes/ModEditWindow.ShaderPackages.cs | 9 +- 13 files changed, 928 insertions(+), 997 deletions(-) create mode 100644 Penumbra.GameData/Files/MtrlFile.ColorDyeSet.cs create mode 100644 Penumbra.GameData/Files/MtrlFile.ColorSet.cs create mode 100644 Penumbra.GameData/Files/ShpkFile.Shader.cs create mode 100644 Penumbra.GameData/Files/ShpkFile.StringPool.cs create mode 100644 Penumbra.GameData/UtilityFunctions.cs diff --git a/Penumbra.GameData/Data/DisassembledShader.cs b/Penumbra.GameData/Data/DisassembledShader.cs index b782bf74..bfae7a74 100644 --- a/Penumbra.GameData/Data/DisassembledShader.cs +++ b/Penumbra.GameData/Data/DisassembledShader.cs @@ -4,23 +4,22 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; using Penumbra.GameData.Interop; -using static Penumbra.GameData.Files.ShpkFile; namespace Penumbra.GameData.Data; -public class DisassembledShader +public partial class DisassembledShader { public struct ResourceBinding { - public string Name; - public ResourceType Type; - public Format Format; - public ResourceDimension Dimension; - public uint Slot; - public uint Elements; - public uint RegisterCount; + public string Name; + public ResourceType Type; + public Format Format; + public ResourceDimension Dimension; + public uint Slot; + public uint Elements; + public uint RegisterCount; public VectorComponents[] Used; - public VectorComponents UsedDynamically; + public VectorComponents UsedDynamically; } // Abbreviated using the uppercased first char of their name @@ -30,7 +29,7 @@ public class DisassembledShader ConstantBuffer = 0x43, // 'C' Sampler = 0x53, // 'S' Texture = 0x54, // 'T' - UAV = 0x55, // 'U' + Uav = 0x55, // 'U' } // Abbreviated using the uppercased first and last char of their name @@ -56,22 +55,22 @@ public class DisassembledShader public struct InputOutput { - public string Name; - public uint Index; + public string Name; + public uint Index; public VectorComponents Mask; - public uint Register; - public string SystemValue; - public Format Format; + public uint Register; + public string SystemValue; + public Format Format; public VectorComponents Used; } [Flags] public enum VectorComponents : byte { - X = 1, - Y = 2, - Z = 4, - W = 8, + X = 1, + Y = 2, + Z = 4, + W = 8, All = 15, } @@ -82,21 +81,31 @@ public class DisassembledShader Vertex = 0x56, // 'V' } - private static readonly Regex ResourceBindingSizeRegex = new(@"\s(\w+)(?:\[\d+\])?;\s*//\s*Offset:\s*0\s*Size:\s*(\d+)$", RegexOptions.Multiline | RegexOptions.NonBacktracking); - private static readonly Regex SM3ConstantBufferUsageRegex = new(@"c(\d+)(?:\[([^\]]+)\])?(?:\.([wxyz]+))?", RegexOptions.NonBacktracking); - private static readonly Regex SM3TextureUsageRegex = new(@"^\s*texld\S*\s+[^,]+,[^,]+,\s*s(\d+)", RegexOptions.NonBacktracking); - private static readonly Regex SM5ConstantBufferUsageRegex = new(@"cb(\d+)\[([^\]]+)\]\.([wxyz]+)", RegexOptions.NonBacktracking); - private static readonly Regex SM5TextureUsageRegex = new(@"^\s*sample_\S*\s+[^.]+\.([wxyz]+),[^,]+,\s*t(\d+)\.([wxyz]+)", RegexOptions.NonBacktracking); - private static readonly char[] Digits = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' }; + [GeneratedRegex(@"\s(\w+)(?:\[\d+\])?;\s*//\s*Offset:\s*0\s*Size:\s*(\d+)$", RegexOptions.Multiline | RegexOptions.NonBacktracking)] + private static partial Regex ResourceBindingSizeRegex(); - public readonly string RawDisassembly; - public readonly uint ShaderModel; - public readonly ShaderStage Stage; - public readonly string BufferDefinitions; + [GeneratedRegex(@"c(\d+)(?:\[([^\]]+)\])?(?:\.([wxyz]+))?", RegexOptions.NonBacktracking)] + private static partial Regex Sm3ConstantBufferUsageRegex(); + + [GeneratedRegex(@"^\s*texld\S*\s+[^,]+,[^,]+,\s*s(\d+)", RegexOptions.NonBacktracking)] + private static partial Regex Sm3TextureUsageRegex(); + + [GeneratedRegex(@"cb(\d+)\[([^\]]+)\]\.([wxyz]+)", RegexOptions.NonBacktracking)] + private static partial Regex Sm5ConstantBufferUsageRegex(); + + [GeneratedRegex(@"^\s*sample_\S*\s+[^.]+\.([wxyz]+),[^,]+,\s*t(\d+)\.([wxyz]+)", RegexOptions.NonBacktracking)] + private static partial Regex Sm5TextureUsageRegex(); + + private static readonly char[] Digits = Enumerable.Range(0, 10).Select(c => (char) ('0' + c)).ToArray(); + + public readonly string RawDisassembly; + public readonly uint ShaderModel; + public readonly ShaderStage Stage; + public readonly string BufferDefinitions; public readonly ResourceBinding[] ResourceBindings; - public readonly InputOutput[] InputSignature; - public readonly InputOutput[] OutputSignature; - public readonly string[] Instructions; + public readonly InputOutput[] InputSignature; + public readonly InputOutput[] OutputSignature; + public readonly string[] Instructions; public DisassembledShader(string rawDisassembly) { @@ -104,49 +113,38 @@ public class DisassembledShader var lines = rawDisassembly.Split('\n'); Instructions = Array.FindAll(lines, ln => !ln.StartsWith("//") && ln.Length > 0); var shaderModel = Instructions[0].Trim().Split('_'); - Stage = (ShaderStage)(byte)char.ToUpper(shaderModel[0][0]); + Stage = (ShaderStage)(byte)char.ToUpper(shaderModel[0][0]); ShaderModel = (uint.Parse(shaderModel[1]) << 8) | uint.Parse(shaderModel[2]); var header = PreParseHeader(lines.AsSpan()[..Array.IndexOf(lines, Instructions[0])]); switch (ShaderModel >> 8) { case 3: - ParseSM3Header(header, out BufferDefinitions, out ResourceBindings, out InputSignature, out OutputSignature); - ParseSM3ResourceUsage(Instructions, ResourceBindings); + ParseSm3Header(header, out BufferDefinitions, out ResourceBindings, out InputSignature, out OutputSignature); + ParseSm3ResourceUsage(Instructions, ResourceBindings); break; case 5: - ParseSM5Header(header, out BufferDefinitions, out ResourceBindings, out InputSignature, out OutputSignature); - ParseSM5ResourceUsage(Instructions, ResourceBindings); + ParseSm5Header(header, out BufferDefinitions, out ResourceBindings, out InputSignature, out OutputSignature); + ParseSm5ResourceUsage(Instructions, ResourceBindings); break; - default: - throw new NotImplementedException(); + default: throw new NotImplementedException(); } } public ResourceBinding? GetResourceBindingByName(ResourceType type, string name) - { - return ResourceBindings.Select(binding => new ResourceBinding?(binding)).FirstOrDefault(binding => binding!.Value.Type == type && binding!.Value.Name == name); - } + => ResourceBindings.FirstOrNull(b => b.Type == type && b.Name == name); public ResourceBinding? GetResourceBindingBySlot(ResourceType type, uint slot) - { - return ResourceBindings.Select(binding => new ResourceBinding?(binding)).FirstOrDefault(binding => binding!.Value.Type == type && binding!.Value.Slot == slot); - } + => ResourceBindings.FirstOrNull(b => b.Type == type && b.Slot == slot); public static DisassembledShader Disassemble(ReadOnlySpan shaderBlob) - { - return new DisassembledShader(D3DCompiler.Disassemble(shaderBlob)); - } + => new(D3DCompiler.Disassemble(shaderBlob)); - private static void ParseSM3Header(Dictionary header, out string bufferDefinitions, out ResourceBinding[] resourceBindings, out InputOutput[] inputSignature, out InputOutput[] outputSignature) + private static void ParseSm3Header(Dictionary header, out string bufferDefinitions, + out ResourceBinding[] resourceBindings, out InputOutput[] inputSignature, out InputOutput[] outputSignature) { - if (header.TryGetValue("Parameters", out var rawParameters)) - { - bufferDefinitions = string.Join('\n', rawParameters); - } - else - { - bufferDefinitions = string.Empty; - } + bufferDefinitions = header.TryGetValue("Parameters", out var rawParameters) + ? string.Join('\n', rawParameters) + : string.Empty; if (header.TryGetValue("Registers", out var rawRegisters)) { var (_, registers) = ParseTable(rawRegisters); @@ -154,10 +152,8 @@ public class DisassembledShader { var type = (ResourceType)(byte)char.ToUpper(register[1][0]); if (type == ResourceType.Sampler) - { type = ResourceType.Texture; - } - uint size = uint.Parse(register[2]); + var size = uint.Parse(register[2]); return new ResourceBinding { Name = register[0], @@ -175,14 +171,15 @@ public class DisassembledShader { resourceBindings = Array.Empty(); } - inputSignature = Array.Empty(); + + inputSignature = Array.Empty(); outputSignature = Array.Empty(); } - private static void ParseSM3ResourceUsage(string[] instructions, ResourceBinding[] resourceBindings) + private static void ParseSm3ResourceUsage(string[] instructions, ResourceBinding[] resourceBindings) { var cbIndices = new Dictionary(); - var tIndices = new Dictionary(); + var tIndices = new Dictionary(); { var i = 0; foreach (var binding in resourceBindings) @@ -191,14 +188,13 @@ public class DisassembledShader { case ResourceType.ConstantBuffer: for (var j = 0u; j < binding.RegisterCount; j++) - { cbIndices[binding.Slot + j] = i; - } break; case ResourceType.Texture: tIndices[binding.Slot] = i; break; } + ++i; } } @@ -206,43 +202,39 @@ public class DisassembledShader { var trimmed = instruction.Trim(); if (trimmed.StartsWith("def") || trimmed.StartsWith("dcl")) - { continue; - } - foreach (Match cbMatch in SM3ConstantBufferUsageRegex.Matches(instruction)) + + foreach (Match cbMatch in Sm3ConstantBufferUsageRegex().Matches(instruction)) { var buffer = uint.Parse(cbMatch.Groups[1].Value); if (cbIndices.TryGetValue(buffer, out var i)) { var swizzle = cbMatch.Groups[3].Success ? ParseVectorComponents(cbMatch.Groups[3].Value) : VectorComponents.All; if (cbMatch.Groups[2].Success) - { resourceBindings[i].UsedDynamically |= swizzle; - } else - { resourceBindings[i].Used[buffer - resourceBindings[i].Slot] |= swizzle; - } } } - var tMatch = SM3TextureUsageRegex.Match(instruction); + + var tMatch = Sm3TextureUsageRegex().Match(instruction); if (tMatch.Success) { var texture = uint.Parse(tMatch.Groups[1].Value); if (tIndices.TryGetValue(texture, out var i)) - { resourceBindings[i].Used[0] = VectorComponents.All; - } } } } - private static void ParseSM5Header(Dictionary header, out string bufferDefinitions, out ResourceBinding[] resourceBindings, out InputOutput[] inputSignature, out InputOutput[] outputSignature) + private static void ParseSm5Header(Dictionary header, out string bufferDefinitions, + out ResourceBinding[] resourceBindings, out InputOutput[] inputSignature, out InputOutput[] outputSignature) { if (header.TryGetValue("Resource Bindings", out var rawResBindings)) { var (head, resBindings) = ParseTable(rawResBindings); - resourceBindings = Array.ConvertAll(resBindings, binding => { + resourceBindings = Array.ConvertAll(resBindings, binding => + { var type = (ResourceType)(byte)char.ToUpper(binding[1][0]); return new ResourceBinding { @@ -261,10 +253,11 @@ public class DisassembledShader { resourceBindings = Array.Empty(); } + if (header.TryGetValue("Buffer Definitions", out var rawBufferDefs)) { bufferDefinitions = string.Join('\n', rawBufferDefs); - foreach (Match match in ResourceBindingSizeRegex.Matches(bufferDefinitions)) + foreach (Match match in ResourceBindingSizeRegex().Matches(bufferDefinitions)) { var name = match.Groups[1].Value; var bytesSize = uint.Parse(match.Groups[2].Value); @@ -272,7 +265,7 @@ public class DisassembledShader if (pos >= 0) { resourceBindings[pos].RegisterCount = (bytesSize + 0xF) >> 4; - resourceBindings[pos].Used = new VectorComponents[resourceBindings[pos].RegisterCount]; + resourceBindings[pos].Used = new VectorComponents[resourceBindings[pos].RegisterCount]; } } } @@ -281,30 +274,32 @@ public class DisassembledShader bufferDefinitions = string.Empty; } - static InputOutput ParseInputOutput(string[] inOut) => new() - { - Name = inOut[0], - Index = uint.Parse(inOut[1]), - Mask = ParseVectorComponents(inOut[2]), - Register = uint.Parse(inOut[3]), - SystemValue = string.Intern(inOut[4]), - Format = (Format)(((byte)char.ToUpper(inOut[5][0]) << 8) | (byte)char.ToUpper(inOut[5][^1])), - Used = ParseVectorComponents(inOut[6]), - }; + static InputOutput ParseInputOutput(string[] inOut) + => new() + { + Name = inOut[0], + Index = uint.Parse(inOut[1]), + Mask = ParseVectorComponents(inOut[2]), + Register = uint.Parse(inOut[3]), + SystemValue = string.Intern(inOut[4]), + Format = (Format)(((byte)char.ToUpper(inOut[5][0]) << 8) | (byte)char.ToUpper(inOut[5][^1])), + Used = ParseVectorComponents(inOut[6]), + }; if (header.TryGetValue("Input signature", out var rawInputSig)) { var (_, inputSig) = ParseTable(rawInputSig); - inputSignature = Array.ConvertAll(inputSig, ParseInputOutput); + inputSignature = Array.ConvertAll(inputSig, ParseInputOutput); } else { inputSignature = Array.Empty(); } + if (header.TryGetValue("Output signature", out var rawOutputSig)) { var (_, outputSig) = ParseTable(rawOutputSig); - outputSignature = Array.ConvertAll(outputSig, ParseInputOutput); + outputSignature = Array.ConvertAll(outputSig, ParseInputOutput); } else { @@ -312,10 +307,10 @@ public class DisassembledShader } } - private static void ParseSM5ResourceUsage(string[] instructions, ResourceBinding[] resourceBindings) + private static void ParseSm5ResourceUsage(string[] instructions, ResourceBinding[] resourceBindings) { var cbIndices = new Dictionary(); - var tIndices = new Dictionary(); + var tIndices = new Dictionary(); { var i = 0; foreach (var binding in resourceBindings) @@ -329,6 +324,7 @@ public class DisassembledShader tIndices[binding.Slot] = i; break; } + ++i; } } @@ -336,10 +332,9 @@ public class DisassembledShader { var trimmed = instruction.Trim(); if (trimmed.StartsWith("def") || trimmed.StartsWith("dcl")) - { continue; - } - foreach (Match cbMatch in SM5ConstantBufferUsageRegex.Matches(instruction)) + + foreach (Match cbMatch in Sm5ConstantBufferUsageRegex().Matches(instruction)) { var buffer = uint.Parse(cbMatch.Groups[1].Value); if (cbIndices.TryGetValue(buffer, out var i)) @@ -348,9 +343,7 @@ public class DisassembledShader if (int.TryParse(cbMatch.Groups[2].Value, out var vector)) { if (vector < resourceBindings[i].Used.Length) - { resourceBindings[i].Used[vector] |= swizzle; - } } else { @@ -358,31 +351,24 @@ public class DisassembledShader } } } - var tMatch = SM5TextureUsageRegex.Match(instruction); + + var tMatch = Sm5TextureUsageRegex().Match(instruction); if (tMatch.Success) { var texture = uint.Parse(tMatch.Groups[2].Value); if (tIndices.TryGetValue(texture, out var i)) { - var outSwizzle = ParseVectorComponents(tMatch.Groups[1].Value); + var outSwizzle = ParseVectorComponents(tMatch.Groups[1].Value); var rawInSwizzle = tMatch.Groups[3].Value; - var inSwizzle = new StringBuilder(4); + var inSwizzle = new StringBuilder(4); if ((outSwizzle & VectorComponents.X) != 0) - { inSwizzle.Append(rawInSwizzle[0]); - } if ((outSwizzle & VectorComponents.Y) != 0) - { inSwizzle.Append(rawInSwizzle[1]); - } if ((outSwizzle & VectorComponents.Z) != 0) - { inSwizzle.Append(rawInSwizzle[2]); - } if ((outSwizzle & VectorComponents.W) != 0) - { inSwizzle.Append(rawInSwizzle[3]); - } resourceBindings[i].Used[0] |= ParseVectorComponents(inSwizzle.ToString()); } } @@ -393,9 +379,9 @@ public class DisassembledShader { components = components.ToUpperInvariant(); return (components.Contains('X') ? VectorComponents.X : 0) - | (components.Contains('Y') ? VectorComponents.Y : 0) - | (components.Contains('Z') ? VectorComponents.Z : 0) - | (components.Contains('W') ? VectorComponents.W : 0); + | (components.Contains('Y') ? VectorComponents.Y : 0) + | (components.Contains('Z') ? VectorComponents.Z : 0) + | (components.Contains('W') ? VectorComponents.W : 0); } private static Dictionary PreParseHeader(ReadOnlySpan header) @@ -405,17 +391,13 @@ public class DisassembledShader void AddSection(string name, ReadOnlySpan section) { while (section.Length > 0 && section[0].Length <= 3) - { section = section[1..]; - } while (section.Length > 0 && section[^1].Length <= 3) - { section = section[..^1]; - } sections.Add(name, Array.ConvertAll(section.ToArray(), ln => ln.Length <= 3 ? string.Empty : ln[3..])); } - var lastSectionName = ""; + var lastSectionName = ""; var lastSectionStart = 0; for (var i = 1; i < header.Length - 1; ++i) { @@ -423,11 +405,12 @@ public class DisassembledShader if (header[i - 1].Length <= 3 && header[i + 1].Length <= 3 && (current = header[i].TrimEnd()).EndsWith(':')) { AddSection(lastSectionName, header[lastSectionStart..(i - 1)]); - lastSectionName = current[3..^1]; + lastSectionName = current[3..^1]; lastSectionStart = i + 2; ++i; // The next line cannot match } } + AddSection(lastSectionName, header[lastSectionStart..]); return sections; @@ -442,9 +425,8 @@ public class DisassembledShader { var start = dashLine.IndexOf('-', i); if (start < 0) - { break; - } + var end = dashLine.IndexOf(' ', start + 1); if (end < 0) { @@ -462,20 +444,17 @@ public class DisassembledShader { var headerLine = lines[0]; for (var i = 0; i < columns.Count; ++i) - { headers[i] = headerLine[columns[i]].Trim(); - } } var data = new List(); foreach (var line in lines[2..]) { var row = new string[columns.Count]; for (var i = 0; i < columns.Count; ++i) - { row[i] = line[columns[i]].Trim(); - } data.Add(row); } + return (headers, data.ToArray()); } } diff --git a/Penumbra.GameData/Files/MtrlFile.ColorDyeSet.cs b/Penumbra.GameData/Files/MtrlFile.ColorDyeSet.cs new file mode 100644 index 00000000..4cd2ff28 --- /dev/null +++ b/Penumbra.GameData/Files/MtrlFile.ColorDyeSet.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Penumbra.GameData.Files; + +public partial class MtrlFile +{ + public unsafe struct ColorDyeSet + { + public struct Row + { + private ushort _data; + + public ushort Template + { + get => (ushort)(_data >> 5); + set => _data = (ushort)((_data & 0x1F) | (value << 5)); + } + + public bool Diffuse + { + get => (_data & 0x01) != 0; + set => _data = (ushort)(value ? _data | 0x01 : _data & 0xFFFE); + } + + public bool Specular + { + get => (_data & 0x02) != 0; + set => _data = (ushort)(value ? _data | 0x02 : _data & 0xFFFD); + } + + public bool Emissive + { + get => (_data & 0x04) != 0; + set => _data = (ushort)(value ? _data | 0x04 : _data & 0xFFFB); + } + + public bool Gloss + { + get => (_data & 0x08) != 0; + set => _data = (ushort)(value ? _data | 0x08 : _data & 0xFFF7); + } + + public bool SpecularStrength + { + get => (_data & 0x10) != 0; + set => _data = (ushort)(value ? _data | 0x10 : _data & 0xFFEF); + } + } + + public struct RowArray : IEnumerable + { + public const int NumRows = 16; + private fixed ushort _rowData[NumRows]; + + public ref Row this[int i] + { + get + { + fixed (ushort* ptr = _rowData) + { + return ref ((Row*)ptr)[i]; + } + } + } + + public IEnumerator GetEnumerator() + { + for (var i = 0; i < NumRows; ++i) + yield return this[i]; + } + + IEnumerator IEnumerable.GetEnumerator() + => GetEnumerator(); + + public ReadOnlySpan AsBytes() + { + fixed (ushort* ptr = _rowData) + { + return new ReadOnlySpan(ptr, NumRows * sizeof(ushort)); + } + } + } + + public RowArray Rows; + public string Name; + public ushort Index; + } +} diff --git a/Penumbra.GameData/Files/MtrlFile.ColorSet.cs b/Penumbra.GameData/Files/MtrlFile.ColorSet.cs new file mode 100644 index 00000000..61647d79 --- /dev/null +++ b/Penumbra.GameData/Files/MtrlFile.ColorSet.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Numerics; + +namespace Penumbra.GameData.Files; + +public partial class MtrlFile +{ + public unsafe struct ColorSet + { + public struct Row + { + public const int Size = 32; + + private fixed ushort _data[16]; + + public Vector3 Diffuse + { + get => new(ToFloat(0), ToFloat(1), ToFloat(2)); + set + { + _data[0] = FromFloat(value.X); + _data[1] = FromFloat(value.Y); + _data[2] = FromFloat(value.Z); + } + } + + public Vector3 Specular + { + get => new(ToFloat(4), ToFloat(5), ToFloat(6)); + set + { + _data[4] = FromFloat(value.X); + _data[5] = FromFloat(value.Y); + _data[6] = FromFloat(value.Z); + } + } + + public Vector3 Emissive + { + get => new(ToFloat(8), ToFloat(9), ToFloat(10)); + set + { + _data[8] = FromFloat(value.X); + _data[9] = FromFloat(value.Y); + _data[10] = FromFloat(value.Z); + } + } + + public Vector2 MaterialRepeat + { + get => new(ToFloat(12), ToFloat(15)); + set + { + _data[12] = FromFloat(value.X); + _data[15] = FromFloat(value.Y); + } + } + + public Vector2 MaterialSkew + { + get => new(ToFloat(13), ToFloat(14)); + set + { + _data[13] = FromFloat(value.X); + _data[14] = FromFloat(value.Y); + } + } + + public float SpecularStrength + { + get => ToFloat(3); + set => _data[3] = FromFloat(value); + } + + public float GlossStrength + { + get => ToFloat(7); + set => _data[7] = FromFloat(value); + } + + public ushort TileSet + { + get => (ushort)(ToFloat(11) * 64f); + set => _data[11] = FromFloat(value / 64f); + } + + private float ToFloat(int idx) + => (float)BitConverter.UInt16BitsToHalf(_data[idx]); + + private static ushort FromFloat(float x) + => BitConverter.HalfToUInt16Bits((Half)x); + } + + public struct RowArray : IEnumerable + { + public const int NumRows = 16; + private fixed byte _rowData[NumRows * Row.Size]; + + public ref Row this[int i] + { + get + { + fixed (byte* ptr = _rowData) + { + return ref ((Row*)ptr)[i]; + } + } + } + + public IEnumerator GetEnumerator() + { + for (var i = 0; i < NumRows; ++i) + yield return this[i]; + } + + IEnumerator IEnumerable.GetEnumerator() + => GetEnumerator(); + + public ReadOnlySpan AsBytes() + { + fixed (byte* ptr = _rowData) + { + return new ReadOnlySpan(ptr, NumRows * Row.Size); + } + } + } + + public RowArray Rows; + public string Name; + public ushort Index; + public bool HasRows; + } +} diff --git a/Penumbra.GameData/Files/MtrlFile.Write.cs b/Penumbra.GameData/Files/MtrlFile.Write.cs index 7faabc68..9bc5a2ce 100644 --- a/Penumbra.GameData/Files/MtrlFile.Write.cs +++ b/Penumbra.GameData/Files/MtrlFile.Write.cs @@ -1,4 +1,3 @@ -using System; using System.IO; using System.Linq; using System.Text; diff --git a/Penumbra.GameData/Files/MtrlFile.cs b/Penumbra.GameData/Files/MtrlFile.cs index 910e4adb..6a51a39b 100644 --- a/Penumbra.GameData/Files/MtrlFile.cs +++ b/Penumbra.GameData/Files/MtrlFile.cs @@ -1,9 +1,7 @@ using System; -using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Numerics; using System.Text; using Lumina.Data.Parsing; using Lumina.Extensions; @@ -13,262 +11,10 @@ namespace Penumbra.GameData.Files; public partial class MtrlFile : IWritable { - public struct UvSet - { - public string Name; - public ushort Index; - } - - public unsafe struct ColorSet - { - public struct Row - { - public const int Size = 32; - - private fixed ushort _data[16]; - - public Vector3 Diffuse - { - get => new(ToFloat(0), ToFloat(1), ToFloat(2)); - set - { - _data[0] = FromFloat(value.X); - _data[1] = FromFloat(value.Y); - _data[2] = FromFloat(value.Z); - } - } - - public Vector3 Specular - { - get => new(ToFloat(4), ToFloat(5), ToFloat(6)); - set - { - _data[4] = FromFloat(value.X); - _data[5] = FromFloat(value.Y); - _data[6] = FromFloat(value.Z); - } - } - - public Vector3 Emissive - { - get => new(ToFloat(8), ToFloat(9), ToFloat(10)); - set - { - _data[8] = FromFloat(value.X); - _data[9] = FromFloat(value.Y); - _data[10] = FromFloat(value.Z); - } - } - - public Vector2 MaterialRepeat - { - get => new(ToFloat(12), ToFloat(15)); - set - { - _data[12] = FromFloat(value.X); - _data[15] = FromFloat(value.Y); - } - } - - public Vector2 MaterialSkew - { - get => new(ToFloat(13), ToFloat(14)); - set - { - _data[13] = FromFloat(value.X); - _data[14] = FromFloat(value.Y); - } - } - - public float SpecularStrength - { - get => ToFloat(3); - set => _data[3] = FromFloat(value); - } - - public float GlossStrength - { - get => ToFloat(7); - set => _data[7] = FromFloat(value); - } - - public ushort TileSet - { - get => (ushort)(ToFloat(11) * 64f); - set => _data[11] = FromFloat(value / 64f); - } - - private float ToFloat(int idx) - => (float)BitConverter.UInt16BitsToHalf(_data[idx]); - - private static ushort FromFloat(float x) - => BitConverter.HalfToUInt16Bits((Half)x); - } - - public struct RowArray : IEnumerable - { - public const int NumRows = 16; - private fixed byte _rowData[NumRows * Row.Size]; - - public ref Row this[int i] - { - get - { - fixed (byte* ptr = _rowData) - { - return ref ((Row*)ptr)[i]; - } - } - } - - public IEnumerator GetEnumerator() - { - for (var i = 0; i < NumRows; ++i) - yield return this[i]; - } - - IEnumerator IEnumerable.GetEnumerator() - => GetEnumerator(); - - public ReadOnlySpan AsBytes() - { - fixed (byte* ptr = _rowData) - { - return new ReadOnlySpan(ptr, NumRows * Row.Size); - } - } - } - - public RowArray Rows; - public string Name; - public ushort Index; - public bool HasRows; - } - - public unsafe struct ColorDyeSet - { - public struct Row - { - private ushort _data; - - public ushort Template - { - get => (ushort)(_data >> 5); - set => _data = (ushort)((_data & 0x1F) | (value << 5)); - } - - public bool Diffuse - { - get => (_data & 0x01) != 0; - set => _data = (ushort)(value ? _data | 0x01 : _data & 0xFFFE); - } - - public bool Specular - { - get => (_data & 0x02) != 0; - set => _data = (ushort)(value ? _data | 0x02 : _data & 0xFFFD); - } - - public bool Emissive - { - get => (_data & 0x04) != 0; - set => _data = (ushort)(value ? _data | 0x04 : _data & 0xFFFB); - } - - public bool Gloss - { - get => (_data & 0x08) != 0; - set => _data = (ushort)(value ? _data | 0x08 : _data & 0xFFF7); - } - - public bool SpecularStrength - { - get => (_data & 0x10) != 0; - set => _data = (ushort)(value ? _data | 0x10 : _data & 0xFFEF); - } - } - - public struct RowArray : IEnumerable - { - public const int NumRows = 16; - private fixed ushort _rowData[NumRows]; - - public ref Row this[int i] - { - get - { - fixed (ushort* ptr = _rowData) - { - return ref ((Row*)ptr)[i]; - } - } - } - - public IEnumerator GetEnumerator() - { - for (var i = 0; i < NumRows; ++i) - yield return this[i]; - } - - IEnumerator IEnumerable.GetEnumerator() - => GetEnumerator(); - - public ReadOnlySpan AsBytes() - { - fixed (ushort* ptr = _rowData) - { - return new ReadOnlySpan(ptr, NumRows * sizeof(ushort)); - } - } - } - - public RowArray Rows; - public string Name; - public ushort Index; - } - - public struct Texture - { - public string Path; - public ushort Flags; - - public bool DX11 - => (Flags & 0x8000) != 0; - } - - public struct Constant - { - public uint Id; - public ushort ByteOffset; - public ushort ByteSize; - } - - public struct ShaderPackageData - { - public string Name; - public ShaderKey[] ShaderKeys; - public Constant[] Constants; - public Sampler[] Samplers; - public float[] ShaderValues; - public uint Flags; - } - - public readonly uint Version; - public bool Valid - { - get - { - foreach (var texture in Textures) - { - if (!texture.Path.Contains('/')) - { - return false; - } - } - return true; - } - } + + public bool Valid + => CheckTextures(); public Texture[] Textures; public UvSet[] UvSets; @@ -277,7 +23,7 @@ public partial class MtrlFile : IWritable public ShaderPackageData ShaderPackage; public byte[] AdditionalData; - public ShpkFile? AssociatedShpk; + public ShpkFile? AssociatedShpk; public bool ApplyDyeTemplate(StmFile stm, int colorSetIdx, int rowIdx, StainId stainId) { @@ -324,15 +70,13 @@ public partial class MtrlFile : IWritable public Span GetConstantValues(Constant constant) { - if ((constant.ByteOffset & 0x3) == 0 && (constant.ByteSize & 0x3) == 0 - && ((constant.ByteOffset + constant.ByteSize) >> 2) <= ShaderPackage.ShaderValues.Length) - { - return ShaderPackage.ShaderValues.AsSpan().Slice(constant.ByteOffset >> 2, constant.ByteSize >> 2); - } - else - { + if ((constant.ByteOffset & 0x3) != 0 + || (constant.ByteSize & 0x3) != 0 + || (constant.ByteOffset + constant.ByteSize) >> 2 > ShaderPackage.ShaderValues.Length) return null; - } + + return ShaderPackage.ShaderValues.AsSpan().Slice(constant.ByteOffset >> 2, constant.ByteSize >> 2); + } public List<(Sampler?, ShpkFile.Resource?)> GetSamplersByTexture() @@ -348,13 +92,7 @@ public partial class MtrlFile : IWritable } return samplers; - } - - // Activator.CreateInstance can't use a ctor with a default value so this has to be made explicit - public MtrlFile(byte[] data) - : this(data, null) - { - } + } public MtrlFile(byte[] data, Func? loadAssociatedShpk = null) { @@ -469,6 +207,41 @@ public partial class MtrlFile : IWritable { strings = strings[offset..]; var end = strings.IndexOf((byte)'\0'); - return Encoding.UTF8.GetString(strings[..end]); + return Encoding.UTF8.GetString(end == -1 ? strings : strings[..end]); + } + + private bool CheckTextures() + => Textures.All(texture => texture.Path.Contains('/')); + + public struct UvSet + { + public string Name; + public ushort Index; + } + + public struct Texture + { + public string Path; + public ushort Flags; + + public bool DX11 + => (Flags & 0x8000) != 0; + } + + public struct Constant + { + public uint Id; + public ushort ByteOffset; + public ushort ByteSize; + } + + public struct ShaderPackageData + { + public string Name; + public ShaderKey[] ShaderKeys; + public Constant[] Constants; + public Sampler[] Samplers; + public float[] ShaderValues; + public uint Flags; } } diff --git a/Penumbra.GameData/Files/ShpkFile.Shader.cs b/Penumbra.GameData/Files/ShpkFile.Shader.cs new file mode 100644 index 00000000..3d94dbb4 --- /dev/null +++ b/Penumbra.GameData/Files/ShpkFile.Shader.cs @@ -0,0 +1,220 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using Lumina.Misc; +using Penumbra.GameData.Data; + +namespace Penumbra.GameData.Files; + +public partial class ShpkFile +{ + public struct Shader + { + public DisassembledShader.ShaderStage Stage; + public DxVersion DirectXVersion; + public Resource[] Constants; + public Resource[] Samplers; + public Resource[] Uavs; + public byte[] AdditionalHeader; + private byte[] _byteData; + private DisassembledShader? _disassembly; + + public byte[] Blob + { + get => _byteData; + set + { + if (_byteData == value) + return; + + if (Stage != DisassembledShader.ShaderStage.Unspecified) + { + // Reject the blob entirely if we can't disassemble it or if we find inconsistencies. + var disasm = DisassembledShader.Disassemble(value); + if (disasm.Stage != Stage || (disasm.ShaderModel >> 8) + 6 != (uint)DirectXVersion) + throw new ArgumentException( + $"The supplied blob is a DirectX {(disasm.ShaderModel >> 8) + 6} {disasm.Stage} shader ; expected a DirectX {(uint)DirectXVersion} {Stage} shader.", + nameof(value)); + + if (disasm.ShaderModel >= 0x0500) + { + var samplers = new Dictionary(); + var textures = new Dictionary(); + foreach (var binding in disasm.ResourceBindings) + { + switch (binding.Type) + { + case DisassembledShader.ResourceType.Texture: + textures[binding.Slot] = NormalizeResourceName(binding.Name); + break; + case DisassembledShader.ResourceType.Sampler: + samplers[binding.Slot] = NormalizeResourceName(binding.Name); + break; + } + } + + if (samplers.Count != textures.Count + || !samplers.All(pair => textures.TryGetValue(pair.Key, out var texName) && pair.Value == texName)) + throw new ArgumentException($"The supplied blob has inconsistent sampler and texture allocation."); + } + + _byteData = value; + _disassembly = disasm; + } + else + { + _byteData = value; + _disassembly = null; + } + + UpdateUsed(); + } + } + + public DisassembledShader? Disassembly + => _disassembly; + + public Resource? GetConstantById(uint id) + => Constants.FirstOrNull(res => res.Id == id); + + public Resource? GetConstantByName(string name) + => Constants.FirstOrNull(res => res.Name == name); + + public Resource? GetSamplerById(uint id) + => Samplers.FirstOrNull(s => s.Id == id); + + public Resource? GetSamplerByName(string name) + => Samplers.FirstOrNull(s => s.Name == name); + + public Resource? GetUavById(uint id) + => Uavs.FirstOrNull(u => u.Id == id); + + public Resource? GetUavByName(string name) + => Uavs.FirstOrNull(u => u.Name == name); + + public void UpdateResources(ShpkFile file) + { + if (_disassembly == null) + throw new InvalidOperationException(); + + var constants = new List(); + var samplers = new List(); + var uavs = new List(); + foreach (var binding in _disassembly.ResourceBindings) + { + switch (binding.Type) + { + case DisassembledShader.ResourceType.ConstantBuffer: + var name = NormalizeResourceName(binding.Name); + // We want to preserve IDs as much as possible, and to deterministically generate new ones in a way that's most compliant with the native ones, to maximize compatibility. + var id = GetConstantByName(name)?.Id ?? file.GetConstantByName(name)?.Id ?? Crc32.Get(name, 0xFFFFFFFFu); + constants.Add(new Resource + { + Id = id, + Name = name, + Slot = (ushort)binding.Slot, + Size = (ushort)binding.RegisterCount, + Used = binding.Used, + UsedDynamically = binding.UsedDynamically, + }); + break; + case DisassembledShader.ResourceType.Texture: + name = NormalizeResourceName(binding.Name); + id = GetSamplerByName(name)?.Id ?? file.GetSamplerByName(name)?.Id ?? Crc32.Get(name, 0xFFFFFFFFu); + samplers.Add(new Resource + { + Id = id, + Name = name, + Slot = (ushort)binding.Slot, + Size = (ushort)binding.Slot, + Used = binding.Used, + UsedDynamically = binding.UsedDynamically, + }); + break; + case DisassembledShader.ResourceType.Uav: + name = NormalizeResourceName(binding.Name); + id = GetUavByName(name)?.Id ?? file.GetUavByName(name)?.Id ?? Crc32.Get(name, 0xFFFFFFFFu); + uavs.Add(new Resource + { + Id = id, + Name = name, + Slot = (ushort)binding.Slot, + Size = (ushort)binding.Slot, + Used = binding.Used, + UsedDynamically = binding.UsedDynamically, + }); + break; + } + } + + Constants = constants.ToArray(); + Samplers = samplers.ToArray(); + Uavs = uavs.ToArray(); + } + + private void UpdateUsed() + { + if (_disassembly != null) + { + var cbUsage = new Dictionary(); + var tUsage = new Dictionary(); + var uUsage = new Dictionary(); + foreach (var binding in _disassembly.ResourceBindings) + { + switch (binding.Type) + { + case DisassembledShader.ResourceType.ConstantBuffer: + cbUsage[NormalizeResourceName(binding.Name)] = (binding.Used, binding.UsedDynamically); + break; + case DisassembledShader.ResourceType.Texture: + tUsage[NormalizeResourceName(binding.Name)] = (binding.Used, binding.UsedDynamically); + break; + case DisassembledShader.ResourceType.Uav: + uUsage[NormalizeResourceName(binding.Name)] = (binding.Used, binding.UsedDynamically); + break; + } + } + + static void CopyUsed(Resource[] resources, + Dictionary used) + { + for (var i = 0; i < resources.Length; ++i) + { + if (used.TryGetValue(resources[i].Name, out var usage)) + { + resources[i].Used = usage.Item1; + resources[i].UsedDynamically = usage.Item2; + } + else + { + resources[i].Used = null; + resources[i].UsedDynamically = null; + } + } + } + + CopyUsed(Constants, cbUsage); + CopyUsed(Samplers, tUsage); + CopyUsed(Uavs, uUsage); + } + else + { + ClearUsed(Constants); + ClearUsed(Samplers); + ClearUsed(Uavs); + } + } + + private static string NormalizeResourceName(string resourceName) + { + var dot = resourceName.IndexOf('.'); + if (dot >= 0) + return resourceName[..dot]; + if (resourceName.Length > 1 && resourceName[^2] is '_' && resourceName[^1] is 'S' or 'T') + return resourceName[..^2]; + + return resourceName; + } + } +} diff --git a/Penumbra.GameData/Files/ShpkFile.StringPool.cs b/Penumbra.GameData/Files/ShpkFile.StringPool.cs new file mode 100644 index 00000000..bad56d20 --- /dev/null +++ b/Penumbra.GameData/Files/ShpkFile.StringPool.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Penumbra.GameData.Files; + +public partial class ShpkFile +{ + public class StringPool + { + public MemoryStream Data; + public List StartingOffsets; + + public StringPool(ReadOnlySpan bytes) + { + Data = new MemoryStream(); + Data.Write(bytes); + StartingOffsets = new List + { + 0, + }; + for (var i = 0; i < bytes.Length; ++i) + { + if (bytes[i] == 0) + StartingOffsets.Add(i + 1); + } + + if (StartingOffsets[^1] == bytes.Length) + StartingOffsets.RemoveAt(StartingOffsets.Count - 1); + else + Data.WriteByte(0); + } + + public string GetString(int offset, int size) + => Encoding.UTF8.GetString(Data.GetBuffer().AsSpan().Slice(offset, size)); + + public string GetNullTerminatedString(int offset) + { + var str = Data.GetBuffer().AsSpan()[offset..]; + var size = str.IndexOf((byte)0); + if (size >= 0) + str = str[..size]; + return Encoding.UTF8.GetString(str); + } + + public (int, int) FindOrAddString(string str) + { + var dataSpan = Data.GetBuffer().AsSpan(); + var bytes = Encoding.UTF8.GetBytes(str); + foreach (var offset in StartingOffsets) + { + if (offset + bytes.Length > Data.Length) + break; + + var strSpan = dataSpan[offset..]; + var match = true; + for (var i = 0; i < bytes.Length; ++i) + { + if (strSpan[i] != bytes[i]) + { + match = false; + break; + } + } + + if (match && strSpan[bytes.Length] == 0) + return (offset, bytes.Length); + } + + Data.Seek(0L, SeekOrigin.End); + var newOffset = (int)Data.Position; + StartingOffsets.Add(newOffset); + Data.Write(bytes); + Data.WriteByte(0); + return (newOffset, bytes.Length); + } + } +} diff --git a/Penumbra.GameData/Files/ShpkFile.Write.cs b/Penumbra.GameData/Files/ShpkFile.Write.cs index dd8b70ac..117ea5e5 100644 --- a/Penumbra.GameData/Files/ShpkFile.Write.cs +++ b/Penumbra.GameData/Files/ShpkFile.Write.cs @@ -8,21 +8,19 @@ public partial class ShpkFile public byte[] Write() { if (SubViewKeys.Length != 2) - { throw new InvalidDataException(); - } - using var stream = new MemoryStream(); - using var blobs = new MemoryStream(); - var strings = new StringPool(ReadOnlySpan.Empty); + using var stream = new MemoryStream(); + using var blobs = new MemoryStream(); + var strings = new StringPool(ReadOnlySpan.Empty); using (var w = new BinaryWriter(stream)) { w.Write(ShPkMagic); w.Write(Version); w.Write(DirectXVersion switch { - DXVersion.DirectX9 => DX9Magic, - DXVersion.DirectX11 => DX11Magic, + DxVersion.DirectX9 => Dx9Magic, + DxVersion.DirectX11 => Dx11Magic, _ => throw new NotImplementedException(), }); var offsetsPosition = stream.Position; @@ -35,7 +33,7 @@ public partial class ShpkFile w.Write((uint)MaterialParams.Length); w.Write((uint)Constants.Length); w.Write((uint)Samplers.Length); - w.Write((uint)UAVs.Length); + w.Write((uint)Uavs.Length); w.Write((uint)SystemKeys.Length); w.Write((uint)SceneKeys.Length); w.Write((uint)MaterialKeys.Length); @@ -43,7 +41,7 @@ public partial class ShpkFile w.Write((uint)Items.Length); WriteShaderArray(w, VertexShaders, blobs, strings); - WriteShaderArray(w, PixelShaders, blobs, strings); + WriteShaderArray(w, PixelShaders, blobs, strings); foreach (var materialParam in MaterialParams) { @@ -53,54 +51,50 @@ public partial class ShpkFile } WriteResourceArray(w, Constants, strings); - WriteResourceArray(w, Samplers, strings); - WriteResourceArray(w, UAVs, strings); + WriteResourceArray(w, Samplers, strings); + WriteResourceArray(w, Uavs, strings); foreach (var key in SystemKeys) { w.Write(key.Id); w.Write(key.DefaultValue); } + foreach (var key in SceneKeys) { w.Write(key.Id); w.Write(key.DefaultValue); } + foreach (var key in MaterialKeys) { w.Write(key.Id); w.Write(key.DefaultValue); } + foreach (var key in SubViewKeys) - { w.Write(key.DefaultValue); - } foreach (var node in Nodes) { - if (node.PassIndices.Length != 16 || node.SystemKeys.Length != SystemKeys.Length || node.SceneKeys.Length != SceneKeys.Length || node.MaterialKeys.Length != MaterialKeys.Length || node.SubViewKeys.Length != SubViewKeys.Length) - { + if (node.PassIndices.Length != 16 + || node.SystemKeys.Length != SystemKeys.Length + || node.SceneKeys.Length != SceneKeys.Length + || node.MaterialKeys.Length != MaterialKeys.Length + || node.SubViewKeys.Length != SubViewKeys.Length) throw new InvalidDataException(); - } + w.Write(node.Id); w.Write(node.Passes.Length); w.Write(node.PassIndices); foreach (var key in node.SystemKeys) - { w.Write(key); - } foreach (var key in node.SceneKeys) - { w.Write(key); - } foreach (var key in node.MaterialKeys) - { w.Write(key); - } foreach (var key in node.SubViewKeys) - { w.Write(key); - } foreach (var pass in node.Passes) { w.Write(pass.Id); @@ -160,21 +154,12 @@ public partial class ShpkFile w.Write(blobSize); w.Write((ushort)shader.Constants.Length); w.Write((ushort)shader.Samplers.Length); - w.Write((ushort)shader.UAVs.Length); + w.Write((ushort)shader.Uavs.Length); w.Write((ushort)0); WriteResourceArray(w, shader.Constants, strings); - WriteResourceArray(w, shader.Samplers, strings); - WriteResourceArray(w, shader.UAVs, strings); - } - } - - private static void WriteUInt32PairArray(BinaryWriter w, (uint, uint)[] array) - { - foreach (var (first, second) in array) - { - w.Write(first); - w.Write(second); + WriteResourceArray(w, shader.Samplers, strings); + WriteResourceArray(w, shader.Uavs, strings); } } } diff --git a/Penumbra.GameData/Files/ShpkFile.cs b/Penumbra.GameData/Files/ShpkFile.cs index ebcd2e58..75651ef9 100644 --- a/Penumbra.GameData/Files/ShpkFile.cs +++ b/Penumbra.GameData/Files/ShpkFile.cs @@ -2,462 +2,74 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; -using Lumina.Data.Parsing; using Lumina.Extensions; -using Lumina.Misc; using Penumbra.GameData.Data; namespace Penumbra.GameData.Files; public partial class ShpkFile : IWritable { - public enum DXVersion : uint - { - DirectX9 = 9, - DirectX11 = 11, - } - - public struct Resource - { - public uint Id; - public string Name; - public ushort Slot; - public ushort Size; - public DisassembledShader.VectorComponents[]? Used; - public DisassembledShader.VectorComponents? UsedDynamically; - } - - public struct Shader - { - public DisassembledShader.ShaderStage Stage; - public DXVersion DirectXVersion; - public Resource[] Constants; - public Resource[] Samplers; - public Resource[] UAVs; - public byte[] AdditionalHeader; - private byte[] _blob; - private DisassembledShader? _disassembly; - - public byte[] Blob - { - get => _blob; - set - { - if (_blob == value) - { - return; - } - if (Stage != DisassembledShader.ShaderStage.Unspecified) - { - // Reject the blob entirely if we can't disassemble it or if we find inconsistencies. - var disasm = DisassembledShader.Disassemble(value); - if (disasm.Stage != Stage || (disasm.ShaderModel >> 8) + 6 != (uint)DirectXVersion) - { - throw new ArgumentException($"The supplied blob is a DirectX {(disasm.ShaderModel >> 8) + 6} {disasm.Stage} shader ; expected a DirectX {(uint)DirectXVersion} {Stage} shader.", nameof(value)); - } - if (disasm.ShaderModel >= 0x0500) - { - var samplers = new Dictionary(); - var textures = new Dictionary(); - foreach (var binding in disasm.ResourceBindings) - { - switch (binding.Type) - { - case DisassembledShader.ResourceType.Texture: - textures[binding.Slot] = NormalizeResourceName(binding.Name); - break; - case DisassembledShader.ResourceType.Sampler: - samplers[binding.Slot] = NormalizeResourceName(binding.Name); - break; - } - } - if (samplers.Count != textures.Count || !samplers.All(pair => textures.TryGetValue(pair.Key, out var texName) && pair.Value == texName)) - { - throw new ArgumentException($"The supplied blob has inconsistent sampler and texture allocation."); - } - } - _blob = value; - _disassembly = disasm; - } - else - { - _blob = value; - _disassembly = null; - } - UpdateUsed(); - } - } - - public DisassembledShader? Disassembly => _disassembly; - - public Resource? GetConstantById(uint id) - { - return Constants.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Id == id); - } - - public Resource? GetConstantByName(string name) - { - return Constants.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Name == name); - } - - public Resource? GetSamplerById(uint id) - { - return Samplers.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Id == id); - } - - public Resource? GetSamplerByName(string name) - { - return Samplers.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Name == name); - } - - public Resource? GetUAVById(uint id) - { - return UAVs.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Id == id); - } - - public Resource? GetUAVByName(string name) - { - return UAVs.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Name == name); - } - - public void UpdateResources(ShpkFile file) - { - if (_disassembly == null) - { - throw new InvalidOperationException(); - } - var constants = new List(); - var samplers = new List(); - var uavs = new List(); - foreach (var binding in _disassembly.ResourceBindings) - { - switch (binding.Type) - { - case DisassembledShader.ResourceType.ConstantBuffer: - var name = NormalizeResourceName(binding.Name); - // We want to preserve IDs as much as possible, and to deterministically generate new ones in a way that's most compliant with the native ones, to maximize compatibility. - var id = GetConstantByName(name)?.Id ?? file.GetConstantByName(name)?.Id ?? Crc32.Get(name, 0xFFFFFFFFu); - constants.Add(new Resource - { - Id = id, - Name = name, - Slot = (ushort)binding.Slot, - Size = (ushort)binding.RegisterCount, - Used = binding.Used, - UsedDynamically = binding.UsedDynamically, - }); - break; - case DisassembledShader.ResourceType.Texture: - name = NormalizeResourceName(binding.Name); - id = GetSamplerByName(name)?.Id ?? file.GetSamplerByName(name)?.Id ?? Crc32.Get(name, 0xFFFFFFFFu); - samplers.Add(new Resource - { - Id = id, - Name = name, - Slot = (ushort)binding.Slot, - Size = (ushort)binding.Slot, - Used = binding.Used, - UsedDynamically = binding.UsedDynamically, - }); - break; - case DisassembledShader.ResourceType.UAV: - name = NormalizeResourceName(binding.Name); - id = GetUAVByName(name)?.Id ?? file.GetUAVByName(name)?.Id ?? Crc32.Get(name, 0xFFFFFFFFu); - uavs.Add(new Resource - { - Id = id, - Name = name, - Slot = (ushort)binding.Slot, - Size = (ushort)binding.Slot, - Used = binding.Used, - UsedDynamically = binding.UsedDynamically, - }); - break; - } - } - Constants = constants.ToArray(); - Samplers = samplers.ToArray(); - UAVs = uavs.ToArray(); - } - - private void UpdateUsed() - { - if (_disassembly != null) - { - var cbUsage = new Dictionary(); - var tUsage = new Dictionary(); - var uUsage = new Dictionary(); - foreach (var binding in _disassembly.ResourceBindings) - { - switch (binding.Type) - { - case DisassembledShader.ResourceType.ConstantBuffer: - cbUsage[NormalizeResourceName(binding.Name)] = (binding.Used, binding.UsedDynamically); - break; - case DisassembledShader.ResourceType.Texture: - tUsage[NormalizeResourceName(binding.Name)] = (binding.Used, binding.UsedDynamically); - break; - case DisassembledShader.ResourceType.UAV: - uUsage[NormalizeResourceName(binding.Name)] = (binding.Used, binding.UsedDynamically); - break; - } - } - static void CopyUsed(Resource[] resources, Dictionary used) - { - for (var i = 0; i < resources.Length; ++i) - { - if (used.TryGetValue(resources[i].Name, out var usage)) - { - resources[i].Used = usage.Item1; - resources[i].UsedDynamically = usage.Item2; - } - else - { - resources[i].Used = null; - resources[i].UsedDynamically = null; - } - } - } - CopyUsed(Constants, cbUsage); - CopyUsed(Samplers, tUsage); - CopyUsed(UAVs, uUsage); - } - else - { - ClearUsed(Constants); - ClearUsed(Samplers); - ClearUsed(UAVs); - } - } - - private static string NormalizeResourceName(string resourceName) - { - var dot = resourceName.IndexOf('.'); - if (dot >= 0) - { - return resourceName[..dot]; - } - else if (resourceName.EndsWith("_S") || resourceName.EndsWith("_T")) - { - return resourceName[..^2]; - } - else - { - return resourceName; - } - } - } - - public struct MaterialParam - { - public uint Id; - public ushort ByteOffset; - public ushort ByteSize; - } - - public struct Pass - { - public uint Id; - public uint VertexShader; - public uint PixelShader; - } - - public struct Key - { - public uint Id; - public uint DefaultValue; - public uint[] Values; - } - - public struct Node - { - public uint Id; - public byte[] PassIndices; - public uint[] SystemKeys; - public uint[] SceneKeys; - public uint[] MaterialKeys; - public uint[] SubViewKeys; - public Pass[] Passes; - } - - public struct Item - { - public uint Id; - public uint Node; - } - - public class StringPool - { - public MemoryStream Data; - public List StartingOffsets; - - public StringPool(ReadOnlySpan bytes) - { - Data = new MemoryStream(); - Data.Write(bytes); - StartingOffsets = new List - { - 0, - }; - for (var i = 0; i < bytes.Length; ++i) - { - if (bytes[i] == 0) - { - StartingOffsets.Add(i + 1); - } - } - if (StartingOffsets[^1] == bytes.Length) - { - StartingOffsets.RemoveAt(StartingOffsets.Count - 1); - } - else - { - Data.WriteByte(0); - } - } - - public string GetString(int offset, int size) - { - return Encoding.UTF8.GetString(Data.GetBuffer().AsSpan().Slice(offset, size)); - } - - public string GetNullTerminatedString(int offset) - { - var str = Data.GetBuffer().AsSpan()[offset..]; - var size = str.IndexOf((byte)0); - if (size >= 0) - { - str = str[..size]; - } - return Encoding.UTF8.GetString(str); - } - - public (int, int) FindOrAddString(string str) - { - var dataSpan = Data.GetBuffer().AsSpan(); - var bytes = Encoding.UTF8.GetBytes(str); - foreach (var offset in StartingOffsets) - { - if (offset + bytes.Length > Data.Length) - { - break; - } - var strSpan = dataSpan[offset..]; - var match = true; - for (var i = 0; i < bytes.Length; ++i) - { - if (strSpan[i] != bytes[i]) - { - match = false; - break; - } - } - if (match && strSpan[bytes.Length] == 0) - { - return (offset, bytes.Length); - } - } - Data.Seek(0L, SeekOrigin.End); - var newOffset = (int)Data.Position; - StartingOffsets.Add(newOffset); - Data.Write(bytes); - Data.WriteByte(0); - return (newOffset, bytes.Length); - } - } - private const uint ShPkMagic = 0x6B506853u; // bytes of ShPk - private const uint DX9Magic = 0x00395844u; // bytes of DX9\0 - private const uint DX11Magic = 0x31315844u; // bytes of DX11 + private const uint Dx9Magic = 0x00395844u; // bytes of DX9\0 + private const uint Dx11Magic = 0x31315844u; // bytes of DX11 public const uint MaterialParamsConstantId = 0x64D12851u; - public uint Version; - public DXVersion DirectXVersion; - public Shader[] VertexShaders; - public Shader[] PixelShaders; - public uint MaterialParamsSize; + public uint Version; + public DxVersion DirectXVersion; + public Shader[] VertexShaders; + public Shader[] PixelShaders; + public uint MaterialParamsSize; public MaterialParam[] MaterialParams; - public Resource[] Constants; - public Resource[] Samplers; - public Resource[] UAVs; - public Key[] SystemKeys; - public Key[] SceneKeys; - public Key[] MaterialKeys; - public Key[] SubViewKeys; - public Node[] Nodes; - public Item[] Items; - public byte[] AdditionalData; + public Resource[] Constants; + public Resource[] Samplers; + public Resource[] Uavs; + public Key[] SystemKeys; + public Key[] SceneKeys; + public Key[] MaterialKeys; + public Key[] SubViewKeys; + public Node[] Nodes; + public Item[] Items; + public byte[] AdditionalData; - public bool Valid { get; private set; } + public bool Valid { get; private set; } private bool _changed; public MaterialParam? GetMaterialParamById(uint id) - { - return MaterialParams.Select(param => new MaterialParam?(param)).FirstOrDefault(param => param!.Value.Id == id); - } + => MaterialParams.FirstOrNull(m => m.Id == id); public Resource? GetConstantById(uint id) - { - return Constants.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Id == id); - } + => Constants.FirstOrNull(c => c.Id == id); public Resource? GetConstantByName(string name) - { - return Constants.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Name == name); - } + => Constants.FirstOrNull(c => c.Name == name); public Resource? GetSamplerById(uint id) - { - return Samplers.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Id == id); - } + => Samplers.FirstOrNull(s => s.Id == id); public Resource? GetSamplerByName(string name) - { - return Samplers.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Name == name); - } + => Samplers.FirstOrNull(s => s.Name == name); - public Resource? GetUAVById(uint id) - { - return UAVs.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Id == id); - } + public Resource? GetUavById(uint id) + => Uavs.FirstOrNull(u => u.Id == id); - public Resource? GetUAVByName(string name) - { - return UAVs.Select(res => new Resource?(res)).FirstOrDefault(res => res!.Value.Name == name); - } + public Resource? GetUavByName(string name) + => Uavs.FirstOrNull(u => u.Name == name); public Key? GetSystemKeyById(uint id) - { - return SystemKeys.Select(key => new Key?(key)).FirstOrDefault(key => key!.Value.Id == id); - } + => SystemKeys.FirstOrNull(k => k.Id == id); public Key? GetSceneKeyById(uint id) - { - return SceneKeys.Select(key => new Key?(key)).FirstOrDefault(key => key!.Value.Id == id); - } + => SceneKeys.FirstOrNull(k => k.Id == id); public Key? GetMaterialKeyById(uint id) - { - return MaterialKeys.Select(key => new Key?(key)).FirstOrDefault(key => key!.Value.Id == id); - } + => MaterialKeys.FirstOrNull(k => k.Id == id); public Node? GetNodeById(uint id) - { - return Nodes.Select(node => new Node?(node)).FirstOrDefault(node => node!.Value.Id == id); - } + => Nodes.FirstOrNull(n => n.Id == id); public Item? GetItemById(uint id) - { - return Items.Select(item => new Item?(item)).FirstOrDefault(item => item!.Value.Id == id); - } - - // Activator.CreateInstance can't use a ctor with a default value so this has to be made explicit - public ShpkFile(byte[] data) - : this(data, false) - { - } + => Items.FirstOrNull(i => i.Id == id); public ShpkFile(byte[] data, bool disassemble = false) { @@ -465,25 +77,23 @@ public partial class ShpkFile : IWritable using var r = new BinaryReader(stream); if (r.ReadUInt32() != ShPkMagic) - { throw new InvalidDataException(); - } + Version = r.ReadUInt32(); DirectXVersion = r.ReadUInt32() switch { - DX9Magic => DXVersion.DirectX9, - DX11Magic => DXVersion.DirectX11, + Dx9Magic => DxVersion.DirectX9, + Dx11Magic => DxVersion.DirectX11, _ => throw new InvalidDataException(), }; if (r.ReadUInt32() != data.Length) - { throw new InvalidDataException(); - } - var blobsOffset = r.ReadUInt32(); - var stringsOffset = r.ReadUInt32(); - var vertexShaderCount = r.ReadUInt32(); - var pixelShaderCount = r.ReadUInt32(); - MaterialParamsSize = r.ReadUInt32(); + + var blobsOffset = r.ReadUInt32(); + var stringsOffset = r.ReadUInt32(); + var vertexShaderCount = r.ReadUInt32(); + var pixelShaderCount = r.ReadUInt32(); + MaterialParamsSize = r.ReadUInt32(); var materialParamCount = r.ReadUInt32(); var constantCount = r.ReadUInt32(); var samplerCount = r.ReadUInt32(); @@ -497,99 +107,105 @@ public partial class ShpkFile : IWritable var blobs = new ReadOnlySpan(data, (int)blobsOffset, (int)(stringsOffset - blobsOffset)); var strings = new StringPool(new ReadOnlySpan(data, (int)stringsOffset, (int)(data.Length - stringsOffset))); - VertexShaders = ReadShaderArray(r, (int)vertexShaderCount, DisassembledShader.ShaderStage.Vertex, DirectXVersion, disassemble, blobs, strings); - PixelShaders = ReadShaderArray(r, (int)pixelShaderCount, DisassembledShader.ShaderStage.Pixel, DirectXVersion, disassemble, blobs, strings); + VertexShaders = ReadShaderArray(r, (int)vertexShaderCount, DisassembledShader.ShaderStage.Vertex, DirectXVersion, disassemble, blobs, + strings); + PixelShaders = ReadShaderArray(r, (int)pixelShaderCount, DisassembledShader.ShaderStage.Pixel, DirectXVersion, disassemble, blobs, + strings); MaterialParams = r.ReadStructuresAsArray((int)materialParamCount); - Constants = ReadResourceArray(r, (int)constantCount, strings); - Samplers = ReadResourceArray(r, (int)samplerCount, strings); - UAVs = ReadResourceArray(r, (int)uavCount, strings); + Constants = ReadResourceArray(r, (int)constantCount, strings); + Samplers = ReadResourceArray(r, (int)samplerCount, strings); + Uavs = ReadResourceArray(r, (int)uavCount, strings); - SystemKeys = ReadKeyArray(r, (int)systemKeyCount); - SceneKeys = ReadKeyArray(r, (int)sceneKeyCount); - MaterialKeys = ReadKeyArray(r, (int)materialKeyCount); + SystemKeys = ReadKeyArray(r, (int)systemKeyCount); + SceneKeys = ReadKeyArray(r, (int)sceneKeyCount); + MaterialKeys = ReadKeyArray(r, (int)materialKeyCount); - var subViewKey1Default = r.ReadUInt32(); - var subViewKey2Default = r.ReadUInt32(); + var subViewKey1Null = r.ReadUInt32(); + var subViewKey2Null = r.ReadUInt32(); - SubViewKeys = new Key[] { - new Key + SubViewKeys = new Key[] + { + new() { - Id = 1, - DefaultValue = subViewKey1Default, - Values = Array.Empty(), + Id = 1, + DefaultValue = subViewKey1Null, + Values = Array.Empty(), }, - new Key + new() { - Id = 2, - DefaultValue = subViewKey2Default, - Values = Array.Empty(), + Id = 2, + DefaultValue = subViewKey2Null, + Values = Array.Empty(), }, }; - Nodes = ReadNodeArray(r, (int)nodeCount, SystemKeys.Length, SceneKeys.Length, MaterialKeys.Length, SubViewKeys.Length); - Items = r.ReadStructuresAsArray((int)itemCount); + Nodes = ReadNodeArray(r, (int)nodeCount, SystemKeys.Length, SceneKeys.Length, MaterialKeys.Length, SubViewKeys.Length); + Items = r.ReadStructuresAsArray((int)itemCount); AdditionalData = r.ReadBytes((int)(blobsOffset - r.BaseStream.Position)); // This should be empty, but just in case. if (disassemble) - { UpdateUsed(); - } UpdateKeyValues(); - Valid = true; + Valid = true; _changed = false; } public void UpdateResources() { var constants = new Dictionary(); - var samplers = new Dictionary(); - var uavs = new Dictionary(); - static void CollectResources(Dictionary resources, Resource[] shaderResources, Func getExistingById, DisassembledShader.ResourceType type) + var samplers = new Dictionary(); + var uavs = new Dictionary(); + + static void CollectResources(Dictionary resources, Resource[] shaderResources, Func getExistingById, + DisassembledShader.ResourceType type) { foreach (var resource in shaderResources) { if (resources.TryGetValue(resource.Id, out var carry) && type != DisassembledShader.ResourceType.ConstantBuffer) - { continue; - } + var existing = getExistingById(resource.Id); resources[resource.Id] = new Resource { - Id = resource.Id, - Name = resource.Name, - Slot = existing?.Slot ?? (type == DisassembledShader.ResourceType.ConstantBuffer ? (ushort)65535 : (ushort)2), - Size = type == DisassembledShader.ResourceType.ConstantBuffer ? Math.Max(carry.Size, resource.Size) : (existing?.Size ?? 0), - Used = null, + Id = resource.Id, + Name = resource.Name, + Slot = existing?.Slot ?? (type == DisassembledShader.ResourceType.ConstantBuffer ? (ushort)65535 : (ushort)2), + Size = type == DisassembledShader.ResourceType.ConstantBuffer ? Math.Max(carry.Size, resource.Size) : existing?.Size ?? 0, + Used = null, UsedDynamically = null, }; } } + foreach (var shader in VertexShaders) { CollectResources(constants, shader.Constants, GetConstantById, DisassembledShader.ResourceType.ConstantBuffer); - CollectResources(samplers, shader.Samplers, GetSamplerById, DisassembledShader.ResourceType.Sampler); - CollectResources(uavs, shader.UAVs, GetUAVById, DisassembledShader.ResourceType.UAV); + CollectResources(samplers, shader.Samplers, GetSamplerById, DisassembledShader.ResourceType.Sampler); + CollectResources(uavs, shader.Uavs, GetUavById, DisassembledShader.ResourceType.Uav); } + foreach (var shader in PixelShaders) { CollectResources(constants, shader.Constants, GetConstantById, DisassembledShader.ResourceType.ConstantBuffer); - CollectResources(samplers, shader.Samplers, GetSamplerById, DisassembledShader.ResourceType.Sampler); - CollectResources(uavs, shader.UAVs, GetUAVById, DisassembledShader.ResourceType.UAV); + CollectResources(samplers, shader.Samplers, GetSamplerById, DisassembledShader.ResourceType.Sampler); + CollectResources(uavs, shader.Uavs, GetUavById, DisassembledShader.ResourceType.Uav); } + Constants = constants.Values.ToArray(); - Samplers = samplers.Values.ToArray(); - UAVs = uavs.Values.ToArray(); - UpdateUsed(); + Samplers = samplers.Values.ToArray(); + Uavs = uavs.Values.ToArray(); + UpdateUsed(); + + // Ceil required size to a multiple of 16 bytes. + // Offsets can be skipped, MaterialParamsConstantId's size is the count. MaterialParamsSize = (GetConstantById(MaterialParamsConstantId)?.Size ?? 0u) << 4; foreach (var param in MaterialParams) - { MaterialParamsSize = Math.Max(MaterialParamsSize, (uint)param.ByteOffset + param.ByteSize); - } MaterialParamsSize = (MaterialParamsSize + 0xFu) & ~0xFu; } @@ -598,55 +214,59 @@ public partial class ShpkFile : IWritable var cUsage = new Dictionary(); var sUsage = new Dictionary(); var uUsage = new Dictionary(); - static void CollectUsed(Dictionary usage, Resource[] resources) + + static void CollectUsed(Dictionary usage, + Resource[] resources) { foreach (var resource in resources) { if (resource.Used == null) - { continue; - } + usage.TryGetValue(resource.Id, out var carry); carry.Item1 ??= Array.Empty(); var combined = new DisassembledShader.VectorComponents[Math.Max(carry.Item1.Length, resource.Used.Length)]; for (var i = 0; i < combined.Length; ++i) - { combined[i] = (i < carry.Item1.Length ? carry.Item1[i] : 0) | (i < resource.Used.Length ? resource.Used[i] : 0); - } usage[resource.Id] = (combined, carry.Item2 | (resource.UsedDynamically ?? 0)); } } - static void CopyUsed(Resource[] resources, Dictionary used) + + static void CopyUsed(Resource[] resources, + Dictionary used) { for (var i = 0; i < resources.Length; ++i) { if (used.TryGetValue(resources[i].Id, out var usage)) { - resources[i].Used = usage.Item1; + resources[i].Used = usage.Item1; resources[i].UsedDynamically = usage.Item2; } else { - resources[i].Used = null; + resources[i].Used = null; resources[i].UsedDynamically = null; } } } + foreach (var shader in VertexShaders) { CollectUsed(cUsage, shader.Constants); CollectUsed(sUsage, shader.Samplers); - CollectUsed(uUsage, shader.UAVs); + CollectUsed(uUsage, shader.Uavs); } + foreach (var shader in PixelShaders) { CollectUsed(cUsage, shader.Constants); CollectUsed(sUsage, shader.Samplers); - CollectUsed(uUsage, shader.UAVs); + CollectUsed(uUsage, shader.Uavs); } + CopyUsed(Constants, cUsage); - CopyUsed(Samplers, sUsage); - CopyUsed(UAVs, uUsage); + CopyUsed(Samplers, sUsage); + CopyUsed(Uavs, uUsage); } public void UpdateKeyValues() @@ -656,46 +276,42 @@ public partial class ShpkFile : IWritable { key.DefaultValue, }); + static void CollectValues(HashSet[] valueSets, uint[] values) { for (var i = 0; i < valueSets.Length; ++i) - { valueSets[i].Add(values[i]); - } } + static void CopyValues(Key[] keys, HashSet[] valueSets) { for (var i = 0; i < keys.Length; ++i) - { keys[i].Values = valueSets[i].ToArray(); - } } - var systemKeyValues = InitializeValueSet(SystemKeys); - var sceneKeyValues = InitializeValueSet(SceneKeys); + + var systemKeyValues = InitializeValueSet(SystemKeys); + var sceneKeyValues = InitializeValueSet(SceneKeys); var materialKeyValues = InitializeValueSet(MaterialKeys); - var subViewKeyValues = InitializeValueSet(SubViewKeys); + var subViewKeyValues = InitializeValueSet(SubViewKeys); foreach (var node in Nodes) { - CollectValues(systemKeyValues, node.SystemKeys); - CollectValues(sceneKeyValues, node.SceneKeys); + CollectValues(systemKeyValues, node.SystemKeys); + CollectValues(sceneKeyValues, node.SceneKeys); CollectValues(materialKeyValues, node.MaterialKeys); - CollectValues(subViewKeyValues, node.SubViewKeys); + CollectValues(subViewKeyValues, node.SubViewKeys); } - CopyValues(SystemKeys, systemKeyValues); - CopyValues(SceneKeys, sceneKeyValues); + + CopyValues(SystemKeys, systemKeyValues); + CopyValues(SceneKeys, sceneKeyValues); CopyValues(MaterialKeys, materialKeyValues); - CopyValues(SubViewKeys, subViewKeyValues); + CopyValues(SubViewKeys, subViewKeyValues); } public void SetInvalid() - { - Valid = false; - } + => Valid = false; public void SetChanged() - { - _changed = true; - } + => _changed = true; public bool IsChanged() { @@ -708,7 +324,7 @@ public partial class ShpkFile : IWritable { for (var i = 0; i < resources.Length; ++i) { - resources[i].Used = null; + resources[i].Used = null; resources[i].UsedDynamically = null; } } @@ -718,29 +334,30 @@ public partial class ShpkFile : IWritable var ret = new Resource[count]; for (var i = 0; i < count; ++i) { - var buf = new Resource(); - - buf.Id = r.ReadUInt32(); + var id = r.ReadUInt32(); var strOffset = r.ReadUInt32(); var strSize = r.ReadUInt32(); - buf.Name = strings.GetString((int)strOffset, (int)strSize); - buf.Slot = r.ReadUInt16(); - buf.Size = r.ReadUInt16(); - - ret[i] = buf; + ret[i] = new Resource + { + Id = id, + Name = strings.GetString((int)strOffset, (int)strSize), + Slot = r.ReadUInt16(), + Size = r.ReadUInt16(), + }; } return ret; } - private static Shader[] ReadShaderArray(BinaryReader r, int count, DisassembledShader.ShaderStage stage, DXVersion directX, bool disassemble, ReadOnlySpan blobs, StringPool strings) + private static Shader[] ReadShaderArray(BinaryReader r, int count, DisassembledShader.ShaderStage stage, DxVersion directX, + bool disassemble, ReadOnlySpan blobs, StringPool strings) { var extraHeaderSize = stage switch { DisassembledShader.ShaderStage.Vertex => directX switch { - DXVersion.DirectX9 => 4, - DXVersion.DirectX11 => 8, + DxVersion.DirectX9 => 4, + DxVersion.DirectX11 => 8, _ => throw new NotImplementedException(), }, _ => 0, @@ -755,23 +372,20 @@ public partial class ShpkFile : IWritable var samplerCount = r.ReadUInt16(); var uavCount = r.ReadUInt16(); if (r.ReadUInt16() != 0) - { throw new NotImplementedException(); - } var rawBlob = blobs.Slice((int)blobOffset, (int)blobSize); - var shader = new Shader(); - - shader.Stage = disassemble ? stage : DisassembledShader.ShaderStage.Unspecified; - shader.DirectXVersion = directX; - shader.Constants = ReadResourceArray(r, constantCount, strings); - shader.Samplers = ReadResourceArray(r, samplerCount, strings); - shader.UAVs = ReadResourceArray(r, uavCount, strings); - shader.AdditionalHeader = rawBlob[..extraHeaderSize].ToArray(); - shader.Blob = rawBlob[extraHeaderSize..].ToArray(); - - ret[i] = shader; + ret[i] = new Shader + { + Stage = disassemble ? stage : DisassembledShader.ShaderStage.Unspecified, + DirectXVersion = directX, + Constants = ReadResourceArray(r, constantCount, strings), + Samplers = ReadResourceArray(r, samplerCount, strings), + Uavs = ReadResourceArray(r, uavCount, strings), + AdditionalHeader = rawBlob[..extraHeaderSize].ToArray(), + Blob = rawBlob[extraHeaderSize..].ToArray(), + }; } return ret; @@ -782,46 +396,91 @@ public partial class ShpkFile : IWritable var ret = new Key[count]; for (var i = 0; i < count; ++i) { - var id = r.ReadUInt32(); - var defaultValue = r.ReadUInt32(); - ret[i] = new Key { - Id = id, - DefaultValue = defaultValue, - Values = Array.Empty(), + Id = r.ReadUInt32(), + DefaultValue = r.ReadUInt32(), + Values = Array.Empty(), }; } return ret; } - private static Node[] ReadNodeArray(BinaryReader r, int count, int systemKeyCount, int sceneKeyCount, int materialKeyCount, int subViewKeyCount) + private static Node[] ReadNodeArray(BinaryReader r, int count, int systemKeyCount, int sceneKeyCount, int materialKeyCount, + int subViewKeyCount) { var ret = new Node[count]; for (var i = 0; i < count; ++i) { - var id = r.ReadUInt32(); - var passCount = r.ReadUInt32(); - var passIndices = r.ReadBytes(16); - var systemKeys = r.ReadStructuresAsArray(systemKeyCount); - var sceneKeys = r.ReadStructuresAsArray(sceneKeyCount); - var materialKeys = r.ReadStructuresAsArray(materialKeyCount); - var subViewKeys = r.ReadStructuresAsArray(subViewKeyCount); - var passes = r.ReadStructuresAsArray((int)passCount); - + var id = r.ReadUInt32(); + var passCount = r.ReadUInt32(); ret[i] = new Node { Id = id, - PassIndices = passIndices, - SystemKeys = systemKeys, - SceneKeys = sceneKeys, - MaterialKeys = materialKeys, - SubViewKeys = subViewKeys, - Passes = passes, + PassIndices = r.ReadBytes(16), + SystemKeys = r.ReadStructuresAsArray(systemKeyCount), + SceneKeys = r.ReadStructuresAsArray(sceneKeyCount), + MaterialKeys = r.ReadStructuresAsArray(materialKeyCount), + SubViewKeys = r.ReadStructuresAsArray(subViewKeyCount), + Passes = r.ReadStructuresAsArray((int)passCount), }; } return ret; } + + public enum DxVersion : uint + { + DirectX9 = 9, + DirectX11 = 11, + } + + public struct Resource + { + public uint Id; + public string Name; + public ushort Slot; + public ushort Size; + public DisassembledShader.VectorComponents[]? Used; + public DisassembledShader.VectorComponents? UsedDynamically; + } + + public struct MaterialParam + { + public uint Id; + public ushort ByteOffset; + public ushort ByteSize; + } + + public struct Pass + { + public uint Id; + public uint VertexShader; + public uint PixelShader; + } + + public struct Key + { + public uint Id; + public uint DefaultValue; + public uint[] Values; + } + + public struct Node + { + public uint Id; + public byte[] PassIndices; + public uint[] SystemKeys; + public uint[] SceneKeys; + public uint[] MaterialKeys; + public uint[] SubViewKeys; + public Pass[] Passes; + } + + public struct Item + { + public uint Id; + public uint Node; + } } diff --git a/Penumbra.GameData/UtilityFunctions.cs b/Penumbra.GameData/UtilityFunctions.cs new file mode 100644 index 00000000..71365c09 --- /dev/null +++ b/Penumbra.GameData/UtilityFunctions.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace Penumbra.GameData; + +public static class UtilityFunctions +{ + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public static T? FirstOrNull(this IEnumerable values, Func predicate) where T : struct + => values.Cast().FirstOrDefault(v => predicate(v!.Value)); +} diff --git a/Penumbra/UI/Classes/ModEditWindow.FileEditor.cs b/Penumbra/UI/Classes/ModEditWindow.FileEditor.cs index 9f4b4562..d3a91f86 100644 --- a/Penumbra/UI/Classes/ModEditWindow.FileEditor.cs +++ b/Penumbra/UI/Classes/ModEditWindow.FileEditor.cs @@ -1,8 +1,8 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Numerics; +using System.Reflection; using Dalamud.Interface; using Dalamud.Interface.ImGuiFileDialog; using ImGuiNET; @@ -11,7 +11,6 @@ using OtterGui.Raii; using Penumbra.GameData.Files; using Penumbra.Mods; using Penumbra.String.Classes; -using SixLabors.ImageSharp.PixelFormats; namespace Penumbra.UI.Classes; @@ -176,9 +175,7 @@ public partial class ModEditWindow } private static T? DefaultParseFile( byte[] bytes ) - { - return Activator.CreateInstance( typeof( T ), bytes ) as T; - } + => Activator.CreateInstance( typeof( T ), BindingFlags.CreateInstance | BindingFlags.OptionalParamBinding, bytes ) as T; private void UpdateCurrentFile( Mod.Editor.FileRegistry path ) { diff --git a/Penumbra/UI/Classes/ModEditWindow.Materials.cs b/Penumbra/UI/Classes/ModEditWindow.Materials.cs index 41e513d6..d9354807 100644 --- a/Penumbra/UI/Classes/ModEditWindow.Materials.cs +++ b/Penumbra/UI/Classes/ModEditWindow.Materials.cs @@ -11,6 +11,7 @@ using Dalamud.Interface.Internal.Notifications; using ImGuiNET; using Lumina.Data.Parsing; using OtterGui; +using OtterGui.Classes; using OtterGui.Raii; using Penumbra.GameData.Files; using Penumbra.String.Classes; diff --git a/Penumbra/UI/Classes/ModEditWindow.ShaderPackages.cs b/Penumbra/UI/Classes/ModEditWindow.ShaderPackages.cs index c117c0e6..4f5e72ba 100644 --- a/Penumbra/UI/Classes/ModEditWindow.ShaderPackages.cs +++ b/Penumbra/UI/Classes/ModEditWindow.ShaderPackages.cs @@ -10,6 +10,7 @@ using ImGuiNET; using Lumina.Misc; using OtterGui.Raii; using OtterGui; +using OtterGui.Classes; using Penumbra.GameData.Data; using Penumbra.GameData.Files; using Penumbra.Util; @@ -80,8 +81,8 @@ public partial class ModEditWindow { var extension = file.DirectXVersion switch { - ShpkFile.DXVersion.DirectX9 => ".cso", - ShpkFile.DXVersion.DirectX11 => ".dxbc", + ShpkFile.DxVersion.DirectX9 => ".cso", + ShpkFile.DxVersion.DirectX11 => ".dxbc", _ => throw new NotImplementedException(), }; var defaultName = new string( objectName.Where( char.IsUpper ).ToArray() ).ToLower() + idx.ToString(); @@ -148,7 +149,7 @@ public partial class ModEditWindow ret |= DrawShaderPackageResourceArray( "Constant Buffers", "slot", true, shader.Constants, true ); ret |= DrawShaderPackageResourceArray( "Samplers", "slot", false, shader.Samplers, true ); - ret |= DrawShaderPackageResourceArray( "Unordered Access Views", "slot", true, shader.UAVs, true ); + ret |= DrawShaderPackageResourceArray( "Unordered Access Views", "slot", true, shader.Uavs, true ); if( shader.AdditionalHeader.Length > 0 ) { @@ -506,7 +507,7 @@ public partial class ModEditWindow ret |= DrawShaderPackageResourceArray( "Constant Buffers", "type", true, file.Constants, disabled ); ret |= DrawShaderPackageResourceArray( "Samplers", "type", false, file.Samplers, disabled ); - ret |= DrawShaderPackageResourceArray( "Unordered Access Views", "type", false, file.UAVs, disabled ); + ret |= DrawShaderPackageResourceArray( "Unordered Access Views", "type", false, file.Uavs, disabled ); static bool DrawKeyArray( string arrayName, bool withId, ShpkFile.Key[] keys, bool _ ) {