Some formatting and naming changes, splitting files and some minor improvements.

This commit is contained in:
Ottermandias 2023-02-23 14:48:46 +01:00
parent 1e471551d4
commit a2b62a8b6a
13 changed files with 928 additions and 997 deletions

View file

@ -4,23 +4,22 @@ using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Penumbra.GameData.Interop;
using static Penumbra.GameData.Files.ShpkFile;
namespace Penumbra.GameData.Data;
public class DisassembledShader
public partial class DisassembledShader
{
public struct ResourceBinding
{
public string Name;
public ResourceType Type;
public Format Format;
public ResourceDimension Dimension;
public uint Slot;
public uint Elements;
public uint RegisterCount;
public string Name;
public ResourceType Type;
public Format Format;
public ResourceDimension Dimension;
public uint Slot;
public uint Elements;
public uint RegisterCount;
public VectorComponents[] Used;
public VectorComponents UsedDynamically;
public VectorComponents UsedDynamically;
}
// Abbreviated using the uppercased first char of their name
@ -30,7 +29,7 @@ public class DisassembledShader
ConstantBuffer = 0x43, // 'C'
Sampler = 0x53, // 'S'
Texture = 0x54, // 'T'
UAV = 0x55, // 'U'
Uav = 0x55, // 'U'
}
// Abbreviated using the uppercased first and last char of their name
@ -56,22 +55,22 @@ public class DisassembledShader
public struct InputOutput
{
public string Name;
public uint Index;
public string Name;
public uint Index;
public VectorComponents Mask;
public uint Register;
public string SystemValue;
public Format Format;
public uint Register;
public string SystemValue;
public Format Format;
public VectorComponents Used;
}
[Flags]
public enum VectorComponents : byte
{
X = 1,
Y = 2,
Z = 4,
W = 8,
X = 1,
Y = 2,
Z = 4,
W = 8,
All = 15,
}
@ -82,21 +81,31 @@ public class DisassembledShader
Vertex = 0x56, // 'V'
}
private static readonly Regex ResourceBindingSizeRegex = new(@"\s(\w+)(?:\[\d+\])?;\s*//\s*Offset:\s*0\s*Size:\s*(\d+)$", RegexOptions.Multiline | RegexOptions.NonBacktracking);
private static readonly Regex SM3ConstantBufferUsageRegex = new(@"c(\d+)(?:\[([^\]]+)\])?(?:\.([wxyz]+))?", RegexOptions.NonBacktracking);
private static readonly Regex SM3TextureUsageRegex = new(@"^\s*texld\S*\s+[^,]+,[^,]+,\s*s(\d+)", RegexOptions.NonBacktracking);
private static readonly Regex SM5ConstantBufferUsageRegex = new(@"cb(\d+)\[([^\]]+)\]\.([wxyz]+)", RegexOptions.NonBacktracking);
private static readonly Regex SM5TextureUsageRegex = new(@"^\s*sample_\S*\s+[^.]+\.([wxyz]+),[^,]+,\s*t(\d+)\.([wxyz]+)", RegexOptions.NonBacktracking);
private static readonly char[] Digits = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };
[GeneratedRegex(@"\s(\w+)(?:\[\d+\])?;\s*//\s*Offset:\s*0\s*Size:\s*(\d+)$", RegexOptions.Multiline | RegexOptions.NonBacktracking)]
private static partial Regex ResourceBindingSizeRegex();
public readonly string RawDisassembly;
public readonly uint ShaderModel;
public readonly ShaderStage Stage;
public readonly string BufferDefinitions;
[GeneratedRegex(@"c(\d+)(?:\[([^\]]+)\])?(?:\.([wxyz]+))?", RegexOptions.NonBacktracking)]
private static partial Regex Sm3ConstantBufferUsageRegex();
[GeneratedRegex(@"^\s*texld\S*\s+[^,]+,[^,]+,\s*s(\d+)", RegexOptions.NonBacktracking)]
private static partial Regex Sm3TextureUsageRegex();
[GeneratedRegex(@"cb(\d+)\[([^\]]+)\]\.([wxyz]+)", RegexOptions.NonBacktracking)]
private static partial Regex Sm5ConstantBufferUsageRegex();
[GeneratedRegex(@"^\s*sample_\S*\s+[^.]+\.([wxyz]+),[^,]+,\s*t(\d+)\.([wxyz]+)", RegexOptions.NonBacktracking)]
private static partial Regex Sm5TextureUsageRegex();
private static readonly char[] Digits = Enumerable.Range(0, 10).Select(c => (char) ('0' + c)).ToArray();
public readonly string RawDisassembly;
public readonly uint ShaderModel;
public readonly ShaderStage Stage;
public readonly string BufferDefinitions;
public readonly ResourceBinding[] ResourceBindings;
public readonly InputOutput[] InputSignature;
public readonly InputOutput[] OutputSignature;
public readonly string[] Instructions;
public readonly InputOutput[] InputSignature;
public readonly InputOutput[] OutputSignature;
public readonly string[] Instructions;
public DisassembledShader(string rawDisassembly)
{
@ -104,49 +113,38 @@ public class DisassembledShader
var lines = rawDisassembly.Split('\n');
Instructions = Array.FindAll(lines, ln => !ln.StartsWith("//") && ln.Length > 0);
var shaderModel = Instructions[0].Trim().Split('_');
Stage = (ShaderStage)(byte)char.ToUpper(shaderModel[0][0]);
Stage = (ShaderStage)(byte)char.ToUpper(shaderModel[0][0]);
ShaderModel = (uint.Parse(shaderModel[1]) << 8) | uint.Parse(shaderModel[2]);
var header = PreParseHeader(lines.AsSpan()[..Array.IndexOf(lines, Instructions[0])]);
switch (ShaderModel >> 8)
{
case 3:
ParseSM3Header(header, out BufferDefinitions, out ResourceBindings, out InputSignature, out OutputSignature);
ParseSM3ResourceUsage(Instructions, ResourceBindings);
ParseSm3Header(header, out BufferDefinitions, out ResourceBindings, out InputSignature, out OutputSignature);
ParseSm3ResourceUsage(Instructions, ResourceBindings);
break;
case 5:
ParseSM5Header(header, out BufferDefinitions, out ResourceBindings, out InputSignature, out OutputSignature);
ParseSM5ResourceUsage(Instructions, ResourceBindings);
ParseSm5Header(header, out BufferDefinitions, out ResourceBindings, out InputSignature, out OutputSignature);
ParseSm5ResourceUsage(Instructions, ResourceBindings);
break;
default:
throw new NotImplementedException();
default: throw new NotImplementedException();
}
}
public ResourceBinding? GetResourceBindingByName(ResourceType type, string name)
{
return ResourceBindings.Select(binding => new ResourceBinding?(binding)).FirstOrDefault(binding => binding!.Value.Type == type && binding!.Value.Name == name);
}
=> ResourceBindings.FirstOrNull(b => b.Type == type && b.Name == name);
public ResourceBinding? GetResourceBindingBySlot(ResourceType type, uint slot)
{
return ResourceBindings.Select(binding => new ResourceBinding?(binding)).FirstOrDefault(binding => binding!.Value.Type == type && binding!.Value.Slot == slot);
}
=> ResourceBindings.FirstOrNull(b => b.Type == type && b.Slot == slot);
public static DisassembledShader Disassemble(ReadOnlySpan<byte> shaderBlob)
{
return new DisassembledShader(D3DCompiler.Disassemble(shaderBlob));
}
=> new(D3DCompiler.Disassemble(shaderBlob));
private static void ParseSM3Header(Dictionary<string, string[]> header, out string bufferDefinitions, out ResourceBinding[] resourceBindings, out InputOutput[] inputSignature, out InputOutput[] outputSignature)
private static void ParseSm3Header(Dictionary<string, string[]> header, out string bufferDefinitions,
out ResourceBinding[] resourceBindings, out InputOutput[] inputSignature, out InputOutput[] outputSignature)
{
if (header.TryGetValue("Parameters", out var rawParameters))
{
bufferDefinitions = string.Join('\n', rawParameters);
}
else
{
bufferDefinitions = string.Empty;
}
bufferDefinitions = header.TryGetValue("Parameters", out var rawParameters)
? string.Join('\n', rawParameters)
: string.Empty;
if (header.TryGetValue("Registers", out var rawRegisters))
{
var (_, registers) = ParseTable(rawRegisters);
@ -154,10 +152,8 @@ public class DisassembledShader
{
var type = (ResourceType)(byte)char.ToUpper(register[1][0]);
if (type == ResourceType.Sampler)
{
type = ResourceType.Texture;
}
uint size = uint.Parse(register[2]);
var size = uint.Parse(register[2]);
return new ResourceBinding
{
Name = register[0],
@ -175,14 +171,15 @@ public class DisassembledShader
{
resourceBindings = Array.Empty<ResourceBinding>();
}
inputSignature = Array.Empty<InputOutput>();
inputSignature = Array.Empty<InputOutput>();
outputSignature = Array.Empty<InputOutput>();
}
private static void ParseSM3ResourceUsage(string[] instructions, ResourceBinding[] resourceBindings)
private static void ParseSm3ResourceUsage(string[] instructions, ResourceBinding[] resourceBindings)
{
var cbIndices = new Dictionary<uint, int>();
var tIndices = new Dictionary<uint, int>();
var tIndices = new Dictionary<uint, int>();
{
var i = 0;
foreach (var binding in resourceBindings)
@ -191,14 +188,13 @@ public class DisassembledShader
{
case ResourceType.ConstantBuffer:
for (var j = 0u; j < binding.RegisterCount; j++)
{
cbIndices[binding.Slot + j] = i;
}
break;
case ResourceType.Texture:
tIndices[binding.Slot] = i;
break;
}
++i;
}
}
@ -206,43 +202,39 @@ public class DisassembledShader
{
var trimmed = instruction.Trim();
if (trimmed.StartsWith("def") || trimmed.StartsWith("dcl"))
{
continue;
}
foreach (Match cbMatch in SM3ConstantBufferUsageRegex.Matches(instruction))
foreach (Match cbMatch in Sm3ConstantBufferUsageRegex().Matches(instruction))
{
var buffer = uint.Parse(cbMatch.Groups[1].Value);
if (cbIndices.TryGetValue(buffer, out var i))
{
var swizzle = cbMatch.Groups[3].Success ? ParseVectorComponents(cbMatch.Groups[3].Value) : VectorComponents.All;
if (cbMatch.Groups[2].Success)
{
resourceBindings[i].UsedDynamically |= swizzle;
}
else
{
resourceBindings[i].Used[buffer - resourceBindings[i].Slot] |= swizzle;
}
}
}
var tMatch = SM3TextureUsageRegex.Match(instruction);
var tMatch = Sm3TextureUsageRegex().Match(instruction);
if (tMatch.Success)
{
var texture = uint.Parse(tMatch.Groups[1].Value);
if (tIndices.TryGetValue(texture, out var i))
{
resourceBindings[i].Used[0] = VectorComponents.All;
}
}
}
}
private static void ParseSM5Header(Dictionary<string, string[]> header, out string bufferDefinitions, out ResourceBinding[] resourceBindings, out InputOutput[] inputSignature, out InputOutput[] outputSignature)
private static void ParseSm5Header(Dictionary<string, string[]> header, out string bufferDefinitions,
out ResourceBinding[] resourceBindings, out InputOutput[] inputSignature, out InputOutput[] outputSignature)
{
if (header.TryGetValue("Resource Bindings", out var rawResBindings))
{
var (head, resBindings) = ParseTable(rawResBindings);
resourceBindings = Array.ConvertAll(resBindings, binding => {
resourceBindings = Array.ConvertAll(resBindings, binding =>
{
var type = (ResourceType)(byte)char.ToUpper(binding[1][0]);
return new ResourceBinding
{
@ -261,10 +253,11 @@ public class DisassembledShader
{
resourceBindings = Array.Empty<ResourceBinding>();
}
if (header.TryGetValue("Buffer Definitions", out var rawBufferDefs))
{
bufferDefinitions = string.Join('\n', rawBufferDefs);
foreach (Match match in ResourceBindingSizeRegex.Matches(bufferDefinitions))
foreach (Match match in ResourceBindingSizeRegex().Matches(bufferDefinitions))
{
var name = match.Groups[1].Value;
var bytesSize = uint.Parse(match.Groups[2].Value);
@ -272,7 +265,7 @@ public class DisassembledShader
if (pos >= 0)
{
resourceBindings[pos].RegisterCount = (bytesSize + 0xF) >> 4;
resourceBindings[pos].Used = new VectorComponents[resourceBindings[pos].RegisterCount];
resourceBindings[pos].Used = new VectorComponents[resourceBindings[pos].RegisterCount];
}
}
}
@ -281,30 +274,32 @@ public class DisassembledShader
bufferDefinitions = string.Empty;
}
static InputOutput ParseInputOutput(string[] inOut) => new()
{
Name = inOut[0],
Index = uint.Parse(inOut[1]),
Mask = ParseVectorComponents(inOut[2]),
Register = uint.Parse(inOut[3]),
SystemValue = string.Intern(inOut[4]),
Format = (Format)(((byte)char.ToUpper(inOut[5][0]) << 8) | (byte)char.ToUpper(inOut[5][^1])),
Used = ParseVectorComponents(inOut[6]),
};
static InputOutput ParseInputOutput(string[] inOut)
=> new()
{
Name = inOut[0],
Index = uint.Parse(inOut[1]),
Mask = ParseVectorComponents(inOut[2]),
Register = uint.Parse(inOut[3]),
SystemValue = string.Intern(inOut[4]),
Format = (Format)(((byte)char.ToUpper(inOut[5][0]) << 8) | (byte)char.ToUpper(inOut[5][^1])),
Used = ParseVectorComponents(inOut[6]),
};
if (header.TryGetValue("Input signature", out var rawInputSig))
{
var (_, inputSig) = ParseTable(rawInputSig);
inputSignature = Array.ConvertAll(inputSig, ParseInputOutput);
inputSignature = Array.ConvertAll(inputSig, ParseInputOutput);
}
else
{
inputSignature = Array.Empty<InputOutput>();
}
if (header.TryGetValue("Output signature", out var rawOutputSig))
{
var (_, outputSig) = ParseTable(rawOutputSig);
outputSignature = Array.ConvertAll(outputSig, ParseInputOutput);
outputSignature = Array.ConvertAll(outputSig, ParseInputOutput);
}
else
{
@ -312,10 +307,10 @@ public class DisassembledShader
}
}
private static void ParseSM5ResourceUsage(string[] instructions, ResourceBinding[] resourceBindings)
private static void ParseSm5ResourceUsage(string[] instructions, ResourceBinding[] resourceBindings)
{
var cbIndices = new Dictionary<uint, int>();
var tIndices = new Dictionary<uint, int>();
var tIndices = new Dictionary<uint, int>();
{
var i = 0;
foreach (var binding in resourceBindings)
@ -329,6 +324,7 @@ public class DisassembledShader
tIndices[binding.Slot] = i;
break;
}
++i;
}
}
@ -336,10 +332,9 @@ public class DisassembledShader
{
var trimmed = instruction.Trim();
if (trimmed.StartsWith("def") || trimmed.StartsWith("dcl"))
{
continue;
}
foreach (Match cbMatch in SM5ConstantBufferUsageRegex.Matches(instruction))
foreach (Match cbMatch in Sm5ConstantBufferUsageRegex().Matches(instruction))
{
var buffer = uint.Parse(cbMatch.Groups[1].Value);
if (cbIndices.TryGetValue(buffer, out var i))
@ -348,9 +343,7 @@ public class DisassembledShader
if (int.TryParse(cbMatch.Groups[2].Value, out var vector))
{
if (vector < resourceBindings[i].Used.Length)
{
resourceBindings[i].Used[vector] |= swizzle;
}
}
else
{
@ -358,31 +351,24 @@ public class DisassembledShader
}
}
}
var tMatch = SM5TextureUsageRegex.Match(instruction);
var tMatch = Sm5TextureUsageRegex().Match(instruction);
if (tMatch.Success)
{
var texture = uint.Parse(tMatch.Groups[2].Value);
if (tIndices.TryGetValue(texture, out var i))
{
var outSwizzle = ParseVectorComponents(tMatch.Groups[1].Value);
var outSwizzle = ParseVectorComponents(tMatch.Groups[1].Value);
var rawInSwizzle = tMatch.Groups[3].Value;
var inSwizzle = new StringBuilder(4);
var inSwizzle = new StringBuilder(4);
if ((outSwizzle & VectorComponents.X) != 0)
{
inSwizzle.Append(rawInSwizzle[0]);
}
if ((outSwizzle & VectorComponents.Y) != 0)
{
inSwizzle.Append(rawInSwizzle[1]);
}
if ((outSwizzle & VectorComponents.Z) != 0)
{
inSwizzle.Append(rawInSwizzle[2]);
}
if ((outSwizzle & VectorComponents.W) != 0)
{
inSwizzle.Append(rawInSwizzle[3]);
}
resourceBindings[i].Used[0] |= ParseVectorComponents(inSwizzle.ToString());
}
}
@ -393,9 +379,9 @@ public class DisassembledShader
{
components = components.ToUpperInvariant();
return (components.Contains('X') ? VectorComponents.X : 0)
| (components.Contains('Y') ? VectorComponents.Y : 0)
| (components.Contains('Z') ? VectorComponents.Z : 0)
| (components.Contains('W') ? VectorComponents.W : 0);
| (components.Contains('Y') ? VectorComponents.Y : 0)
| (components.Contains('Z') ? VectorComponents.Z : 0)
| (components.Contains('W') ? VectorComponents.W : 0);
}
private static Dictionary<string, string[]> PreParseHeader(ReadOnlySpan<string> header)
@ -405,17 +391,13 @@ public class DisassembledShader
void AddSection(string name, ReadOnlySpan<string> section)
{
while (section.Length > 0 && section[0].Length <= 3)
{
section = section[1..];
}
while (section.Length > 0 && section[^1].Length <= 3)
{
section = section[..^1];
}
sections.Add(name, Array.ConvertAll(section.ToArray(), ln => ln.Length <= 3 ? string.Empty : ln[3..]));
}
var lastSectionName = "";
var lastSectionName = "";
var lastSectionStart = 0;
for (var i = 1; i < header.Length - 1; ++i)
{
@ -423,11 +405,12 @@ public class DisassembledShader
if (header[i - 1].Length <= 3 && header[i + 1].Length <= 3 && (current = header[i].TrimEnd()).EndsWith(':'))
{
AddSection(lastSectionName, header[lastSectionStart..(i - 1)]);
lastSectionName = current[3..^1];
lastSectionName = current[3..^1];
lastSectionStart = i + 2;
++i; // The next line cannot match
}
}
AddSection(lastSectionName, header[lastSectionStart..]);
return sections;
@ -442,9 +425,8 @@ public class DisassembledShader
{
var start = dashLine.IndexOf('-', i);
if (start < 0)
{
break;
}
var end = dashLine.IndexOf(' ', start + 1);
if (end < 0)
{
@ -462,20 +444,17 @@ public class DisassembledShader
{
var headerLine = lines[0];
for (var i = 0; i < columns.Count; ++i)
{
headers[i] = headerLine[columns[i]].Trim();
}
}
var data = new List<string[]>();
foreach (var line in lines[2..])
{
var row = new string[columns.Count];
for (var i = 0; i < columns.Count; ++i)
{
row[i] = line[columns[i]].Trim();
}
data.Add(row);
}
return (headers, data.ToArray());
}
}

View file

@ -0,0 +1,90 @@
using System;
using System.Collections;
using System.Collections.Generic;
namespace Penumbra.GameData.Files;
public partial class MtrlFile
{
public unsafe struct ColorDyeSet
{
public struct Row
{
private ushort _data;
public ushort Template
{
get => (ushort)(_data >> 5);
set => _data = (ushort)((_data & 0x1F) | (value << 5));
}
public bool Diffuse
{
get => (_data & 0x01) != 0;
set => _data = (ushort)(value ? _data | 0x01 : _data & 0xFFFE);
}
public bool Specular
{
get => (_data & 0x02) != 0;
set => _data = (ushort)(value ? _data | 0x02 : _data & 0xFFFD);
}
public bool Emissive
{
get => (_data & 0x04) != 0;
set => _data = (ushort)(value ? _data | 0x04 : _data & 0xFFFB);
}
public bool Gloss
{
get => (_data & 0x08) != 0;
set => _data = (ushort)(value ? _data | 0x08 : _data & 0xFFF7);
}
public bool SpecularStrength
{
get => (_data & 0x10) != 0;
set => _data = (ushort)(value ? _data | 0x10 : _data & 0xFFEF);
}
}
public struct RowArray : IEnumerable<Row>
{
public const int NumRows = 16;
private fixed ushort _rowData[NumRows];
public ref Row this[int i]
{
get
{
fixed (ushort* ptr = _rowData)
{
return ref ((Row*)ptr)[i];
}
}
}
public IEnumerator<Row> GetEnumerator()
{
for (var i = 0; i < NumRows; ++i)
yield return this[i];
}
IEnumerator IEnumerable.GetEnumerator()
=> GetEnumerator();
public ReadOnlySpan<byte> AsBytes()
{
fixed (ushort* ptr = _rowData)
{
return new ReadOnlySpan<byte>(ptr, NumRows * sizeof(ushort));
}
}
}
public RowArray Rows;
public string Name;
public ushort Index;
}
}

View file

@ -0,0 +1,135 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Numerics;
namespace Penumbra.GameData.Files;
public partial class MtrlFile
{
public unsafe struct ColorSet
{
public struct Row
{
public const int Size = 32;
private fixed ushort _data[16];
public Vector3 Diffuse
{
get => new(ToFloat(0), ToFloat(1), ToFloat(2));
set
{
_data[0] = FromFloat(value.X);
_data[1] = FromFloat(value.Y);
_data[2] = FromFloat(value.Z);
}
}
public Vector3 Specular
{
get => new(ToFloat(4), ToFloat(5), ToFloat(6));
set
{
_data[4] = FromFloat(value.X);
_data[5] = FromFloat(value.Y);
_data[6] = FromFloat(value.Z);
}
}
public Vector3 Emissive
{
get => new(ToFloat(8), ToFloat(9), ToFloat(10));
set
{
_data[8] = FromFloat(value.X);
_data[9] = FromFloat(value.Y);
_data[10] = FromFloat(value.Z);
}
}
public Vector2 MaterialRepeat
{
get => new(ToFloat(12), ToFloat(15));
set
{
_data[12] = FromFloat(value.X);
_data[15] = FromFloat(value.Y);
}
}
public Vector2 MaterialSkew
{
get => new(ToFloat(13), ToFloat(14));
set
{
_data[13] = FromFloat(value.X);
_data[14] = FromFloat(value.Y);
}
}
public float SpecularStrength
{
get => ToFloat(3);
set => _data[3] = FromFloat(value);
}
public float GlossStrength
{
get => ToFloat(7);
set => _data[7] = FromFloat(value);
}
public ushort TileSet
{
get => (ushort)(ToFloat(11) * 64f);
set => _data[11] = FromFloat(value / 64f);
}
private float ToFloat(int idx)
=> (float)BitConverter.UInt16BitsToHalf(_data[idx]);
private static ushort FromFloat(float x)
=> BitConverter.HalfToUInt16Bits((Half)x);
}
public struct RowArray : IEnumerable<Row>
{
public const int NumRows = 16;
private fixed byte _rowData[NumRows * Row.Size];
public ref Row this[int i]
{
get
{
fixed (byte* ptr = _rowData)
{
return ref ((Row*)ptr)[i];
}
}
}
public IEnumerator<Row> GetEnumerator()
{
for (var i = 0; i < NumRows; ++i)
yield return this[i];
}
IEnumerator IEnumerable.GetEnumerator()
=> GetEnumerator();
public ReadOnlySpan<byte> AsBytes()
{
fixed (byte* ptr = _rowData)
{
return new ReadOnlySpan<byte>(ptr, NumRows * Row.Size);
}
}
}
public RowArray Rows;
public string Name;
public ushort Index;
public bool HasRows;
}
}

View file

@ -1,4 +1,3 @@
using System;
using System.IO;
using System.Linq;
using System.Text;

View file

@ -1,9 +1,7 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Text;
using Lumina.Data.Parsing;
using Lumina.Extensions;
@ -13,262 +11,10 @@ namespace Penumbra.GameData.Files;
public partial class MtrlFile : IWritable
{
public struct UvSet
{
public string Name;
public ushort Index;
}
public unsafe struct ColorSet
{
public struct Row
{
public const int Size = 32;
private fixed ushort _data[16];
public Vector3 Diffuse
{
get => new(ToFloat(0), ToFloat(1), ToFloat(2));
set
{
_data[0] = FromFloat(value.X);
_data[1] = FromFloat(value.Y);
_data[2] = FromFloat(value.Z);
}
}
public Vector3 Specular
{
get => new(ToFloat(4), ToFloat(5), ToFloat(6));
set
{
_data[4] = FromFloat(value.X);
_data[5] = FromFloat(value.Y);
_data[6] = FromFloat(value.Z);
}
}
public Vector3 Emissive
{
get => new(ToFloat(8), ToFloat(9), ToFloat(10));
set
{
_data[8] = FromFloat(value.X);
_data[9] = FromFloat(value.Y);
_data[10] = FromFloat(value.Z);
}
}
public Vector2 MaterialRepeat
{
get => new(ToFloat(12), ToFloat(15));
set
{
_data[12] = FromFloat(value.X);
_data[15] = FromFloat(value.Y);
}
}
public Vector2 MaterialSkew
{
get => new(ToFloat(13), ToFloat(14));
set
{
_data[13] = FromFloat(value.X);
_data[14] = FromFloat(value.Y);
}
}
public float SpecularStrength
{
get => ToFloat(3);
set => _data[3] = FromFloat(value);
}
public float GlossStrength
{
get => ToFloat(7);
set => _data[7] = FromFloat(value);
}
public ushort TileSet
{
get => (ushort)(ToFloat(11) * 64f);
set => _data[11] = FromFloat(value / 64f);
}
private float ToFloat(int idx)
=> (float)BitConverter.UInt16BitsToHalf(_data[idx]);
private static ushort FromFloat(float x)
=> BitConverter.HalfToUInt16Bits((Half)x);
}
public struct RowArray : IEnumerable<Row>
{
public const int NumRows = 16;
private fixed byte _rowData[NumRows * Row.Size];
public ref Row this[int i]
{
get
{
fixed (byte* ptr = _rowData)
{
return ref ((Row*)ptr)[i];
}
}
}
public IEnumerator<Row> GetEnumerator()
{
for (var i = 0; i < NumRows; ++i)
yield return this[i];
}
IEnumerator IEnumerable.GetEnumerator()
=> GetEnumerator();
public ReadOnlySpan<byte> AsBytes()
{
fixed (byte* ptr = _rowData)
{
return new ReadOnlySpan<byte>(ptr, NumRows * Row.Size);
}
}
}
public RowArray Rows;
public string Name;
public ushort Index;
public bool HasRows;
}
public unsafe struct ColorDyeSet
{
public struct Row
{
private ushort _data;
public ushort Template
{
get => (ushort)(_data >> 5);
set => _data = (ushort)((_data & 0x1F) | (value << 5));
}
public bool Diffuse
{
get => (_data & 0x01) != 0;
set => _data = (ushort)(value ? _data | 0x01 : _data & 0xFFFE);
}
public bool Specular
{
get => (_data & 0x02) != 0;
set => _data = (ushort)(value ? _data | 0x02 : _data & 0xFFFD);
}
public bool Emissive
{
get => (_data & 0x04) != 0;
set => _data = (ushort)(value ? _data | 0x04 : _data & 0xFFFB);
}
public bool Gloss
{
get => (_data & 0x08) != 0;
set => _data = (ushort)(value ? _data | 0x08 : _data & 0xFFF7);
}
public bool SpecularStrength
{
get => (_data & 0x10) != 0;
set => _data = (ushort)(value ? _data | 0x10 : _data & 0xFFEF);
}
}
public struct RowArray : IEnumerable<Row>
{
public const int NumRows = 16;
private fixed ushort _rowData[NumRows];
public ref Row this[int i]
{
get
{
fixed (ushort* ptr = _rowData)
{
return ref ((Row*)ptr)[i];
}
}
}
public IEnumerator<Row> GetEnumerator()
{
for (var i = 0; i < NumRows; ++i)
yield return this[i];
}
IEnumerator IEnumerable.GetEnumerator()
=> GetEnumerator();
public ReadOnlySpan<byte> AsBytes()
{
fixed (ushort* ptr = _rowData)
{
return new ReadOnlySpan<byte>(ptr, NumRows * sizeof(ushort));
}
}
}
public RowArray Rows;
public string Name;
public ushort Index;
}
public struct Texture
{
public string Path;
public ushort Flags;
public bool DX11
=> (Flags & 0x8000) != 0;
}
public struct Constant
{
public uint Id;
public ushort ByteOffset;
public ushort ByteSize;
}
public struct ShaderPackageData
{
public string Name;
public ShaderKey[] ShaderKeys;
public Constant[] Constants;
public Sampler[] Samplers;
public float[] ShaderValues;
public uint Flags;
}
public readonly uint Version;
public bool Valid
{
get
{
foreach (var texture in Textures)
{
if (!texture.Path.Contains('/'))
{
return false;
}
}
return true;
}
}
public bool Valid
=> CheckTextures();
public Texture[] Textures;
public UvSet[] UvSets;
@ -277,7 +23,7 @@ public partial class MtrlFile : IWritable
public ShaderPackageData ShaderPackage;
public byte[] AdditionalData;
public ShpkFile? AssociatedShpk;
public ShpkFile? AssociatedShpk;
public bool ApplyDyeTemplate(StmFile stm, int colorSetIdx, int rowIdx, StainId stainId)
{
@ -324,15 +70,13 @@ public partial class MtrlFile : IWritable
public Span<float> GetConstantValues(Constant constant)
{
if ((constant.ByteOffset & 0x3) == 0 && (constant.ByteSize & 0x3) == 0
&& ((constant.ByteOffset + constant.ByteSize) >> 2) <= ShaderPackage.ShaderValues.Length)
{
return ShaderPackage.ShaderValues.AsSpan().Slice(constant.ByteOffset >> 2, constant.ByteSize >> 2);
}
else
{
if ((constant.ByteOffset & 0x3) != 0
|| (constant.ByteSize & 0x3) != 0
|| (constant.ByteOffset + constant.ByteSize) >> 2 > ShaderPackage.ShaderValues.Length)
return null;
}
return ShaderPackage.ShaderValues.AsSpan().Slice(constant.ByteOffset >> 2, constant.ByteSize >> 2);
}
public List<(Sampler?, ShpkFile.Resource?)> GetSamplersByTexture()
@ -350,12 +94,6 @@ public partial class MtrlFile : IWritable
return samplers;
}
// Activator.CreateInstance can't use a ctor with a default value so this has to be made explicit
public MtrlFile(byte[] data)
: this(data, null)
{
}
public MtrlFile(byte[] data, Func<string, ShpkFile?>? loadAssociatedShpk = null)
{
using var stream = new MemoryStream(data);
@ -469,6 +207,41 @@ public partial class MtrlFile : IWritable
{
strings = strings[offset..];
var end = strings.IndexOf((byte)'\0');
return Encoding.UTF8.GetString(strings[..end]);
return Encoding.UTF8.GetString(end == -1 ? strings : strings[..end]);
}
private bool CheckTextures()
=> Textures.All(texture => texture.Path.Contains('/'));
public struct UvSet
{
public string Name;
public ushort Index;
}
public struct Texture
{
public string Path;
public ushort Flags;
public bool DX11
=> (Flags & 0x8000) != 0;
}
public struct Constant
{
public uint Id;
public ushort ByteOffset;
public ushort ByteSize;
}
public struct ShaderPackageData
{
public string Name;
public ShaderKey[] ShaderKeys;
public Constant[] Constants;
public Sampler[] Samplers;
public float[] ShaderValues;
public uint Flags;
}
}

View file

@ -0,0 +1,220 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using Lumina.Misc;
using Penumbra.GameData.Data;
namespace Penumbra.GameData.Files;
public partial class ShpkFile
{
public struct Shader
{
public DisassembledShader.ShaderStage Stage;
public DxVersion DirectXVersion;
public Resource[] Constants;
public Resource[] Samplers;
public Resource[] Uavs;
public byte[] AdditionalHeader;
private byte[] _byteData;
private DisassembledShader? _disassembly;
public byte[] Blob
{
get => _byteData;
set
{
if (_byteData == value)
return;
if (Stage != DisassembledShader.ShaderStage.Unspecified)
{
// Reject the blob entirely if we can't disassemble it or if we find inconsistencies.
var disasm = DisassembledShader.Disassemble(value);
if (disasm.Stage != Stage || (disasm.ShaderModel >> 8) + 6 != (uint)DirectXVersion)
throw new ArgumentException(
$"The supplied blob is a DirectX {(disasm.ShaderModel >> 8) + 6} {disasm.Stage} shader ; expected a DirectX {(uint)DirectXVersion} {Stage} shader.",
nameof(value));
if (disasm.ShaderModel >= 0x0500)
{
var samplers = new Dictionary<uint, string>();
var textures = new Dictionary<uint, string>();
foreach (var binding in disasm.ResourceBindings)
{
switch (binding.Type)
{
case DisassembledShader.ResourceType.Texture:
textures[binding.Slot] = NormalizeResourceName(binding.Name);
break;
case DisassembledShader.ResourceType.Sampler:
samplers[binding.Slot] = NormalizeResourceName(binding.Name);
break;
}
}
if (samplers.Count != textures.Count
|| !samplers.All(pair => textures.TryGetValue(pair.Key, out var texName) && pair.Value == texName))
throw new ArgumentException($"The supplied blob has inconsistent sampler and texture allocation.");
}
_byteData = value;
_disassembly = disasm;
}
else
{
_byteData = value;
_disassembly = null;
}
UpdateUsed();
}
}
public DisassembledShader? Disassembly
=> _disassembly;
public Resource? GetConstantById(uint id)
=> Constants.FirstOrNull(res => res.Id == id);
public Resource? GetConstantByName(string name)
=> Constants.FirstOrNull(res => res.Name == name);
public Resource? GetSamplerById(uint id)
=> Samplers.FirstOrNull(s => s.Id == id);
public Resource? GetSamplerByName(string name)
=> Samplers.FirstOrNull(s => s.Name == name);
public Resource? GetUavById(uint id)
=> Uavs.FirstOrNull(u => u.Id == id);
public Resource? GetUavByName(string name)
=> Uavs.FirstOrNull(u => u.Name == name);
public void UpdateResources(ShpkFile file)
{
if (_disassembly == null)
throw new InvalidOperationException();
var constants = new List<Resource>();
var samplers = new List<Resource>();
var uavs = new List<Resource>();
foreach (var binding in _disassembly.ResourceBindings)
{
switch (binding.Type)
{
case DisassembledShader.ResourceType.ConstantBuffer:
var name = NormalizeResourceName(binding.Name);
// We want to preserve IDs as much as possible, and to deterministically generate new ones in a way that's most compliant with the native ones, to maximize compatibility.
var id = GetConstantByName(name)?.Id ?? file.GetConstantByName(name)?.Id ?? Crc32.Get(name, 0xFFFFFFFFu);
constants.Add(new Resource
{
Id = id,
Name = name,
Slot = (ushort)binding.Slot,
Size = (ushort)binding.RegisterCount,
Used = binding.Used,
UsedDynamically = binding.UsedDynamically,
});
break;
case DisassembledShader.ResourceType.Texture:
name = NormalizeResourceName(binding.Name);
id = GetSamplerByName(name)?.Id ?? file.GetSamplerByName(name)?.Id ?? Crc32.Get(name, 0xFFFFFFFFu);
samplers.Add(new Resource
{
Id = id,
Name = name,
Slot = (ushort)binding.Slot,
Size = (ushort)binding.Slot,
Used = binding.Used,
UsedDynamically = binding.UsedDynamically,
});
break;
case DisassembledShader.ResourceType.Uav:
name = NormalizeResourceName(binding.Name);
id = GetUavByName(name)?.Id ?? file.GetUavByName(name)?.Id ?? Crc32.Get(name, 0xFFFFFFFFu);
uavs.Add(new Resource
{
Id = id,
Name = name,
Slot = (ushort)binding.Slot,
Size = (ushort)binding.Slot,
Used = binding.Used,
UsedDynamically = binding.UsedDynamically,
});
break;
}
}
Constants = constants.ToArray();
Samplers = samplers.ToArray();
Uavs = uavs.ToArray();
}
private void UpdateUsed()
{
if (_disassembly != null)
{
var cbUsage = new Dictionary<string, (DisassembledShader.VectorComponents[], DisassembledShader.VectorComponents)>();
var tUsage = new Dictionary<string, (DisassembledShader.VectorComponents[], DisassembledShader.VectorComponents)>();
var uUsage = new Dictionary<string, (DisassembledShader.VectorComponents[], DisassembledShader.VectorComponents)>();
foreach (var binding in _disassembly.ResourceBindings)
{
switch (binding.Type)
{
case DisassembledShader.ResourceType.ConstantBuffer:
cbUsage[NormalizeResourceName(binding.Name)] = (binding.Used, binding.UsedDynamically);
break;
case DisassembledShader.ResourceType.Texture:
tUsage[NormalizeResourceName(binding.Name)] = (binding.Used, binding.UsedDynamically);
break;
case DisassembledShader.ResourceType.Uav:
uUsage[NormalizeResourceName(binding.Name)] = (binding.Used, binding.UsedDynamically);
break;
}
}
static void CopyUsed(Resource[] resources,
Dictionary<string, (DisassembledShader.VectorComponents[], DisassembledShader.VectorComponents)> used)
{
for (var i = 0; i < resources.Length; ++i)
{
if (used.TryGetValue(resources[i].Name, out var usage))
{
resources[i].Used = usage.Item1;
resources[i].UsedDynamically = usage.Item2;
}
else
{
resources[i].Used = null;
resources[i].UsedDynamically = null;
}
}
}
CopyUsed(Constants, cbUsage);
CopyUsed(Samplers, tUsage);
CopyUsed(Uavs, uUsage);
}
else
{
ClearUsed(Constants);
ClearUsed(Samplers);
ClearUsed(Uavs);
}
}
private static string NormalizeResourceName(string resourceName)
{
var dot = resourceName.IndexOf('.');
if (dot >= 0)
return resourceName[..dot];
if (resourceName.Length > 1 && resourceName[^2] is '_' && resourceName[^1] is 'S' or 'T')
return resourceName[..^2];
return resourceName;
}
}
}

View file

@ -0,0 +1,79 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace Penumbra.GameData.Files;
public partial class ShpkFile
{
public class StringPool
{
public MemoryStream Data;
public List<int> StartingOffsets;
public StringPool(ReadOnlySpan<byte> bytes)
{
Data = new MemoryStream();
Data.Write(bytes);
StartingOffsets = new List<int>
{
0,
};
for (var i = 0; i < bytes.Length; ++i)
{
if (bytes[i] == 0)
StartingOffsets.Add(i + 1);
}
if (StartingOffsets[^1] == bytes.Length)
StartingOffsets.RemoveAt(StartingOffsets.Count - 1);
else
Data.WriteByte(0);
}
public string GetString(int offset, int size)
=> Encoding.UTF8.GetString(Data.GetBuffer().AsSpan().Slice(offset, size));
public string GetNullTerminatedString(int offset)
{
var str = Data.GetBuffer().AsSpan()[offset..];
var size = str.IndexOf((byte)0);
if (size >= 0)
str = str[..size];
return Encoding.UTF8.GetString(str);
}
public (int, int) FindOrAddString(string str)
{
var dataSpan = Data.GetBuffer().AsSpan();
var bytes = Encoding.UTF8.GetBytes(str);
foreach (var offset in StartingOffsets)
{
if (offset + bytes.Length > Data.Length)
break;
var strSpan = dataSpan[offset..];
var match = true;
for (var i = 0; i < bytes.Length; ++i)
{
if (strSpan[i] != bytes[i])
{
match = false;
break;
}
}
if (match && strSpan[bytes.Length] == 0)
return (offset, bytes.Length);
}
Data.Seek(0L, SeekOrigin.End);
var newOffset = (int)Data.Position;
StartingOffsets.Add(newOffset);
Data.Write(bytes);
Data.WriteByte(0);
return (newOffset, bytes.Length);
}
}
}

View file

@ -8,21 +8,19 @@ public partial class ShpkFile
public byte[] Write()
{
if (SubViewKeys.Length != 2)
{
throw new InvalidDataException();
}
using var stream = new MemoryStream();
using var blobs = new MemoryStream();
var strings = new StringPool(ReadOnlySpan<byte>.Empty);
using var stream = new MemoryStream();
using var blobs = new MemoryStream();
var strings = new StringPool(ReadOnlySpan<byte>.Empty);
using (var w = new BinaryWriter(stream))
{
w.Write(ShPkMagic);
w.Write(Version);
w.Write(DirectXVersion switch
{
DXVersion.DirectX9 => DX9Magic,
DXVersion.DirectX11 => DX11Magic,
DxVersion.DirectX9 => Dx9Magic,
DxVersion.DirectX11 => Dx11Magic,
_ => throw new NotImplementedException(),
});
var offsetsPosition = stream.Position;
@ -35,7 +33,7 @@ public partial class ShpkFile
w.Write((uint)MaterialParams.Length);
w.Write((uint)Constants.Length);
w.Write((uint)Samplers.Length);
w.Write((uint)UAVs.Length);
w.Write((uint)Uavs.Length);
w.Write((uint)SystemKeys.Length);
w.Write((uint)SceneKeys.Length);
w.Write((uint)MaterialKeys.Length);
@ -43,7 +41,7 @@ public partial class ShpkFile
w.Write((uint)Items.Length);
WriteShaderArray(w, VertexShaders, blobs, strings);
WriteShaderArray(w, PixelShaders, blobs, strings);
WriteShaderArray(w, PixelShaders, blobs, strings);
foreach (var materialParam in MaterialParams)
{
@ -53,54 +51,50 @@ public partial class ShpkFile
}
WriteResourceArray(w, Constants, strings);
WriteResourceArray(w, Samplers, strings);
WriteResourceArray(w, UAVs, strings);
WriteResourceArray(w, Samplers, strings);
WriteResourceArray(w, Uavs, strings);
foreach (var key in SystemKeys)
{
w.Write(key.Id);
w.Write(key.DefaultValue);
}
foreach (var key in SceneKeys)
{
w.Write(key.Id);
w.Write(key.DefaultValue);
}
foreach (var key in MaterialKeys)
{
w.Write(key.Id);
w.Write(key.DefaultValue);
}
foreach (var key in SubViewKeys)
{
w.Write(key.DefaultValue);
}
foreach (var node in Nodes)
{
if (node.PassIndices.Length != 16 || node.SystemKeys.Length != SystemKeys.Length || node.SceneKeys.Length != SceneKeys.Length || node.MaterialKeys.Length != MaterialKeys.Length || node.SubViewKeys.Length != SubViewKeys.Length)
{
if (node.PassIndices.Length != 16
|| node.SystemKeys.Length != SystemKeys.Length
|| node.SceneKeys.Length != SceneKeys.Length
|| node.MaterialKeys.Length != MaterialKeys.Length
|| node.SubViewKeys.Length != SubViewKeys.Length)
throw new InvalidDataException();
}
w.Write(node.Id);
w.Write(node.Passes.Length);
w.Write(node.PassIndices);
foreach (var key in node.SystemKeys)
{
w.Write(key);
}
foreach (var key in node.SceneKeys)
{
w.Write(key);
}
foreach (var key in node.MaterialKeys)
{
w.Write(key);
}
foreach (var key in node.SubViewKeys)
{
w.Write(key);
}
foreach (var pass in node.Passes)
{
w.Write(pass.Id);
@ -160,21 +154,12 @@ public partial class ShpkFile
w.Write(blobSize);
w.Write((ushort)shader.Constants.Length);
w.Write((ushort)shader.Samplers.Length);
w.Write((ushort)shader.UAVs.Length);
w.Write((ushort)shader.Uavs.Length);
w.Write((ushort)0);
WriteResourceArray(w, shader.Constants, strings);
WriteResourceArray(w, shader.Samplers, strings);
WriteResourceArray(w, shader.UAVs, strings);
}
}
private static void WriteUInt32PairArray(BinaryWriter w, (uint, uint)[] array)
{
foreach (var (first, second) in array)
{
w.Write(first);
w.Write(second);
WriteResourceArray(w, shader.Samplers, strings);
WriteResourceArray(w, shader.Uavs, strings);
}
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
namespace Penumbra.GameData;
public static class UtilityFunctions
{
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
public static T? FirstOrNull<T>(this IEnumerable<T> values, Func<T, bool> predicate) where T : struct
=> values.Cast<T?>().FirstOrDefault(v => predicate(v!.Value));
}

View file

@ -1,8 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Reflection;
using Dalamud.Interface;
using Dalamud.Interface.ImGuiFileDialog;
using ImGuiNET;
@ -11,7 +11,6 @@ using OtterGui.Raii;
using Penumbra.GameData.Files;
using Penumbra.Mods;
using Penumbra.String.Classes;
using SixLabors.ImageSharp.PixelFormats;
namespace Penumbra.UI.Classes;
@ -176,9 +175,7 @@ public partial class ModEditWindow
}
private static T? DefaultParseFile( byte[] bytes )
{
return Activator.CreateInstance( typeof( T ), bytes ) as T;
}
=> Activator.CreateInstance( typeof( T ), BindingFlags.CreateInstance | BindingFlags.OptionalParamBinding, bytes ) as T;
private void UpdateCurrentFile( Mod.Editor.FileRegistry path )
{

View file

@ -11,6 +11,7 @@ using Dalamud.Interface.Internal.Notifications;
using ImGuiNET;
using Lumina.Data.Parsing;
using OtterGui;
using OtterGui.Classes;
using OtterGui.Raii;
using Penumbra.GameData.Files;
using Penumbra.String.Classes;

View file

@ -10,6 +10,7 @@ using ImGuiNET;
using Lumina.Misc;
using OtterGui.Raii;
using OtterGui;
using OtterGui.Classes;
using Penumbra.GameData.Data;
using Penumbra.GameData.Files;
using Penumbra.Util;
@ -80,8 +81,8 @@ public partial class ModEditWindow
{
var extension = file.DirectXVersion switch
{
ShpkFile.DXVersion.DirectX9 => ".cso",
ShpkFile.DXVersion.DirectX11 => ".dxbc",
ShpkFile.DxVersion.DirectX9 => ".cso",
ShpkFile.DxVersion.DirectX11 => ".dxbc",
_ => throw new NotImplementedException(),
};
var defaultName = new string( objectName.Where( char.IsUpper ).ToArray() ).ToLower() + idx.ToString();
@ -148,7 +149,7 @@ public partial class ModEditWindow
ret |= DrawShaderPackageResourceArray( "Constant Buffers", "slot", true, shader.Constants, true );
ret |= DrawShaderPackageResourceArray( "Samplers", "slot", false, shader.Samplers, true );
ret |= DrawShaderPackageResourceArray( "Unordered Access Views", "slot", true, shader.UAVs, true );
ret |= DrawShaderPackageResourceArray( "Unordered Access Views", "slot", true, shader.Uavs, true );
if( shader.AdditionalHeader.Length > 0 )
{
@ -506,7 +507,7 @@ public partial class ModEditWindow
ret |= DrawShaderPackageResourceArray( "Constant Buffers", "type", true, file.Constants, disabled );
ret |= DrawShaderPackageResourceArray( "Samplers", "type", false, file.Samplers, disabled );
ret |= DrawShaderPackageResourceArray( "Unordered Access Views", "type", false, file.UAVs, disabled );
ret |= DrawShaderPackageResourceArray( "Unordered Access Views", "type", false, file.Uavs, disabled );
static bool DrawKeyArray( string arrayName, bool withId, ShpkFile.Key[] keys, bool _ )
{