ToxicMushroom/Penumbra
1
0
Fork 0
mirror of https://github.com/xivdev/Penumbra.git synced 2025-12-12 18:27:24 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixed formatting in SqPackStream, applied null-checking, replaced user-facing asserts with throws.

Browse source
This commit is contained in:
Ottermandias 2021-04-18 23:14:48 +02:00
parent 63407f775f
commit 7c56de8c81
2 changed files with 151 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Penumbra/Importer/TexToolsImport.cs
View file

@ -65,7 +65,7 @@ namespace Penumbra.Importer
} }
// You can in no way rely on any file paths in TTMPs so we need to just do this, sorry // You can in no way rely on any file paths in TTMPs so we need to just do this, sorry
private ZipEntry FindZipEntry( ZipFile file, string fileName ) private static ZipEntry? FindZipEntry( ZipFile file, string fileName )
{ {
for( var i = 0; i < file.Count; i++ ) for( var i = 0; i < file.Count; i++ )
{ {
@ -84,7 +84,10 @@ namespace Penumbra.Importer
// write shitty zip garbage to disk // write shitty zip garbage to disk
var entry = FindZipEntry( file, entryName ); var entry = FindZipEntry( file, entryName );
Debug.Assert( entry != null, $"Could not find in mod zip: {entryName}" ); if( entry == null )
{
throw new FileNotFoundException( $"ZIP does not contain a file named {entryName}." );
}
using var s = file.GetInputStream( entry ); using var s = file.GetInputStream( entry );
@ -100,7 +103,10 @@ namespace Penumbra.Importer
using var extractedModPack = new ZipFile( zfs ); using var extractedModPack = new ZipFile( zfs );
var mpl = FindZipEntry( extractedModPack, "TTMPL.mpl" ); var mpl = FindZipEntry( extractedModPack, "TTMPL.mpl" );
Debug.Assert( mpl != null, "Could not find mod meta in ZIP." ); if( mpl == null )
{
throw new FileNotFoundException( "ZIP does not contain a TTMPL.mpl file." );
}
var modRaw = GetStringFromZipEntry( extractedModPack, mpl, Encoding.UTF8 ); var modRaw = GetStringFromZipEntry( extractedModPack, mpl, Encoding.UTF8 );

250
Penumbra/Util/PenumbraSqPackStream.cs
View file

@ -1,14 +1,10 @@
using System; using System;
using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.IO; using System.IO;
using System.IO.Compression; using System.IO.Compression;
using System.Linq;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using System.Text; using System.Text;
using System.Threading.Tasks;
using Lumina; using Lumina;
using Lumina.Data;
using Lumina.Data.Structs; using Lumina.Data.Structs;
namespace Penumbra.Util namespace Penumbra.Util
@ -19,12 +15,14 @@ namespace Penumbra.Util
protected BinaryReader Reader { get; set; } protected BinaryReader Reader { get; set; }
public PenumbraSqPackStream( FileInfo file ) : this( file.OpenRead() ) {} public PenumbraSqPackStream( FileInfo file )
: this( file.OpenRead() )
{ }
public PenumbraSqPackStream( Stream stream ) public PenumbraSqPackStream( Stream stream )
{ {
BaseStream = stream; BaseStream = stream;
Reader = new BinaryReader( BaseStream ); Reader = new BinaryReader( BaseStream );
} }
public SqPackHeader GetSqPackHeader() public SqPackHeader GetSqPackHeader()
@ -48,7 +46,7 @@ namespace Penumbra.Util
BaseStream.Position = offset; BaseStream.Position = offset;
var fileInfo = Reader.ReadStructure< SqPackFileInfo >(); var fileInfo = Reader.ReadStructure< SqPackFileInfo >();
var file = Activator.CreateInstance< T >(); var file = Activator.CreateInstance< T >();
// check if we need to read the extended model header or just default to the standard file header // check if we need to read the extended model header or just default to the standard file header
if( fileInfo.Type == FileType.Model ) if( fileInfo.Type == FileType.Model )
@ -59,32 +57,31 @@ namespace Penumbra.Util
file.FileInfo = new PenumbraFileInfo file.FileInfo = new PenumbraFileInfo
{ {
HeaderSize = modelFileInfo.Size, HeaderSize = modelFileInfo.Size,
Type = modelFileInfo.Type, Type = modelFileInfo.Type,
BlockCount = modelFileInfo.UsedNumberOfBlocks, BlockCount = modelFileInfo.UsedNumberOfBlocks,
RawFileSize = modelFileInfo.RawFileSize, RawFileSize = modelFileInfo.RawFileSize,
Offset = offset, Offset = offset,
// todo: is this useful? // todo: is this useful?
ModelBlock = modelFileInfo ModelBlock = modelFileInfo,
}; };
} }
else else
{ {
file.FileInfo = new PenumbraFileInfo file.FileInfo = new PenumbraFileInfo
{ {
HeaderSize = fileInfo.Size, HeaderSize = fileInfo.Size,
Type = fileInfo.Type, Type = fileInfo.Type,
BlockCount = fileInfo.NumberOfBlocks, BlockCount = fileInfo.NumberOfBlocks,
RawFileSize = fileInfo.RawFileSize, RawFileSize = fileInfo.RawFileSize,
Offset = offset Offset = offset,
}; };
} }
switch( fileInfo.Type ) switch( fileInfo.Type )
{ {
case FileType.Empty: case FileType.Empty: throw new FileNotFoundException( $"The file located at 0x{offset:x} is empty." );
throw new FileNotFoundException( $"The file located at 0x{offset:x} is empty." );
case FileType.Standard: case FileType.Standard:
ReadStandardFile( file, ms ); ReadStandardFile( file, ms );
@ -98,16 +95,17 @@ namespace Penumbra.Util
ReadTextureFile( file, ms ); ReadTextureFile( file, ms );
break; break;
default: default: throw new NotImplementedException( $"File Type {( uint )fileInfo.Type} is not implemented." );
throw new NotImplementedException( $"File Type {(UInt32)fileInfo.Type} is not implemented." );
} }
file.Data = ms.ToArray(); file.Data = ms.ToArray();
if( file.Data.Length != file.FileInfo.RawFileSize ) if( file.Data.Length != file.FileInfo.RawFileSize )
{
Debug.WriteLine( "Read data size does not match file size." ); Debug.WriteLine( "Read data size does not match file size." );
}
file.FileStream = new MemoryStream( file.Data, false ); file.FileStream = new MemoryStream( file.Data, false );
file.Reader = new BinaryReader( file.FileStream ); file.Reader = new BinaryReader( file.FileStream );
file.FileStream.Position = 0; file.FileStream.Position = 0;
file.LoadFile(); file.LoadFile();
@ -117,7 +115,7 @@ namespace Penumbra.Util
private void ReadStandardFile( PenumbraFileResource resource, MemoryStream ms ) private void ReadStandardFile( PenumbraFileResource resource, MemoryStream ms )
{ {
var blocks = Reader.ReadStructures< DatStdFileBlockInfos >( (int)resource.FileInfo.BlockCount ); var blocks = Reader.ReadStructures< DatStdFileBlockInfos >( ( int )resource.FileInfo!.BlockCount );
foreach( var block in blocks ) foreach( var block in blocks )
{ {
@ -128,9 +126,10 @@ namespace Penumbra.Util
ms.Position = 0; ms.Position = 0;
} }
private unsafe void ReadModelFile( PenumbraFileResource resource, MemoryStream ms ) { private unsafe void ReadModelFile( PenumbraFileResource resource, MemoryStream ms )
var mdlBlock = resource.FileInfo.ModelBlock; {
long baseOffset = resource.FileInfo.Offset + resource.FileInfo.HeaderSize; var mdlBlock = resource.FileInfo!.ModelBlock;
var baseOffset = resource.FileInfo.Offset + resource.FileInfo.HeaderSize;
// 1/1/3/3/3 stack/runtime/vertex/egeo/index // 1/1/3/3/3 stack/runtime/vertex/egeo/index
// TODO: consider testing if this is more reliable than the Explorer method // TODO: consider testing if this is more reliable than the Explorer method
@ -139,89 +138,111 @@ namespace Penumbra.Util
// but it seems to work fine in explorer... // but it seems to work fine in explorer...
int totalBlocks = mdlBlock.StackBlockNum; int totalBlocks = mdlBlock.StackBlockNum;
totalBlocks += mdlBlock.RuntimeBlockNum; totalBlocks += mdlBlock.RuntimeBlockNum;
for( int i = 0; i < 3; i++ ) for( var i = 0; i < 3; i++ )
{
totalBlocks += mdlBlock.VertexBufferBlockNum[ i ]; totalBlocks += mdlBlock.VertexBufferBlockNum[ i ];
for( int i = 0; i < 3; i++ ) }
totalBlocks += mdlBlock.EdgeGeometryVertexBufferBlockNum[ i ];
for( int i = 0; i < 3; i++ )
totalBlocks += mdlBlock.IndexBufferBlockNum[ i ];
var compressedBlockSizes = Reader.ReadStructures< UInt16 >( totalBlocks ); for( var i = 0; i < 3; i++ )
int currentBlock = 0; {
int stackSize; totalBlocks += mdlBlock.EdgeGeometryVertexBufferBlockNum[ i ];
int runtimeSize; }
int[] vertexDataOffsets = new int[3];
int[] indexDataOffsets = new int[3]; for( var i = 0; i < 3; i++ )
int[] vertexBufferSizes = new int[3]; {
int[] indexBufferSizes = new int[3]; totalBlocks += mdlBlock.IndexBufferBlockNum[ i ];
}
var compressedBlockSizes = Reader.ReadStructures< ushort >( totalBlocks );
var currentBlock = 0;
var vertexDataOffsets = new int[3];
var indexDataOffsets = new int[3];
var vertexBufferSizes = new int[3];
var indexBufferSizes = new int[3];
ms.Seek( 0x44, SeekOrigin.Begin ); ms.Seek( 0x44, SeekOrigin.Begin );
Reader.Seek( baseOffset + mdlBlock.StackOffset ); Reader.Seek( baseOffset + mdlBlock.StackOffset );
long stackStart = ms.Position; var stackStart = ms.Position;
for( int i = 0; i < mdlBlock.StackBlockNum; i++ ) { for( var i = 0; i < mdlBlock.StackBlockNum; i++ )
long lastPos = Reader.BaseStream.Position; {
var lastPos = Reader.BaseStream.Position;
ReadFileBlock( ms ); ReadFileBlock( ms );
Reader.Seek( lastPos + compressedBlockSizes[ currentBlock ] ); Reader.Seek( lastPos + compressedBlockSizes[ currentBlock ] );
currentBlock++; currentBlock++;
} }
long stackEnd = ms.Position; var stackEnd = ms.Position;
stackSize = (int) ( stackEnd - stackStart ); var stackSize = ( int )( stackEnd - stackStart );
Reader.Seek( baseOffset + mdlBlock.RuntimeOffset ); Reader.Seek( baseOffset + mdlBlock.RuntimeOffset );
long runtimeStart = ms.Position; var runtimeStart = ms.Position;
for( int i = 0; i < mdlBlock.RuntimeBlockNum; i++ ) { for( var i = 0; i < mdlBlock.RuntimeBlockNum; i++ )
long lastPos = Reader.BaseStream.Position; {
var lastPos = Reader.BaseStream.Position;
ReadFileBlock( ms ); ReadFileBlock( ms );
Reader.Seek( lastPos + compressedBlockSizes[ currentBlock ] ); Reader.Seek( lastPos + compressedBlockSizes[ currentBlock ] );
currentBlock++; currentBlock++;
} }
long runtimeEnd = ms.Position; var runtimeEnd = ms.Position;
runtimeSize = (int) ( runtimeEnd - runtimeStart ); var runtimeSize = ( int )( runtimeEnd - runtimeStart );
for( int i = 0; i < 3; i++ ) { for( var i = 0; i < 3; i++ )
{
if( mdlBlock.VertexBufferBlockNum[ i ] != 0 ) { if( mdlBlock.VertexBufferBlockNum[ i ] != 0 )
int currentVertexOffset = (int) ms.Position; {
var currentVertexOffset = ( int )ms.Position;
if( i == 0 || currentVertexOffset != vertexDataOffsets[ i - 1 ] ) if( i == 0 || currentVertexOffset != vertexDataOffsets[ i - 1 ] )
{
vertexDataOffsets[ i ] = currentVertexOffset; vertexDataOffsets[ i ] = currentVertexOffset;
}
else else
{
vertexDataOffsets[ i ] = 0; vertexDataOffsets[ i ] = 0;
}
Reader.Seek( baseOffset + mdlBlock.VertexBufferOffset[ i ] ); Reader.Seek( baseOffset + mdlBlock.VertexBufferOffset[ i ] );
for( int j = 0; j < mdlBlock.VertexBufferBlockNum[ i ]; j++ ) { for( var j = 0; j < mdlBlock.VertexBufferBlockNum[ i ]; j++ )
long lastPos = Reader.BaseStream.Position; {
vertexBufferSizes[ i ] += (int) ReadFileBlock( ms ); var lastPos = Reader.BaseStream.Position;
vertexBufferSizes[ i ] += ( int )ReadFileBlock( ms );
Reader.Seek( lastPos + compressedBlockSizes[ currentBlock ] ); Reader.Seek( lastPos + compressedBlockSizes[ currentBlock ] );
currentBlock++; currentBlock++;
} }
} }
if( mdlBlock.EdgeGeometryVertexBufferBlockNum[ i ] != 0 ) { if( mdlBlock.EdgeGeometryVertexBufferBlockNum[ i ] != 0 )
for( int j = 0; j < mdlBlock.EdgeGeometryVertexBufferBlockNum[ i ]; j++ ) { {
long lastPos = Reader.BaseStream.Position; for( var j = 0; j < mdlBlock.EdgeGeometryVertexBufferBlockNum[ i ]; j++ )
{
var lastPos = Reader.BaseStream.Position;
ReadFileBlock( ms ); ReadFileBlock( ms );
Reader.Seek( lastPos + compressedBlockSizes[ currentBlock ] ); Reader.Seek( lastPos + compressedBlockSizes[ currentBlock ] );
currentBlock++; currentBlock++;
} }
} }
if( mdlBlock.IndexBufferBlockNum[ i ] != 0 ) { if( mdlBlock.IndexBufferBlockNum[ i ] != 0 )
int currentIndexOffset = (int) ms.Position; {
var currentIndexOffset = ( int )ms.Position;
if( i == 0 || currentIndexOffset != indexDataOffsets[ i - 1 ] ) if( i == 0 || currentIndexOffset != indexDataOffsets[ i - 1 ] )
{
indexDataOffsets[ i ] = currentIndexOffset; indexDataOffsets[ i ] = currentIndexOffset;
}
else else
{
indexDataOffsets[ i ] = 0; indexDataOffsets[ i ] = 0;
}
// i guess this is only needed in the vertex area, for i = 0 // i guess this is only needed in the vertex area, for i = 0
// Reader.Seek( baseOffset + mdlBlock.IndexBufferOffset[ i ] ); // Reader.Seek( baseOffset + mdlBlock.IndexBufferOffset[ i ] );
for( int j = 0; j < mdlBlock.IndexBufferBlockNum[ i ]; j++ ) { for( var j = 0; j < mdlBlock.IndexBufferBlockNum[ i ]; j++ )
long lastPos = Reader.BaseStream.Position; {
indexBufferSizes[ i ] += (int) ReadFileBlock( ms ); var lastPos = Reader.BaseStream.Position;
indexBufferSizes[ i ] += ( int )ReadFileBlock( ms );
Reader.Seek( lastPos + compressedBlockSizes[ currentBlock ] ); Reader.Seek( lastPos + compressedBlockSizes[ currentBlock ] );
currentBlock++; currentBlock++;
} }
@ -234,33 +255,45 @@ namespace Penumbra.Util
ms.Write( BitConverter.GetBytes( runtimeSize ) ); ms.Write( BitConverter.GetBytes( runtimeSize ) );
ms.Write( BitConverter.GetBytes( mdlBlock.VertexDeclarationNum ) ); ms.Write( BitConverter.GetBytes( mdlBlock.VertexDeclarationNum ) );
ms.Write( BitConverter.GetBytes( mdlBlock.MaterialNum ) ); ms.Write( BitConverter.GetBytes( mdlBlock.MaterialNum ) );
for( int i = 0; i < 3; i++ ) for( var i = 0; i < 3; i++ )
{
ms.Write( BitConverter.GetBytes( vertexDataOffsets[ i ] ) ); ms.Write( BitConverter.GetBytes( vertexDataOffsets[ i ] ) );
for( int i = 0; i < 3; i++ ) }
for( var i = 0; i < 3; i++ )
{
ms.Write( BitConverter.GetBytes( indexDataOffsets[ i ] ) ); ms.Write( BitConverter.GetBytes( indexDataOffsets[ i ] ) );
for( int i = 0; i < 3; i++ ) }
for( var i = 0; i < 3; i++ )
{
ms.Write( BitConverter.GetBytes( vertexBufferSizes[ i ] ) ); ms.Write( BitConverter.GetBytes( vertexBufferSizes[ i ] ) );
for( int i = 0; i < 3; i++ ) }
for( var i = 0; i < 3; i++ )
{
ms.Write( BitConverter.GetBytes( indexBufferSizes[ i ] ) ); ms.Write( BitConverter.GetBytes( indexBufferSizes[ i ] ) );
ms.Write( new [] {mdlBlock.NumLods} ); }
ms.Write( new[] { mdlBlock.NumLods } );
ms.Write( BitConverter.GetBytes( mdlBlock.IndexBufferStreamingEnabled ) ); ms.Write( BitConverter.GetBytes( mdlBlock.IndexBufferStreamingEnabled ) );
ms.Write( BitConverter.GetBytes( mdlBlock.EdgeGeometryEnabled ) ); ms.Write( BitConverter.GetBytes( mdlBlock.EdgeGeometryEnabled ) );
ms.Write( new byte[] {0} ); ms.Write( new byte[] { 0 } );
} }
private void ReadTextureFile( PenumbraFileResource resource, MemoryStream ms ) private void ReadTextureFile( PenumbraFileResource resource, MemoryStream ms )
{ {
var blocks = Reader.ReadStructures< LodBlock >( (int)resource.FileInfo.BlockCount ); var blocks = Reader.ReadStructures< LodBlock >( ( int )resource.FileInfo!.BlockCount );
// if there is a mipmap header, the comp_offset // if there is a mipmap header, the comp_offset
// will not be 0 // will not be 0
uint mipMapSize = blocks[ 0 ].CompressedOffset; var mipMapSize = blocks[ 0 ].CompressedOffset;
if( mipMapSize != 0 ) if( mipMapSize != 0 )
{ {
long originalPos = BaseStream.Position; var originalPos = BaseStream.Position;
BaseStream.Position = resource.FileInfo.Offset + resource.FileInfo.HeaderSize; BaseStream.Position = resource.FileInfo.Offset + resource.FileInfo.HeaderSize;
ms.Write( Reader.ReadBytes( (int)mipMapSize ) ); ms.Write( Reader.ReadBytes( ( int )mipMapSize ) );
BaseStream.Position = originalPos; BaseStream.Position = originalPos;
} }
@ -269,12 +302,12 @@ namespace Penumbra.Util
for( byte i = 0; i < blocks.Count; i++ ) for( byte i = 0; i < blocks.Count; i++ )
{ {
// start from comp_offset // start from comp_offset
long runningBlockTotal = blocks[ i ].CompressedOffset + resource.FileInfo.Offset + resource.FileInfo.HeaderSize; var runningBlockTotal = blocks[ i ].CompressedOffset + resource.FileInfo.Offset + resource.FileInfo.HeaderSize;
ReadFileBlock( runningBlockTotal, ms, true ); ReadFileBlock( runningBlockTotal, ms, true );
for( int j = 1; j < blocks[ i ].BlockCount; j++ ) for( var j = 1; j < blocks[ i ].BlockCount; j++ )
{ {
runningBlockTotal += (UInt32)Reader.ReadInt16(); runningBlockTotal += ( uint )Reader.ReadInt16();
ReadFileBlock( runningBlockTotal, ms, true ); ReadFileBlock( runningBlockTotal, ms, true );
} }
@ -283,25 +316,24 @@ namespace Penumbra.Util
} }
} }
protected uint ReadFileBlock( MemoryStream dest, bool resetPosition = false ) { protected uint ReadFileBlock( MemoryStream dest, bool resetPosition = false )
return ReadFileBlock( Reader.BaseStream.Position, dest, resetPosition ); => ReadFileBlock( Reader.BaseStream.Position, dest, resetPosition );
}
protected uint ReadFileBlock( long offset, MemoryStream dest, bool resetPosition = false ) protected uint ReadFileBlock( long offset, MemoryStream dest, bool resetPosition = false )
{ {
long originalPosition = BaseStream.Position; var originalPosition = BaseStream.Position;
BaseStream.Position = offset; BaseStream.Position = offset;
var blockHeader = Reader.ReadStructure< DatBlockHeader >(); var blockHeader = Reader.ReadStructure< DatBlockHeader >();
// uncompressed block // uncompressed block
if( blockHeader.CompressedSize == 32000 ) if( blockHeader.CompressedSize == 32000 )
{ {
dest.Write( Reader.ReadBytes( (int)blockHeader.UncompressedSize ) ); dest.Write( Reader.ReadBytes( ( int )blockHeader.UncompressedSize ) );
return blockHeader.UncompressedSize; return blockHeader.UncompressedSize;
} }
var data = Reader.ReadBytes( (int)blockHeader.UncompressedSize ); var data = Reader.ReadBytes( ( int )blockHeader.UncompressedSize );
using( var compressedStream = new MemoryStream( data ) ) using( var compressedStream = new MemoryStream( data ) )
{ {
@ -311,7 +343,9 @@ namespace Penumbra.Util
} }
if( resetPosition ) if( resetPosition )
{
BaseStream.Position = originalPosition; BaseStream.Position = originalPosition;
}
return blockHeader.UncompressedSize; return blockHeader.UncompressedSize;
} }
@ -323,10 +357,10 @@ namespace Penumbra.Util
public class PenumbraFileInfo public class PenumbraFileInfo
{ {
public UInt32 HeaderSize; public uint HeaderSize;
public FileType Type; public FileType Type;
public UInt32 RawFileSize; public uint RawFileSize;
public UInt32 BlockCount; public uint BlockCount;
public long Offset { get; internal set; } public long Offset { get; internal set; }
@ -336,20 +370,20 @@ namespace Penumbra.Util
public class PenumbraFileResource public class PenumbraFileResource
{ {
public PenumbraFileResource() public PenumbraFileResource()
{ { }
}
public PenumbraFileInfo FileInfo { get; internal set; } public PenumbraFileInfo? FileInfo { get; internal set; }
public byte[] Data { get; internal set; } public byte[] Data { get; internal set; } = new byte[0];
public Span< byte > DataSpan => Data.AsSpan(); public Span< byte > DataSpan
=> Data.AsSpan();
public MemoryStream FileStream { get; internal set; } public MemoryStream? FileStream { get; internal set; }
public BinaryReader Reader { get; internal set; } public BinaryReader? Reader { get; internal set; }
public ParsedFilePath FilePath { get; internal set; } public ParsedFilePath? FilePath { get; internal set; }
/// <summary> /// <summary>
/// Called once the files are read out from the dats. Used to further parse the file into usable data structures. /// Called once the files are read out from the dats. Used to further parse the file into usable data structures.
@ -380,22 +414,22 @@ namespace Penumbra.Util
} }
[StructLayout( LayoutKind.Sequential )] [StructLayout( LayoutKind.Sequential )]
struct DatBlockHeader private struct DatBlockHeader
{ {
public UInt32 Size; public uint Size;
public UInt32 unknown1; public uint unknown1;
public UInt32 CompressedSize; public uint CompressedSize;
public UInt32 UncompressedSize; public uint UncompressedSize;
}; };
[StructLayout( LayoutKind.Sequential )] [StructLayout( LayoutKind.Sequential )]
struct LodBlock private struct LodBlock
{ {
public UInt32 CompressedOffset; public uint CompressedOffset;
public UInt32 CompressedSize; public uint CompressedSize;
public UInt32 DecompressedSize; public uint DecompressedSize;
public UInt32 BlockOffset; public uint BlockOffset;
public UInt32 BlockCount; public uint BlockCount;
} }
} }
} }