Add oodle compression support (unofficial)

This commit is contained in:
VaDiM
2025-05-09 03:41:53 +03:00
parent 66229e564a
commit 876bafdda1
36 changed files with 430 additions and 125 deletions
+2
View File
@@ -16,6 +16,8 @@
<PackageReference Include="System.IO.Compression" Version="4.3.0" />
<PackageReference Include="System.Text.Json" Version="9.0.0" />
<PackageReference Include="Microsoft.Bcl.Numerics" Version="9.0.1" />
<ProjectReference Include="..\AssetStudio.PInvoke\AssetStudio.PInvoke.csproj" />
</ItemGroup>
</Project>
+8 -4
View File
@@ -13,8 +13,9 @@ namespace AssetStudio
{
public class AssetsManager
{
public bool ZstdEnabled = true;
public bool LoadingViaTypeTreeEnabled = true;
public CompressionType CustomBlockCompression = CompressionType.Auto;
public CompressionType CustomBlockInfoCompression = CompressionType.Auto;
public List<SerializedFile> assetsFileList = new List<SerializedFile>();
internal Dictionary<string, int> assetsFileIndexCache = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
@@ -177,7 +178,10 @@ namespace AssetStudio
private bool LoadFile(FileReader reader)
{
switch (reader?.FileType)
if (reader == null)
return false;
switch (reader.FileType)
{
case FileType.AssetsFile:
return LoadAssetsFile(reader);
@@ -306,7 +310,7 @@ namespace AssetStudio
try
{
var bundleFile = new BundleFile(bundleReader, ZstdEnabled, specifiedUnityVersion);
var bundleFile = new BundleFile(bundleReader, CustomBlockInfoCompression, CustomBlockCompression, specifiedUnityVersion);
var isLoaded = LoadBundleFiles(bundleReader, bundleFile, originalPath);
if (!isLoaded)
return false;
@@ -322,7 +326,7 @@ namespace AssetStudio
bundleReader.FileName = $"{reader.FileName}_0x{bundleStream.Offset:X}";
}
Logger.Info($"[MultiBundle] Loading \"{reader.FileName}\" from offset: 0x{bundleStream.Offset:X}");
bundleFile = new BundleFile(bundleReader, ZstdEnabled, specifiedUnityVersion);
bundleFile = new BundleFile(bundleReader, CustomBlockInfoCompression, CustomBlockCompression, specifiedUnityVersion);
isLoaded = LoadBundleFiles(bundleReader, bundleFile, originalPath ?? reader.FullPath);
}
return isLoaded;
@@ -0,0 +1,43 @@
using System;
using System.Runtime.InteropServices;
#if NETFRAMEWORK
using AssetStudio.PInvoke;
#endif
namespace BundleCompression.Oodle
{
public static class OodleLZ
{
private const string LibName = "ooz";
#if NETFRAMEWORK
static OodleLZ()
{
DllLoader.PreloadDll(LibName);
}
#endif
[DllImport(LibName)]
private static extern int Ooz_Decompress(
in byte srcBuffer,
UIntPtr srcLen,
ref byte dstBuffer,
UIntPtr dstLen,
int fuzzSafetyFlag,
int crcCheckFlag,
int logVerbosityFlag,
UIntPtr rawBuffer,
UIntPtr rawBufferSize,
UIntPtr chunkDecodeCallback,
UIntPtr chunkDecodeContext,
UIntPtr scratchBuf,
UIntPtr scratchBufSize,
int threadPhase);
public static int Decompress(ReadOnlySpan<byte> srcSpanBuffer, Span<byte> dstSpanBuffer)
{
return Ooz_Decompress(in srcSpanBuffer[0], (UIntPtr)srcSpanBuffer.Length, ref dstSpanBuffer[0], (UIntPtr)dstSpanBuffer.Length,
0, 0, 0, UIntPtr.Zero, UIntPtr.Zero, UIntPtr.Zero, UIntPtr.Zero, UIntPtr.Zero, UIntPtr.Zero, 0);
}
}
}
@@ -2,12 +2,11 @@ using System;
using System.IO;
using SevenZip.Compression.LZMA;
namespace AssetStudio
namespace BundleCompression.Lzma
{
public static class SevenZipHelper
public static class SevenZipLzma
{
public static MemoryStream StreamDecompress(MemoryStream inStream)
public static MemoryStream DecompressStream(MemoryStream inStream)
{
var decoder = new Decoder();
@@ -34,7 +33,7 @@ namespace AssetStudio
return newOutStream;
}
public static void StreamDecompress(Stream compressedStream, Stream decompressedStream, long compressedSize, long decompressedSize)
public static long DecompressStream(Stream compressedStream, Stream decompressedStream, long compressedSize, long decompressedSize)
{
var basePosition = compressedStream.Position;
var decoder = new Decoder();
@@ -44,6 +43,7 @@ namespace AssetStudio
decoder.SetDecoderProperties(properties);
decoder.Code(compressedStream, decompressedStream, compressedSize - 5, decompressedSize, null);
compressedStream.Position = basePosition + compressedSize;
return decompressedStream.Position;
}
}
}
+65
View File
@@ -0,0 +1,65 @@
using BundleCompression.Lzma;
using BundleCompression.Oodle;
using System;
using System.IO;
using System.Text.RegularExpressions;
using K4os.Compression.LZ4;
using ZstdSharp;
namespace AssetStudio
{
public static class BundleDecompressionHelper
{
private static readonly Decompressor ZstdDecompressor = new Decompressor();
private static readonly string MsgPattern = @"\. ";
public static MemoryStream DecompressLzmaStream(MemoryStream inStream)
{
return SevenZipLzma.DecompressStream(inStream);
}
public static long DecompressLzmaStream(Stream compressedStream, Stream decompressedStream, long compressedSize, long decompressedSize, ref string errorMsg)
{
var numWrite = -1L;
try
{
numWrite = SevenZipLzma.DecompressStream(compressedStream, decompressedStream, compressedSize, decompressedSize);
}
catch (Exception e)
{
Logger.Debug(e.ToString());
errorMsg = $"({Regex.Split(e.Message, MsgPattern, RegexOptions.CultureInvariant)[0]})";
}
return numWrite;
}
public static int DecompressBlock(CompressionType type, ReadOnlySpan<byte> srcBuffer, Span<byte> dstBuffer, ref string errorMsg)
{
var numWrite = -1;
try
{
switch (type)
{
case CompressionType.Lz4:
case CompressionType.Lz4HC:
numWrite = LZ4Codec.Decode(srcBuffer, dstBuffer);
break;
case CompressionType.Zstd:
numWrite = ZstdDecompressor.Unwrap(srcBuffer, dstBuffer);
break;
case CompressionType.Oodle:
numWrite = OodleLZ.Decompress(srcBuffer, dstBuffer);
break;
default:
throw new NotSupportedException();
}
}
catch (Exception e)
{
Logger.Debug(e.ToString());
errorMsg = $"({Regex.Split(e.Message, MsgPattern, RegexOptions.CultureInvariant)[0]})";
}
return numWrite;
}
}
}
+100 -56
View File
@@ -1,6 +1,4 @@
using K4os.Compression.LZ4;
using ZstdSharp;
using System;
using System;
using System.IO;
using System.Linq;
@@ -32,12 +30,14 @@ namespace AssetStudio
public enum CompressionType
{
Auto = -1,
None,
Lzma,
Lz4,
Lz4HC,
Lzham,
Custom,
Zstd, //custom
Oodle, //custom
}
public class BundleFile
@@ -77,7 +77,7 @@ namespace AssetStudio
public StreamFile[] fileList;
public BundleFile(FileReader reader, bool useZstd, UnityVersion specUnityVer = null)
public BundleFile(FileReader reader, CompressionType customBlockInfoCompression, CompressionType customBlockCompression, UnityVersion specUnityVer = null)
{
m_Header = new Header();
m_Header.signature = reader.ReadStringToNull();
@@ -152,10 +152,10 @@ namespace AssetStudio
throw new NotSupportedException(msg);
}
ReadBlocksInfoAndDirectory(reader, unityVer);
ReadBlocksInfoAndDirectory(reader, customBlockInfoCompression, unityVer);
using (var blocksStream = CreateBlocksStream(reader.FullPath))
{
ReadBlocks(reader, blocksStream, useZstd);
ReadBlocks(reader, customBlockCompression, blocksStream);
ReadFiles(blocksStream, reader.FullPath);
}
break;
@@ -224,7 +224,7 @@ namespace AssetStudio
{
using (var memoryStream = new MemoryStream(uncompressedBytes))
{
using (var decompressStream = SevenZipHelper.StreamDecompress(memoryStream))
using (var decompressStream = BundleDecompressionHelper.DecompressLzmaStream(memoryStream))
{
uncompressedBytes = decompressStream.ToArray();
}
@@ -287,7 +287,7 @@ namespace AssetStudio
}
}
private void ReadBlocksInfoAndDirectory(FileReader reader, UnityVersion unityVer)
private void ReadBlocksInfoAndDirectory(FileReader reader, CompressionType customBlockInfoCompression, UnityVersion unityVer, bool silent = false)
{
byte[] blocksInfoBytes;
@@ -308,52 +308,82 @@ namespace AssetStudio
}
}
var compressedSize = (int)m_Header.compressedBlocksInfoSize;
var uncompressedSize = (int)m_Header.uncompressedBlocksInfoSize;
if ((m_Header.flags & ArchiveFlags.BlocksInfoAtTheEnd) != 0)
{
var position = reader.Position;
reader.Position = m_Header.size - m_Header.compressedBlocksInfoSize;
blocksInfoBytes = reader.ReadBytes((int)m_Header.compressedBlocksInfoSize);
reader.Position = m_Header.size - compressedSize;
blocksInfoBytes = reader.ReadBytes(compressedSize);
reader.Position = position;
}
else //0x40 BlocksAndDirectoryInfoCombined
{
blocksInfoBytes = reader.ReadBytes((int)m_Header.compressedBlocksInfoSize);
blocksInfoBytes = reader.ReadBytes(compressedSize);
}
MemoryStream blocksInfoUncompressedStream;
var uncompressedSize = m_Header.uncompressedBlocksInfoSize;
var compressionType = (CompressionType)(m_Header.flags & ArchiveFlags.CompressionTypeMask);
if (customBlockInfoCompression == CompressionType.Auto)
{
if (!silent && compressionType > CompressionType.Lzham && Enum.IsDefined(typeof(CompressionType), compressionType))
{
Logger.Warning($"Non-standard blockInfo compression type: {(int)compressionType}. Trying to decompress as {compressionType} archive..");
}
}
else if (compressionType != CompressionType.None)
{
compressionType = customBlockInfoCompression;
if (!silent)
{
Logger.Info($"Custom blockInfo compression type: {customBlockInfoCompression}");
}
}
Logger.Debug($"BlockInfo compression: {compressionType}");
int numWrite;
var errorMsg = string.Empty;
MemoryStream blocksInfoUncompressedStream;
switch (compressionType)
{
case CompressionType.None:
{
blocksInfoUncompressedStream = new MemoryStream(blocksInfoBytes);
numWrite = compressedSize;
break;
}
case CompressionType.Lzma:
{
blocksInfoUncompressedStream = new MemoryStream((int) (uncompressedSize));
blocksInfoUncompressedStream = new MemoryStream(uncompressedSize);
using (var blocksInfoCompressedStream = new MemoryStream(blocksInfoBytes))
{
SevenZipHelper.StreamDecompress(blocksInfoCompressedStream, blocksInfoUncompressedStream,
m_Header.compressedBlocksInfoSize, m_Header.uncompressedBlocksInfoSize);
numWrite = (int)BundleDecompressionHelper.DecompressLzmaStream(blocksInfoCompressedStream, blocksInfoUncompressedStream, compressedSize, uncompressedSize, ref errorMsg);
}
blocksInfoUncompressedStream.Position = 0;
break;
}
case CompressionType.Lz4:
case CompressionType.Lz4HC:
case CompressionType.Zstd:
case CompressionType.Oodle:
{
var uncompressedBytes = new byte[uncompressedSize];
var numWrite = LZ4Codec.Decode(blocksInfoBytes, uncompressedBytes);
if (numWrite != uncompressedSize)
{
throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes");
}
numWrite = BundleDecompressionHelper.DecompressBlock(compressionType, blocksInfoBytes, uncompressedBytes, ref errorMsg);
blocksInfoUncompressedStream = new MemoryStream(uncompressedBytes);
break;
}
case CompressionType.Lzham:
throw new IOException($"Unsupported blockInfo compression type: {compressionType}.\n");
default:
throw new IOException($"Unsupported block info compression type {compressionType}");
throw new IOException($"Unknown blockInfo compression type: {compressionType}.\nYou may try to specify the compression type manually.\n");
}
if (numWrite != uncompressedSize)
{
var msg = $"{compressionType} blockInfo decompression error. {errorMsg}\nWrite {numWrite} bytes but expected {uncompressedSize} bytes.";
var exMsg = compressionType > CompressionType.Lz4HC
? "Wrong compression type or blockInfo data might be encrypted."
: "BlockInfo data might be encrypted.";
throw new IOException($"{msg}\n{exMsg}\n");
}
using (var blocksInfoReader = new EndianBinaryReader(blocksInfoUncompressedStream))
@@ -390,62 +420,65 @@ namespace AssetStudio
}
}
private void ReadBlocks(FileReader reader, Stream blocksStream, bool useZstd)
private void ReadBlocks(FileReader reader, CompressionType customBlockCompression, Stream blocksStream)
{
var zstdCodec = new Decompressor();
var i = 0;
var showCustomTypeWarning = true;
foreach (var blockInfo in m_BlocksInfo)
{
var compressionType = (CompressionType)(blockInfo.flags & StorageBlockFlags.CompressionTypeMask);
if (customBlockCompression == CompressionType.Auto)
{
if (showCustomTypeWarning && compressionType > CompressionType.Lzham && Enum.IsDefined(typeof(CompressionType), compressionType))
{
Logger.Warning($"Non-standard block compression type: {(int)compressionType}. Trying to decompress as {compressionType} archive..");
showCustomTypeWarning = false;
}
}
else if (compressionType != CompressionType.None)
{
compressionType = customBlockCompression;
if (showCustomTypeWarning)
{
Logger.Info($"Custom block compression type: {customBlockCompression}");
showCustomTypeWarning = false;
}
}
Logger.Debug($"Block compression: {compressionType}");
long numWrite;
var errorMsg = string.Empty;
switch (compressionType)
{
case CompressionType.None:
{
reader.BaseStream.CopyTo(blocksStream, blockInfo.compressedSize);
numWrite = blockInfo.compressedSize;
break;
}
case CompressionType.Lzma:
{
SevenZipHelper.StreamDecompress(reader.BaseStream, blocksStream, blockInfo.compressedSize, blockInfo.uncompressedSize);
numWrite = BundleDecompressionHelper.DecompressLzmaStream(reader.BaseStream, blocksStream, blockInfo.compressedSize, blockInfo.uncompressedSize, ref errorMsg);
break;
}
case CompressionType.Lz4:
case CompressionType.Lz4HC:
case CompressionType.Custom:
case CompressionType.Zstd:
case CompressionType.Oodle:
{
var compressedSize = (int)blockInfo.compressedSize;
var compressedBytes = BigArrayPool<byte>.Shared.Rent(compressedSize);
_ = reader.Read(compressedBytes, 0, compressedSize);
var uncompressedSize = (int)blockInfo.uncompressedSize;
var compressedBytes = BigArrayPool<byte>.Shared.Rent(compressedSize);
var uncompressedBytes = BigArrayPool<byte>.Shared.Rent(uncompressedSize);
try
{
var compTypeStr = compressionType.ToString();
if (compressionType == CompressionType.Custom)
{
compTypeStr = useZstd ? "Zstd" : "Lz4";
if (i == 0)
{
Logger.Debug($"Custom block compression type was detected. Trying to decompress as {compTypeStr} archive..");
i++;
}
}
_ = reader.Read(compressedBytes, 0, compressedSize);
var compressedSpan = new ReadOnlySpan<byte>(compressedBytes, 0, compressedSize);
var uncompressedSpan = new Span<byte>(uncompressedBytes, 0, uncompressedSize);
int numWrite;
if (compressionType == CompressionType.Custom && useZstd)
{
numWrite = zstdCodec.Unwrap(compressedBytes, 0, compressedSize, uncompressedBytes, 0, uncompressedSize);
}
else
{
numWrite = LZ4Codec.Decode(compressedBytes, 0, compressedSize, uncompressedBytes, 0, uncompressedSize);
}
if (numWrite != uncompressedSize)
{
throw new IOException($"{compTypeStr} block decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes");
}
blocksStream.Write(uncompressedBytes, 0, uncompressedSize);
numWrite = BundleDecompressionHelper.DecompressBlock(compressionType, compressedSpan, uncompressedSpan, ref errorMsg);
if (numWrite == uncompressedSize)
blocksStream.Write(uncompressedBytes, 0, uncompressedSize);
}
finally
{
@@ -454,8 +487,19 @@ namespace AssetStudio
}
break;
}
case CompressionType.Lzham:
throw new IOException($"Unsupported block compression type: {compressionType}.\n");
default:
throw new IOException($"Unsupported block compression type {compressionType}");
throw new IOException($"Unknown block compression type: {compressionType}.\nYou may try to specify the compression type manually.\n");
}
if (numWrite != blockInfo.uncompressedSize)
{
var msg = $"{compressionType} block decompression error. {errorMsg}\nWrite {numWrite} bytes but expected {blockInfo.uncompressedSize} bytes.";
var exMsg = compressionType > CompressionType.Lz4HC
? "Wrong compression type or block data might be encrypted."
: "Block data might be encrypted.";
throw new IOException($"{msg}\n{exMsg}\n");
}
}
blocksStream.Position = 0;