refactor BundleFile read

This commit is contained in:
Perfare 2020-04-07 16:13:04 +08:00
parent 07074b3deb
commit 6678ce082b
7 changed files with 272 additions and 206 deletions

View File

@ -144,6 +144,7 @@
<Compile Include="SerializedFileHeader.cs" /> <Compile Include="SerializedFileHeader.cs" />
<Compile Include="SerializedType.cs" /> <Compile Include="SerializedType.cs" />
<Compile Include="SevenZipHelper.cs" /> <Compile Include="SevenZipHelper.cs" />
<Compile Include="StreamFile.cs" />
<Compile Include="TypeTreeHelper.cs" /> <Compile Include="TypeTreeHelper.cs" />
<Compile Include="TypeTreeNode.cs" /> <Compile Include="TypeTreeNode.cs" />
<Compile Include="UType.cs" /> <Compile Include="UType.cs" />

View File

@ -157,7 +157,7 @@ namespace AssetStudio
if (SerializedFile.IsSerializedFile(subReader)) if (SerializedFile.IsSerializedFile(subReader))
{ {
var dummyPath = Path.GetDirectoryName(fullName) + Path.DirectorySeparatorChar + file.fileName; var dummyPath = Path.GetDirectoryName(fullName) + Path.DirectorySeparatorChar + file.fileName;
LoadAssetsFromMemory(dummyPath, subReader, parentPath ?? fullName, bundleFile.versionEngine); LoadAssetsFromMemory(dummyPath, subReader, parentPath ?? fullName, bundleFile.m_Header.unityRevision);
} }
else else
{ {

View File

@ -1,248 +1,302 @@
using System.Collections.Generic; using System;
using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using Lz4; using Lz4;
namespace AssetStudio namespace AssetStudio
{ {
public class StreamFile public class BundleFile
{ {
public string fileName; public class Header
public Stream stream; {
public string signature;
public uint version;
public string unityVersion;
public string unityRevision;
public long size;
public uint compressedBlocksInfoSize;
public uint uncompressedBlocksInfoSize;
public uint flags;
} }
public class BlockInfo public class StorageBlock
{ {
public uint compressedSize; public uint compressedSize;
public uint uncompressedSize; public uint uncompressedSize;
public short flag; public ushort flags;
} }
public class BundleFile public class Node
{ {
private string path; public long offset;
public string versionPlayer; public long size;
public string versionEngine; public uint flags;
public List<StreamFile> fileList = new List<StreamFile>(); public string path;
}
public BundleFile(EndianBinaryReader bundleReader, string path) public Header m_Header;
private StorageBlock[] m_BlocksInfo;
private Node[] m_DirectoryInfo;
public StreamFile[] fileList;
public BundleFile(EndianBinaryReader reader, string path)
{ {
this.path = path; m_Header = new Header();
var signature = bundleReader.ReadStringToNull(); m_Header.signature = reader.ReadStringToNull();
switch (signature) switch (m_Header.signature)
{ {
case "UnityArchive":
break; //TODO
case "UnityWeb": case "UnityWeb":
case "UnityRaw": case "UnityRaw":
case "\xFA\xFA\xFA\xFA\xFA\xFA\xFA\xFA": ReadHeaderAndBlocksInfo(reader);
using (var blocksStream = CreateBlocksStream(path))
{ {
var format = bundleReader.ReadInt32(); ReadBlocksAndDirectory(reader, blocksStream);
versionPlayer = bundleReader.ReadStringToNull(); ReadFiles(blocksStream, path);
versionEngine = bundleReader.ReadStringToNull();
if (format < 6)
{
int bundleSize = bundleReader.ReadInt32();
}
else if (format == 6)
{
ReadFormat6(bundleReader, true);
return;
}
short dummy2 = bundleReader.ReadInt16();
int offset = bundleReader.ReadInt16();
int dummy3 = bundleReader.ReadInt32();
int lzmaChunks = bundleReader.ReadInt32();
int lzmaSize = 0;
long streamSize = 0;
for (int i = 0; i < lzmaChunks; i++)
{
lzmaSize = bundleReader.ReadInt32();
streamSize = bundleReader.ReadInt32();
}
bundleReader.Position = offset;
switch (signature)
{
case "\xFA\xFA\xFA\xFA\xFA\xFA\xFA\xFA": //.bytes
case "UnityWeb":
{
var lzmaBuffer = bundleReader.ReadBytes(lzmaSize);
using (var lzmaStream = new EndianBinaryReader(SevenZipHelper.StreamDecompress(new MemoryStream(lzmaBuffer))))
{
GetAssetsFiles(lzmaStream, 0);
} }
break; break;
}
case "UnityRaw":
{
GetAssetsFiles(bundleReader, offset);
break;
}
}
break;
}
case "UnityFS": case "UnityFS":
ReadHeader(reader);
ReadBlocksInfoAndDirectory(reader);
using (var blocksStream = CreateBlocksStream(path))
{ {
var format = bundleReader.ReadInt32(); ReadBlocks(reader, blocksStream);
versionPlayer = bundleReader.ReadStringToNull(); ReadFiles(blocksStream, path);
versionEngine = bundleReader.ReadStringToNull();
if (format == 6)
{
ReadFormat6(bundleReader);
} }
break; break;
} }
} }
}
private void GetAssetsFiles(EndianBinaryReader reader, int offset) private void ReadHeaderAndBlocksInfo(EndianBinaryReader reader)
{ {
int fileCount = reader.ReadInt32(); var isCompressed = m_Header.signature == "UnityWeb";
for (int i = 0; i < fileCount; i++) m_Header.version = reader.ReadUInt32();
m_Header.unityVersion = reader.ReadStringToNull();
m_Header.unityRevision = reader.ReadStringToNull();
if (m_Header.version >= 4)
{ {
var file = new StreamFile(); var hash = reader.ReadBytes(16);
file.fileName = Path.GetFileName(reader.ReadStringToNull()); var crc = reader.ReadUInt32();
int fileOffset = reader.ReadInt32();
fileOffset += offset;
int fileSize = reader.ReadInt32();
long nextFile = reader.Position;
reader.Position = fileOffset;
var buffer = reader.ReadBytes(fileSize);
file.stream = new MemoryStream(buffer);
fileList.Add(file);
reader.Position = nextFile;
} }
} var minimumStreamedBytes = reader.ReadUInt32();
var headerSize = reader.ReadUInt32();
private void ReadFormat6(EndianBinaryReader bundleReader, bool padding = false) var numberOfLevelsToDownloadBeforeStreaming = reader.ReadUInt32();
var levelCount = reader.ReadInt32();
m_BlocksInfo = new StorageBlock[levelCount];
for (int i = 0; i < levelCount; i++)
{ {
var bundleSize = bundleReader.ReadInt64(); m_BlocksInfo[i] = new StorageBlock()
int compressedSize = bundleReader.ReadInt32();
int uncompressedSize = bundleReader.ReadInt32();
int flag = bundleReader.ReadInt32();
if (padding)
bundleReader.ReadByte();
byte[] blocksInfoBytes;
if ((flag & 0x80) != 0)//at end of file
{ {
var position = bundleReader.Position; compressedSize = reader.ReadUInt32(),
bundleReader.Position = bundleReader.BaseStream.Length - compressedSize; uncompressedSize = reader.ReadUInt32(),
blocksInfoBytes = bundleReader.ReadBytes(compressedSize); flags = (ushort)(isCompressed ? 1 : 0)
bundleReader.Position = position;
}
else
{
blocksInfoBytes = bundleReader.ReadBytes(compressedSize);
}
var blocksInfoCompressedStream = new MemoryStream(blocksInfoBytes);
MemoryStream blocksInfoDecompressedStream;
switch (flag & 0x3F)
{
default://None
{
blocksInfoDecompressedStream = blocksInfoCompressedStream;
break;
}
case 1://LZMA
{
blocksInfoDecompressedStream = SevenZipHelper.StreamDecompress(blocksInfoCompressedStream);
blocksInfoCompressedStream.Close();
break;
}
case 2://LZ4
case 3://LZ4HC
{
byte[] uncompressedBytes = new byte[uncompressedSize];
using (var decoder = new Lz4DecoderStream(blocksInfoCompressedStream))
{
decoder.Read(uncompressedBytes, 0, uncompressedSize);
}
blocksInfoDecompressedStream = new MemoryStream(uncompressedBytes);
break;
}
//case 4:LZHAM?
}
using (var blocksInfoReader = new EndianBinaryReader(blocksInfoDecompressedStream))
{
blocksInfoReader.Position = 0x10;
int blockcount = blocksInfoReader.ReadInt32();
var blockInfos = new BlockInfo[blockcount];
for (int i = 0; i < blockcount; i++)
{
blockInfos[i] = new BlockInfo
{
uncompressedSize = blocksInfoReader.ReadUInt32(),
compressedSize = blocksInfoReader.ReadUInt32(),
flag = blocksInfoReader.ReadInt16()
}; };
} }
Stream dataStream; if (m_Header.version >= 2)
var uncompressedSizeSum = blockInfos.Sum(x => x.uncompressedSize); {
if (uncompressedSizeSum > int.MaxValue) var completeFileSize = reader.ReadUInt32();
}
if (m_Header.version >= 3)
{
var fileInfoHeaderSize = reader.ReadUInt32();
}
reader.Position = headerSize;
}
private Stream CreateBlocksStream(string path)
{
Stream blocksStream;
var uncompressedSizeSum = m_BlocksInfo.Sum(x => x.uncompressedSize);
if (uncompressedSizeSum >= int.MaxValue)
{ {
/*var memoryMappedFile = MemoryMappedFile.CreateNew(Path.GetFileName(path), uncompressedSizeSum); /*var memoryMappedFile = MemoryMappedFile.CreateNew(Path.GetFileName(path), uncompressedSizeSum);
assetsDataStream = memoryMappedFile.CreateViewStream();*/ assetsDataStream = memoryMappedFile.CreateViewStream();*/
dataStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose); blocksStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose);
} }
else else
{ {
dataStream = new MemoryStream((int)uncompressedSizeSum); blocksStream = new MemoryStream((int)uncompressedSizeSum);
} }
foreach (var blockInfo in blockInfos) return blocksStream;
{
switch (blockInfo.flag & 0x3F)
{
default://None
{
bundleReader.BaseStream.CopyTo(dataStream, blockInfo.compressedSize);
break;
} }
case 1://LZMA
private void ReadBlocksAndDirectory(EndianBinaryReader reader, Stream blocksStream)
{ {
SevenZipHelper.StreamDecompress(bundleReader.BaseStream, dataStream, blockInfo.compressedSize, blockInfo.uncompressedSize); foreach (var blockInfo in m_BlocksInfo)
break;
}
case 2://LZ4
case 3://LZ4HC
{ {
var lz4Stream = new Lz4DecoderStream(bundleReader.BaseStream, blockInfo.compressedSize); var uncompressedBytes = reader.ReadBytes((int)blockInfo.compressedSize);
lz4Stream.CopyTo(dataStream, blockInfo.uncompressedSize); if (blockInfo.flags == 1)
break; {
} using (var memoryStream = new MemoryStream(uncompressedBytes))
//case 4:LZHAM? {
using (var decompressStream = SevenZipHelper.StreamDecompress(memoryStream))
{
uncompressedBytes = decompressStream.ToArray();
} }
} }
dataStream.Position = 0; }
using (dataStream) blocksStream.Write(uncompressedBytes, 0, uncompressedBytes.Length);
}
blocksStream.Position = 0;
var blocksReader = new EndianBinaryReader(blocksStream);
var nodesCount = blocksReader.ReadInt32();
m_DirectoryInfo = new Node[nodesCount];
for (int i = 0; i < nodesCount; i++)
{ {
var entryinfo_count = blocksInfoReader.ReadInt32(); m_DirectoryInfo[i] = new Node
for (int i = 0; i < entryinfo_count; i++)
{ {
path = blocksReader.ReadStringToNull(),
offset = blocksReader.ReadUInt32(),
size = blocksReader.ReadUInt32()
};
}
}
public void ReadFiles(Stream blocksStream, string path)
{
fileList = new StreamFile[m_DirectoryInfo.Length];
for (int i = 0; i < m_DirectoryInfo.Length; i++)
{
var node = m_DirectoryInfo[i];
var file = new StreamFile(); var file = new StreamFile();
var entryinfo_offset = blocksInfoReader.ReadInt64(); fileList[i] = file;
var entryinfo_size = blocksInfoReader.ReadInt64(); file.fileName = Path.GetFileName(node.path);
flag = blocksInfoReader.ReadInt32(); if (node.size >= int.MaxValue)
file.fileName = Path.GetFileName(blocksInfoReader.ReadStringToNull());
if (entryinfo_size > int.MaxValue)
{ {
/*var memoryMappedFile = MemoryMappedFile.CreateNew(file.fileName, entryinfo_size); /*var memoryMappedFile = MemoryMappedFile.CreateNew(file.fileName, entryinfo_size);
file.stream = memoryMappedFile.CreateViewStream();*/ file.stream = memoryMappedFile.CreateViewStream();*/
var extractPath = path + "_unpacked\\"; var extractPath = path + "_unpacked" + Path.DirectorySeparatorChar;
Directory.CreateDirectory(extractPath); Directory.CreateDirectory(extractPath);
file.stream = File.Create(extractPath + file.fileName); file.stream = File.Create(extractPath + file.fileName);
} }
else else
{ {
file.stream = new MemoryStream((int)entryinfo_size); file.stream = new MemoryStream((int)node.size);
} }
dataStream.Position = entryinfo_offset; blocksStream.Position = node.offset;
dataStream.CopyTo(file.stream, entryinfo_size); blocksStream.CopyTo(file.stream, node.size);
file.stream.Position = 0; file.stream.Position = 0;
fileList.Add(file); }
}
private void ReadHeader(EndianBinaryReader reader)
{
m_Header.version = reader.ReadUInt32();
m_Header.unityVersion = reader.ReadStringToNull();
m_Header.unityRevision = reader.ReadStringToNull();
m_Header.size = reader.ReadInt64();
m_Header.compressedBlocksInfoSize = reader.ReadUInt32();
m_Header.uncompressedBlocksInfoSize = reader.ReadUInt32();
m_Header.flags = reader.ReadUInt32();
}
private void ReadBlocksInfoAndDirectory(EndianBinaryReader reader)
{
byte[] blocksInfoBytes;
if ((m_Header.flags & 0x80) != 0) //kArchiveBlocksInfoAtTheEnd
{
var position = reader.Position;
reader.Position = reader.BaseStream.Length - m_Header.compressedBlocksInfoSize;
blocksInfoBytes = reader.ReadBytes((int)m_Header.compressedBlocksInfoSize);
reader.Position = position;
}
else //0x40 kArchiveBlocksAndDirectoryInfoCombined
{
if (m_Header.version >= 7)
{
reader.AlignStream(16);
}
blocksInfoBytes = reader.ReadBytes((int)m_Header.compressedBlocksInfoSize);
}
var blocksInfoCompressedStream = new MemoryStream(blocksInfoBytes);
MemoryStream blocksInfoUncompresseddStream;
switch (m_Header.flags & 0x3F) //kArchiveCompressionTypeMask
{
default: //None
{
blocksInfoUncompresseddStream = blocksInfoCompressedStream;
break;
}
case 1: //LZMA
{
blocksInfoUncompresseddStream = SevenZipHelper.StreamDecompress(blocksInfoCompressedStream);
blocksInfoCompressedStream.Close();
break;
}
case 2: //LZ4
case 3: //LZ4HC
{
var uncompressedBytes = new byte[m_Header.uncompressedBlocksInfoSize];
using (var decoder = new Lz4DecoderStream(blocksInfoCompressedStream))
{
decoder.Read(uncompressedBytes, 0, uncompressedBytes.Length);
}
blocksInfoUncompresseddStream = new MemoryStream(uncompressedBytes);
break;
}
}
using (var blocksInfoReader = new EndianBinaryReader(blocksInfoUncompresseddStream))
{
var uncompressedDataHash = blocksInfoReader.ReadBytes(16);
var blocksInfoCount = blocksInfoReader.ReadInt32();
m_BlocksInfo = new StorageBlock[blocksInfoCount];
for (int i = 0; i < blocksInfoCount; i++)
{
m_BlocksInfo[i] = new StorageBlock
{
uncompressedSize = blocksInfoReader.ReadUInt32(),
compressedSize = blocksInfoReader.ReadUInt32(),
flags = blocksInfoReader.ReadUInt16()
};
}
var nodesCount = blocksInfoReader.ReadInt32();
m_DirectoryInfo = new Node[nodesCount];
for (int i = 0; i < nodesCount; i++)
{
m_DirectoryInfo[i] = new Node
{
offset = blocksInfoReader.ReadInt64(),
size = blocksInfoReader.ReadInt64(),
flags = blocksInfoReader.ReadUInt32(),
path = blocksInfoReader.ReadStringToNull(),
};
} }
} }
} }
private void ReadBlocks(EndianBinaryReader reader, Stream blocksStream)
{
foreach (var blockInfo in m_BlocksInfo)
{
switch (blockInfo.flags & 0x3F) //kStorageBlockCompressionTypeMask
{
default: //None
{
reader.BaseStream.CopyTo(blocksStream, blockInfo.compressedSize);
break;
}
case 1: //LZMA
{
SevenZipHelper.StreamDecompress(reader.BaseStream, blocksStream, blockInfo.compressedSize, blockInfo.uncompressedSize);
break;
}
case 2: //LZ4
case 3: //LZ4HC
{
var compressedStream = new MemoryStream(reader.ReadBytes((int)blockInfo.compressedSize));
using (var lz4Stream = new Lz4DecoderStream(compressedStream))
{
lz4Stream.CopyTo(blocksStream, blockInfo.uncompressedSize);
}
break;
}
}
}
blocksStream.Position = 0;
} }
} }
} }

View File

@ -77,7 +77,7 @@ namespace AssetStudio
{ {
case "UnityWeb": case "UnityWeb":
case "UnityRaw": case "UnityRaw":
case "\xFA\xFA\xFA\xFA\xFA\xFA\xFA\xFA": case "UnityArchive":
case "UnityFS": case "UnityFS":
return FileType.BundleFile; return FileType.BundleFile;
case "UnityWebData1.0": case "UnityWebData1.0":

10
AssetStudio/StreamFile.cs Normal file
View File

@ -0,0 +1,10 @@
using System.IO;
namespace AssetStudio
{
public class StreamFile
{
public string fileName;
public Stream stream;
}
}

View File

@ -12,7 +12,7 @@ namespace AssetStudio
{ {
public static byte[] gzipMagic = { 0x1f, 0x8b }; public static byte[] gzipMagic = { 0x1f, 0x8b };
public static byte[] brotliMagic = { 0x62, 0x72, 0x6F, 0x74, 0x6C, 0x69 }; public static byte[] brotliMagic = { 0x62, 0x72, 0x6F, 0x74, 0x6C, 0x69 };
public List<StreamFile> fileList = new List<StreamFile>(); public StreamFile[] fileList;
private class WebData private class WebData
{ {
@ -78,14 +78,15 @@ namespace AssetStudio
data.path = Encoding.UTF8.GetString(reader.ReadBytes(pathLength)); data.path = Encoding.UTF8.GetString(reader.ReadBytes(pathLength));
dataList.Add(data); dataList.Add(data);
} }
fileList = new StreamFile[dataList.Count];
foreach (var data in dataList) for (int i = 0; i < dataList.Count; i++)
{ {
var data = dataList[i];
var file = new StreamFile(); var file = new StreamFile();
file.fileName = Path.GetFileName(data.path); file.fileName = Path.GetFileName(data.path);
reader.BaseStream.Position = data.dataOffset; reader.BaseStream.Position = data.dataOffset;
file.stream = new MemoryStream(reader.ReadBytes(data.dataLength)); file.stream = new MemoryStream(reader.ReadBytes(data.dataLength));
fileList.Add(file); fileList[i] = file;
} }
} }
} }

View File

@ -55,7 +55,7 @@ namespace AssetStudioGUI
StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFileName)} ..."); StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFileName)} ...");
var bundleFile = new BundleFile(reader, bundleFileName); var bundleFile = new BundleFile(reader, bundleFileName);
reader.Dispose(); reader.Dispose();
if (bundleFile.fileList.Count > 0) if (bundleFile.fileList.Length > 0)
{ {
var extractPath = bundleFileName + "_unpacked\\"; var extractPath = bundleFileName + "_unpacked\\";
return ExtractStreamFile(extractPath, bundleFile.fileList); return ExtractStreamFile(extractPath, bundleFile.fileList);
@ -68,7 +68,7 @@ namespace AssetStudioGUI
StatusStripUpdate($"Decompressing {Path.GetFileName(webFileName)} ..."); StatusStripUpdate($"Decompressing {Path.GetFileName(webFileName)} ...");
var webFile = new WebFile(reader); var webFile = new WebFile(reader);
reader.Dispose(); reader.Dispose();
if (webFile.fileList.Count > 0) if (webFile.fileList.Length > 0)
{ {
var extractPath = webFileName + "_unpacked\\"; var extractPath = webFileName + "_unpacked\\";
return ExtractStreamFile(extractPath, webFile.fileList); return ExtractStreamFile(extractPath, webFile.fileList);
@ -76,7 +76,7 @@ namespace AssetStudioGUI
return 0; return 0;
} }
private static int ExtractStreamFile(string extractPath, List<StreamFile> fileList) private static int ExtractStreamFile(string extractPath,StreamFile[] fileList)
{ {
int extractedCount = 0; int extractedCount = 0;
foreach (var file in fileList) foreach (var file in fileList)