refactor BundleFile read

AssetStudio/AssetStudio.csproj

@@ -144,6 +144,7 @@
     <Compile Include="SerializedFileHeader.cs" />
     <Compile Include="SerializedType.cs" />
     <Compile Include="SevenZipHelper.cs" />
+    <Compile Include="StreamFile.cs" />
     <Compile Include="TypeTreeHelper.cs" />
     <Compile Include="TypeTreeNode.cs" />
     <Compile Include="UType.cs" />

AssetStudio/AssetsManager.cs

@@ -157,7 +157,7 @@ namespace AssetStudio
                     if (SerializedFile.IsSerializedFile(subReader))
                     {
                         var dummyPath = Path.GetDirectoryName(fullName) + Path.DirectorySeparatorChar + file.fileName;
-                        LoadAssetsFromMemory(dummyPath, subReader, parentPath ?? fullName, bundleFile.versionEngine);
+                        LoadAssetsFromMemory(dummyPath, subReader, parentPath ?? fullName, bundleFile.m_Header.unityRevision);
                     }
                     else
                     {

AssetStudio/BundleFile.cs

@@ -1,248 +1,302 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Lz4;
 
 namespace AssetStudio
 {
-    public class StreamFile
+    public class BundleFile
     {
-        public string fileName;
+        public class Header
-        public Stream stream;
+        {
+            public string signature;
+            public uint version;
+            public string unityVersion;
+            public string unityRevision;
+            public long size;
+            public uint compressedBlocksInfoSize;
+            public uint uncompressedBlocksInfoSize;
+            public uint flags;
         }
 
-    public class BlockInfo
+        public class StorageBlock
         {
             public uint compressedSize;
             public uint uncompressedSize;
-        public short flag;
+            public ushort flags;
         }
 
-    public class BundleFile
+        public class Node
         {
-        private string path;
+            public long offset;
-        public string versionPlayer;
+            public long size;
-        public string versionEngine;
+            public uint flags;
-        public List<StreamFile> fileList = new List<StreamFile>();
+            public string path;
+        }
 
-        public BundleFile(EndianBinaryReader bundleReader, string path)
+        public Header m_Header;
+        private StorageBlock[] m_BlocksInfo;
+        private Node[] m_DirectoryInfo;
+
+        public StreamFile[] fileList;
+
+        public BundleFile(EndianBinaryReader reader, string path)
         {
-            this.path = path;
+            m_Header = new Header();
-            var signature = bundleReader.ReadStringToNull();
+            m_Header.signature = reader.ReadStringToNull();
-            switch (signature)
+            switch (m_Header.signature)
             {
+                case "UnityArchive":
+                    break; //TODO
                 case "UnityWeb":
                 case "UnityRaw":
-                case "\xFA\xFA\xFA\xFA\xFA\xFA\xFA\xFA":
+                    ReadHeaderAndBlocksInfo(reader);
+                    using (var blocksStream = CreateBlocksStream(path))
                     {
-                        var format = bundleReader.ReadInt32();
+                        ReadBlocksAndDirectory(reader, blocksStream);
-                        versionPlayer = bundleReader.ReadStringToNull();
+                        ReadFiles(blocksStream, path);
-                        versionEngine = bundleReader.ReadStringToNull();
-                        if (format < 6)
-                        {
-                            int bundleSize = bundleReader.ReadInt32();
-                        }
-                        else if (format == 6)
-                        {
-                            ReadFormat6(bundleReader, true);
-                            return;
-                        }
-                        short dummy2 = bundleReader.ReadInt16();
-                        int offset = bundleReader.ReadInt16();
-                        int dummy3 = bundleReader.ReadInt32();
-                        int lzmaChunks = bundleReader.ReadInt32();
-
-                        int lzmaSize = 0;
-                        long streamSize = 0;
-
-                        for (int i = 0; i < lzmaChunks; i++)
-                        {
-                            lzmaSize = bundleReader.ReadInt32();
-                            streamSize = bundleReader.ReadInt32();
-                        }
-
-                        bundleReader.Position = offset;
-                        switch (signature)
-                        {
-                            case "\xFA\xFA\xFA\xFA\xFA\xFA\xFA\xFA": //.bytes
-                            case "UnityWeb":
-                                {
-                                    var lzmaBuffer = bundleReader.ReadBytes(lzmaSize);
-                                    using (var lzmaStream = new EndianBinaryReader(SevenZipHelper.StreamDecompress(new MemoryStream(lzmaBuffer))))
-                                    {
-                                        GetAssetsFiles(lzmaStream, 0);
                     }
                     break;
-                                }
-                            case "UnityRaw":
-                                {
-                                    GetAssetsFiles(bundleReader, offset);
-                                    break;
-                                }
-                        }
-                        break;
-                    }
                 case "UnityFS":
+                    ReadHeader(reader);
+                    ReadBlocksInfoAndDirectory(reader);
+                    using (var blocksStream = CreateBlocksStream(path))
                     {
-                        var format = bundleReader.ReadInt32();
+                        ReadBlocks(reader, blocksStream);
-                        versionPlayer = bundleReader.ReadStringToNull();
+                        ReadFiles(blocksStream, path);
-                        versionEngine = bundleReader.ReadStringToNull();
-                        if (format == 6)
-                        {
-                            ReadFormat6(bundleReader);
                     }
                     break;
             }
         }
-        }
 
-        private void GetAssetsFiles(EndianBinaryReader reader, int offset)
+        private void ReadHeaderAndBlocksInfo(EndianBinaryReader reader)
         {
-            int fileCount = reader.ReadInt32();
+            var isCompressed = m_Header.signature == "UnityWeb";
-            for (int i = 0; i < fileCount; i++)
+            m_Header.version = reader.ReadUInt32();
+            m_Header.unityVersion = reader.ReadStringToNull();
+            m_Header.unityRevision = reader.ReadStringToNull();
+            if (m_Header.version >= 4)
             {
-                var file = new StreamFile();
+                var hash = reader.ReadBytes(16);
-                file.fileName = Path.GetFileName(reader.ReadStringToNull());
+                var crc = reader.ReadUInt32();
-                int fileOffset = reader.ReadInt32();
-                fileOffset += offset;
-                int fileSize = reader.ReadInt32();
-                long nextFile = reader.Position;
-                reader.Position = fileOffset;
-                var buffer = reader.ReadBytes(fileSize);
-                file.stream = new MemoryStream(buffer);
-                fileList.Add(file);
-                reader.Position = nextFile;
             }
-        }
+            var minimumStreamedBytes = reader.ReadUInt32();
-
+            var headerSize = reader.ReadUInt32();
-        private void ReadFormat6(EndianBinaryReader bundleReader, bool padding = false)
+            var numberOfLevelsToDownloadBeforeStreaming = reader.ReadUInt32();
+            var levelCount = reader.ReadInt32();
+            m_BlocksInfo = new StorageBlock[levelCount];
+            for (int i = 0; i < levelCount; i++)
             {
-            var bundleSize = bundleReader.ReadInt64();
+                m_BlocksInfo[i] = new StorageBlock()
-            int compressedSize = bundleReader.ReadInt32();
-            int uncompressedSize = bundleReader.ReadInt32();
-            int flag = bundleReader.ReadInt32();
-            if (padding)
-                bundleReader.ReadByte();
-            byte[] blocksInfoBytes;
-            if ((flag & 0x80) != 0)//at end of file
                 {
-                var position = bundleReader.Position;
+                    compressedSize = reader.ReadUInt32(),
-                bundleReader.Position = bundleReader.BaseStream.Length - compressedSize;
+                    uncompressedSize = reader.ReadUInt32(),
-                blocksInfoBytes = bundleReader.ReadBytes(compressedSize);
+                    flags = (ushort)(isCompressed ? 1 : 0)
-                bundleReader.Position = position;
-            }
-            else
-            {
-                blocksInfoBytes = bundleReader.ReadBytes(compressedSize);
-            }
-            var blocksInfoCompressedStream = new MemoryStream(blocksInfoBytes);
-            MemoryStream blocksInfoDecompressedStream;
-            switch (flag & 0x3F)
-            {
-                default://None
-                    {
-                        blocksInfoDecompressedStream = blocksInfoCompressedStream;
-                        break;
-                    }
-                case 1://LZMA
-                    {
-                        blocksInfoDecompressedStream = SevenZipHelper.StreamDecompress(blocksInfoCompressedStream);
-                        blocksInfoCompressedStream.Close();
-                        break;
-                    }
-                case 2://LZ4
-                case 3://LZ4HC
-                    {
-                        byte[] uncompressedBytes = new byte[uncompressedSize];
-                        using (var decoder = new Lz4DecoderStream(blocksInfoCompressedStream))
-                        {
-                            decoder.Read(uncompressedBytes, 0, uncompressedSize);
-                        }
-                        blocksInfoDecompressedStream = new MemoryStream(uncompressedBytes);
-                        break;
-                    }
-                    //case 4:LZHAM?
-            }
-            using (var blocksInfoReader = new EndianBinaryReader(blocksInfoDecompressedStream))
-            {
-                blocksInfoReader.Position = 0x10;
-                int blockcount = blocksInfoReader.ReadInt32();
-                var blockInfos = new BlockInfo[blockcount];
-                for (int i = 0; i < blockcount; i++)
-                {
-                    blockInfos[i] = new BlockInfo
-                    {
-                        uncompressedSize = blocksInfoReader.ReadUInt32(),
-                        compressedSize = blocksInfoReader.ReadUInt32(),
-                        flag = blocksInfoReader.ReadInt16()
                 };
             }
-                Stream dataStream;
+            if (m_Header.version >= 2)
-                var uncompressedSizeSum = blockInfos.Sum(x => x.uncompressedSize);
+            {
-                if (uncompressedSizeSum > int.MaxValue)
+                var completeFileSize = reader.ReadUInt32();
+            }
+            if (m_Header.version >= 3)
+            {
+                var fileInfoHeaderSize = reader.ReadUInt32();
+            }
+            reader.Position = headerSize;
+        }
+
+        private Stream CreateBlocksStream(string path)
+        {
+            Stream blocksStream;
+            var uncompressedSizeSum = m_BlocksInfo.Sum(x => x.uncompressedSize);
+            if (uncompressedSizeSum >= int.MaxValue)
             {
                 /*var memoryMappedFile = MemoryMappedFile.CreateNew(Path.GetFileName(path), uncompressedSizeSum);
                 assetsDataStream = memoryMappedFile.CreateViewStream();*/
-                    dataStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose);
+                blocksStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose);
             }
             else
             {
-                    dataStream = new MemoryStream((int)uncompressedSizeSum);
+                blocksStream = new MemoryStream((int)uncompressedSizeSum);
             }
-                foreach (var blockInfo in blockInfos)
+            return blocksStream;
-                {
-                    switch (blockInfo.flag & 0x3F)
-                    {
-                        default://None
-                            {
-                                bundleReader.BaseStream.CopyTo(dataStream, blockInfo.compressedSize);
-                                break;
         }
-                        case 1://LZMA
+
+        private void ReadBlocksAndDirectory(EndianBinaryReader reader, Stream blocksStream)
         {
-                                SevenZipHelper.StreamDecompress(bundleReader.BaseStream, dataStream, blockInfo.compressedSize, blockInfo.uncompressedSize);
+            foreach (var blockInfo in m_BlocksInfo)
-                                break;
-                            }
-                        case 2://LZ4
-                        case 3://LZ4HC
             {
-                                var lz4Stream = new Lz4DecoderStream(bundleReader.BaseStream, blockInfo.compressedSize);
+                var uncompressedBytes = reader.ReadBytes((int)blockInfo.compressedSize);
-                                lz4Stream.CopyTo(dataStream, blockInfo.uncompressedSize);
+                if (blockInfo.flags == 1)
-                                break;
+                {
-                            }
+                    using (var memoryStream = new MemoryStream(uncompressedBytes))
-                            //case 4:LZHAM?
+                    {
+                        using (var decompressStream = SevenZipHelper.StreamDecompress(memoryStream))
+                        {
+                            uncompressedBytes = decompressStream.ToArray();
                         }
                     }
-                dataStream.Position = 0;
+                }
-                using (dataStream)
+                blocksStream.Write(uncompressedBytes, 0, uncompressedBytes.Length);
+            }
+            blocksStream.Position = 0;
+            var blocksReader = new EndianBinaryReader(blocksStream);
+            var nodesCount = blocksReader.ReadInt32();
+            m_DirectoryInfo = new Node[nodesCount];
+            for (int i = 0; i < nodesCount; i++)
             {
-                    var entryinfo_count = blocksInfoReader.ReadInt32();
+                m_DirectoryInfo[i] = new Node
-                    for (int i = 0; i < entryinfo_count; i++)
                 {
+                    path = blocksReader.ReadStringToNull(),
+                    offset = blocksReader.ReadUInt32(),
+                    size = blocksReader.ReadUInt32()
+                };
+            }
+        }
+
+        public void ReadFiles(Stream blocksStream, string path)
+        {
+            fileList = new StreamFile[m_DirectoryInfo.Length];
+            for (int i = 0; i < m_DirectoryInfo.Length; i++)
+            {
+                var node = m_DirectoryInfo[i];
                 var file = new StreamFile();
-                        var entryinfo_offset = blocksInfoReader.ReadInt64();
+                fileList[i] = file;
-                        var entryinfo_size = blocksInfoReader.ReadInt64();
+                file.fileName = Path.GetFileName(node.path);
-                        flag = blocksInfoReader.ReadInt32();
+                if (node.size >= int.MaxValue)
-                        file.fileName = Path.GetFileName(blocksInfoReader.ReadStringToNull());
-                        if (entryinfo_size > int.MaxValue)
                 {
                     /*var memoryMappedFile = MemoryMappedFile.CreateNew(file.fileName, entryinfo_size);
                     file.stream = memoryMappedFile.CreateViewStream();*/
-                            var extractPath = path + "_unpacked\\";
+                    var extractPath = path + "_unpacked" + Path.DirectorySeparatorChar;
                     Directory.CreateDirectory(extractPath);
                     file.stream = File.Create(extractPath + file.fileName);
                 }
                 else
                 {
-                            file.stream = new MemoryStream((int)entryinfo_size);
+                    file.stream = new MemoryStream((int)node.size);
                 }
-                        dataStream.Position = entryinfo_offset;
+                blocksStream.Position = node.offset;
-                        dataStream.CopyTo(file.stream, entryinfo_size);
+                blocksStream.CopyTo(file.stream, node.size);
                 file.stream.Position = 0;
-                        fileList.Add(file);
+            }
+        }
+
+        private void ReadHeader(EndianBinaryReader reader)
+        {
+            m_Header.version = reader.ReadUInt32();
+            m_Header.unityVersion = reader.ReadStringToNull();
+            m_Header.unityRevision = reader.ReadStringToNull();
+            m_Header.size = reader.ReadInt64();
+            m_Header.compressedBlocksInfoSize = reader.ReadUInt32();
+            m_Header.uncompressedBlocksInfoSize = reader.ReadUInt32();
+            m_Header.flags = reader.ReadUInt32();
+        }
+
+        private void ReadBlocksInfoAndDirectory(EndianBinaryReader reader)
+        {
+            byte[] blocksInfoBytes;
+            if ((m_Header.flags & 0x80) != 0) //kArchiveBlocksInfoAtTheEnd
+            {
+                var position = reader.Position;
+                reader.Position = reader.BaseStream.Length - m_Header.compressedBlocksInfoSize;
+                blocksInfoBytes = reader.ReadBytes((int)m_Header.compressedBlocksInfoSize);
+                reader.Position = position;
+            }
+            else //0x40 kArchiveBlocksAndDirectoryInfoCombined
+            {
+                if (m_Header.version >= 7)
+                {
+                    reader.AlignStream(16);
+                }
+                blocksInfoBytes = reader.ReadBytes((int)m_Header.compressedBlocksInfoSize);
+            }
+            var blocksInfoCompressedStream = new MemoryStream(blocksInfoBytes);
+            MemoryStream blocksInfoUncompresseddStream;
+            switch (m_Header.flags & 0x3F) //kArchiveCompressionTypeMask
+            {
+                default: //None
+                    {
+                        blocksInfoUncompresseddStream = blocksInfoCompressedStream;
+                        break;
+                    }
+                case 1: //LZMA
+                    {
+                        blocksInfoUncompresseddStream = SevenZipHelper.StreamDecompress(blocksInfoCompressedStream);
+                        blocksInfoCompressedStream.Close();
+                        break;
+                    }
+                case 2: //LZ4
+                case 3: //LZ4HC
+                    {
+                        var uncompressedBytes = new byte[m_Header.uncompressedBlocksInfoSize];
+                        using (var decoder = new Lz4DecoderStream(blocksInfoCompressedStream))
+                        {
+                            decoder.Read(uncompressedBytes, 0, uncompressedBytes.Length);
+                        }
+                        blocksInfoUncompresseddStream = new MemoryStream(uncompressedBytes);
+                        break;
+                    }
+            }
+            using (var blocksInfoReader = new EndianBinaryReader(blocksInfoUncompresseddStream))
+            {
+                var uncompressedDataHash = blocksInfoReader.ReadBytes(16);
+                var blocksInfoCount = blocksInfoReader.ReadInt32();
+                m_BlocksInfo = new StorageBlock[blocksInfoCount];
+                for (int i = 0; i < blocksInfoCount; i++)
+                {
+                    m_BlocksInfo[i] = new StorageBlock
+                    {
+                        uncompressedSize = blocksInfoReader.ReadUInt32(),
+                        compressedSize = blocksInfoReader.ReadUInt32(),
+                        flags = blocksInfoReader.ReadUInt16()
+                    };
+                }
+
+                var nodesCount = blocksInfoReader.ReadInt32();
+                m_DirectoryInfo = new Node[nodesCount];
+                for (int i = 0; i < nodesCount; i++)
+                {
+                    m_DirectoryInfo[i] = new Node
+                    {
+                        offset = blocksInfoReader.ReadInt64(),
+                        size = blocksInfoReader.ReadInt64(),
+                        flags = blocksInfoReader.ReadUInt32(),
+                        path = blocksInfoReader.ReadStringToNull(),
+                    };
                 }
             }
         }
+
+        private void ReadBlocks(EndianBinaryReader reader, Stream blocksStream)
+        {
+            foreach (var blockInfo in m_BlocksInfo)
+            {
+                switch (blockInfo.flags & 0x3F) //kStorageBlockCompressionTypeMask
+                {
+                    default: //None
+                        {
+                            reader.BaseStream.CopyTo(blocksStream, blockInfo.compressedSize);
+                            break;
+                        }
+                    case 1: //LZMA
+                        {
+                            SevenZipHelper.StreamDecompress(reader.BaseStream, blocksStream, blockInfo.compressedSize, blockInfo.uncompressedSize);
+                            break;
+                        }
+                    case 2: //LZ4
+                    case 3: //LZ4HC
+                        {
+                            var compressedStream = new MemoryStream(reader.ReadBytes((int)blockInfo.compressedSize));
+                            using (var lz4Stream = new Lz4DecoderStream(compressedStream))
+                            {
+                                lz4Stream.CopyTo(blocksStream, blockInfo.uncompressedSize);
+                            }
+                            break;
+                        }
+                }
+            }
+            blocksStream.Position = 0;
         }
     }
 }

AssetStudio/ImportHelper.cs

@@ -77,7 +77,7 @@ namespace AssetStudio
             {
                 case "UnityWeb":
                 case "UnityRaw":
-                case "\xFA\xFA\xFA\xFA\xFA\xFA\xFA\xFA":
+                case "UnityArchive":
                 case "UnityFS":
                     return FileType.BundleFile;
                 case "UnityWebData1.0":

AssetStudio/StreamFile.cs

@@ -0,0 +1,10 @@
+using System.IO;
+
+namespace AssetStudio
+{
+    public class StreamFile
+    {
+        public string fileName;
+        public Stream stream;
+    }
+}

AssetStudio/WebFile.cs

@@ -12,7 +12,7 @@ namespace AssetStudio
     {
         public static byte[] gzipMagic = { 0x1f, 0x8b };
         public static byte[] brotliMagic = { 0x62, 0x72, 0x6F, 0x74, 0x6C, 0x69 };
-        public List<StreamFile> fileList = new List<StreamFile>();
+        public StreamFile[] fileList;
 
         private class WebData
         {
@@ -78,14 +78,15 @@ namespace AssetStudio
                 data.path = Encoding.UTF8.GetString(reader.ReadBytes(pathLength));
                 dataList.Add(data);
             }
-
+            fileList = new StreamFile[dataList.Count];
-            foreach (var data in dataList)
+            for (int i = 0; i < dataList.Count; i++)
             {
+                var data = dataList[i];
                 var file = new StreamFile();
                 file.fileName = Path.GetFileName(data.path);
                 reader.BaseStream.Position = data.dataOffset;
                 file.stream = new MemoryStream(reader.ReadBytes(data.dataLength));
-                fileList.Add(file);
+                fileList[i] = file;
             }
         }
     }

AssetStudioGUI/Studio.cs

@@ -55,7 +55,7 @@ namespace AssetStudioGUI
             StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFileName)} ...");
             var bundleFile = new BundleFile(reader, bundleFileName);
             reader.Dispose();
-            if (bundleFile.fileList.Count > 0)
+            if (bundleFile.fileList.Length > 0)
             {
                 var extractPath = bundleFileName + "_unpacked\\";
                 return ExtractStreamFile(extractPath, bundleFile.fileList);
@@ -68,7 +68,7 @@ namespace AssetStudioGUI
             StatusStripUpdate($"Decompressing {Path.GetFileName(webFileName)} ...");
             var webFile = new WebFile(reader);
             reader.Dispose();
-            if (webFile.fileList.Count > 0)
+            if (webFile.fileList.Length > 0)
             {
                 var extractPath = webFileName + "_unpacked\\";
                 return ExtractStreamFile(extractPath, webFile.fileList);
@@ -76,7 +76,7 @@ namespace AssetStudioGUI
             return 0;
         }
 
-        private static int ExtractStreamFile(string extractPath, List<StreamFile> fileList)
+        private static int ExtractStreamFile(string extractPath,StreamFile[] fileList)
         {
             int extractedCount = 0;
             foreach (var file in fileList)