support load over 2GB file, Fixed #149

AssetStudio/7zip/SevenZipHelper.cs

@@ -1,144 +0,0 @@
-using System;
-using System.IO;
-
-
-namespace SevenZip.Compression.LZMA
-{
-    public static class SevenZipHelper
-    {
-
-        static int dictionary = 1 << 23;
-
-        // static Int32 posStateBits = 2;
-        // static  Int32 litContextBits = 3; // for normal files
-        // UInt32 litContextBits = 0; // for 32-bit data
-        // static  Int32 litPosBits = 0;
-        // UInt32 litPosBits = 2; // for 32-bit data
-        // static   Int32 algorithm = 2;
-        // static    Int32 numFastBytes = 128;
-
-        static bool eos = false;
-
-
-
-
-
-        static CoderPropID[] propIDs =
-                   {
-                    CoderPropID.DictionarySize,
-                    CoderPropID.PosStateBits,
-                    CoderPropID.LitContextBits,
-                    CoderPropID.LitPosBits,
-                    CoderPropID.Algorithm,
-                    CoderPropID.NumFastBytes,
-                    CoderPropID.MatchFinder,
-                    CoderPropID.EndMarker
-                };
-
-        // these are the default properties, keeping it simple for now:
-        static object[] properties =
-                   {
-                    (Int32)(dictionary),
-                    (Int32)(2),
-                    (Int32)(3),
-                    (Int32)(0),
-                    (Int32)(2),
-                    (Int32)(128),
-                    "bt4",
-                    eos
-                };
-
-
-        public static byte[] Compress(byte[] inputBytes)
-        {
-
-            MemoryStream inStream = new MemoryStream(inputBytes);
-            MemoryStream outStream = new MemoryStream();
-            Encoder encoder = new Encoder();
-            encoder.SetCoderProperties(propIDs, properties);
-            encoder.WriteCoderProperties(outStream);
-            long fileSize = inStream.Length;
-            for (int i = 0; i < 8; i++)
-                outStream.WriteByte((Byte)(fileSize >> (8 * i)));
-            encoder.Code(inStream, outStream, -1, -1, null);
-            return outStream.ToArray();
-        }
-
-        public static byte[] Decompress(byte[] inputBytes)
-        {
-            MemoryStream newInStream = new MemoryStream(inputBytes);
-
-            Decoder decoder = new Decoder();
-
-            newInStream.Seek(0, 0);
-            MemoryStream newOutStream = new MemoryStream();
-
-            byte[] properties2 = new byte[5];
-            if (newInStream.Read(properties2, 0, 5) != 5)
-                throw (new Exception("input .lzma is too short"));
-            long outSize = 0;
-            for (int i = 0; i < 8; i++)
-            {
-                int v = newInStream.ReadByte();
-                if (v < 0)
-                    throw (new Exception("Can't Read 1"));
-                outSize |= ((long)(byte)v) << (8 * i);
-            }
-            decoder.SetDecoderProperties(properties2);
-
-            long compressedSize = newInStream.Length - newInStream.Position;
-            decoder.Code(newInStream, newOutStream, compressedSize, outSize, null);
-
-            byte[] b = newOutStream.ToArray();
-
-            return b;
-        }
-
-
-        public static MemoryStream StreamDecompress(MemoryStream newInStream)
-        {
-            Decoder decoder = new Decoder();
-
-            newInStream.Seek(0, 0);
-            MemoryStream newOutStream = new MemoryStream();
-
-            byte[] properties2 = new byte[5];
-            if (newInStream.Read(properties2, 0, 5) != 5)
-                throw (new Exception("input .lzma is too short"));
-            long outSize = 0;
-            for (int i = 0; i < 8; i++)
-            {
-                int v = newInStream.ReadByte();
-                if (v < 0)
-                    throw (new Exception("Can't Read 1"));
-                outSize |= ((long)(byte)v) << (8 * i);
-            }
-            decoder.SetDecoderProperties(properties2);
-
-            long compressedSize = newInStream.Length - newInStream.Position;
-            decoder.Code(newInStream, newOutStream, compressedSize, outSize, null);
-
-            newOutStream.Position = 0;
-            return newOutStream;
-        }
-
-        public static MemoryStream StreamDecompress(MemoryStream newInStream, long outSize)
-        {
-            Decoder decoder = new Decoder();
-
-            newInStream.Seek(0, 0);
-            MemoryStream newOutStream = new MemoryStream();
-
-            byte[] properties2 = new byte[5];
-            if (newInStream.Read(properties2, 0, 5) != 5)
-                throw (new Exception("input .lzma is too short"));
-            decoder.SetDecoderProperties(properties2);
-
-            long compressedSize = newInStream.Length - newInStream.Position;
-            decoder.Code(newInStream, newOutStream, compressedSize, outSize, null);
-
-            newOutStream.Position = 0;
-            return newOutStream;
-        }
-    }
-}

AssetStudio/AssetStudio-x86.csproj

@@ -136,9 +136,6 @@
     <Compile Include="7zip\ICoder.cs">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="7zip\SevenZipHelper.cs">
-      <SubType>Code</SubType>
-    </Compile>
     <Compile Include="AssetStudioForm.cs">
       <SubType>Form</SubType>
     </Compile>
@@ -170,6 +167,7 @@
     <Compile Include="StudioClasses\BuildTarget.cs" />
     <Compile Include="StudioClasses\ModelConverter.cs" />
     <Compile Include="StudioClasses\ResourcesHelper.cs" />
+    <Compile Include="StudioClasses\SevenZipHelper.cs" />
     <Compile Include="StudioClasses\ShaderResource.Designer.cs">
       <DependentUpon>ShaderResource.resx</DependentUpon>
       <AutoGen>True</AutoGen>
@@ -179,6 +177,7 @@
     <Compile Include="StudioClasses\EndianBinaryReader.cs" />
     <Compile Include="StudioClasses\Exporter.cs" />
     <Compile Include="StudioClasses\Importer.cs" />
+    <Compile Include="StudioClasses\StreamExtensions.cs" />
     <Compile Include="StudioClasses\Studio.cs" />
     <Compile Include="StudioClasses\ClassIDReference.cs" />
     <Compile Include="ExportOptions.cs">

AssetStudio/AssetStudio.csproj

@@ -136,7 +136,7 @@
     <Compile Include="7zip\ICoder.cs">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="7zip\SevenZipHelper.cs">
+    <Compile Include="StudioClasses\SevenZipHelper.cs">
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="Classes\Animation.cs" />
@@ -172,6 +172,7 @@
     <Compile Include="StudioClasses\SpriteHelper.cs" />
     <Compile Include="StudioClasses\Exporter.cs" />
     <Compile Include="StudioClasses\Importer.cs" />
+    <Compile Include="StudioClasses\StreamExtensions.cs" />
     <Compile Include="StudioClasses\Studio.cs" />
     <Compile Include="StudioClasses\EndianBinaryReader.cs" />
     <Compile Include="ExportOptions.cs">

AssetStudio/StudioClasses/BundleFile.cs

@@ -1,25 +1,34 @@
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using Lz4;
 using SevenZip.Compression.LZMA;
 
 namespace AssetStudio
 {
-    public class MemoryFile
+    public class StreamFile
     {
         public string fileName;
-        public MemoryStream stream;
+        public Stream stream;
+    }
+
+    public class BlockInfo
+    {
+        public uint compressedSize;
+        public uint uncompressedSize;
+        public short flag;
     }
 
     public class BundleFile
     {
-        public int format;
+        private string path;
         public string versionPlayer;
         public string versionEngine;
-        public List<MemoryFile> fileList = new List<MemoryFile>();
+        public List<StreamFile> fileList = new List<StreamFile>();
 
-        public BundleFile(EndianBinaryReader bundleReader)
+        public BundleFile(EndianBinaryReader bundleReader, string path)
         {
+            this.path = path;
             var signature = bundleReader.ReadStringToNull();
             switch (signature)
             {
@@ -27,7 +36,7 @@ namespace AssetStudio
                 case "UnityRaw":
                 case "\xFA\xFA\xFA\xFA\xFA\xFA\xFA\xFA":
                     {
-                        format = bundleReader.ReadInt32();
+                        var format = bundleReader.ReadInt32();
                         versionPlayer = bundleReader.ReadStringToNull();
                         versionEngine = bundleReader.ReadStringToNull();
                         if (format < 6)
@@ -75,7 +84,8 @@ namespace AssetStudio
                         break;
                     }
                 case "UnityFS":
-                    format = bundleReader.ReadInt32();
+                    {
+                        var format = bundleReader.ReadInt32();
                         versionPlayer = bundleReader.ReadStringToNull();
                         versionEngine = bundleReader.ReadStringToNull();
                         if (format == 6)
@@ -85,13 +95,14 @@ namespace AssetStudio
                         break;
                     }
             }
+        }
 
         private void GetAssetsFiles(EndianBinaryReader reader, int offset)
         {
             int fileCount = reader.ReadInt32();
             for (int i = 0; i < fileCount; i++)
             {
-                var file = new MemoryFile();
+                var file = new StreamFile();
                 file.fileName = reader.ReadStringToNull();
                 int fileOffset = reader.ReadInt32();
                 fileOffset += offset;
@@ -151,63 +162,82 @@ namespace AssetStudio
                     }
                     //case 4:LZHAM?
             }
-            using (var blocksInfo = new EndianBinaryReader(blocksInfoStream))
+            using (var blocksInfoReader = new EndianBinaryReader(blocksInfoStream))
             {
-                blocksInfo.Position = 0x10;
+                blocksInfoReader.Position = 0x10;
-                int blockcount = blocksInfo.ReadInt32();
+                int blockcount = blocksInfoReader.ReadInt32();
-                var assetsDataStream = new MemoryStream();
+                var blockInfos = new BlockInfo[blockcount];
                 for (int i = 0; i < blockcount; i++)
                 {
-                    uncompressedSize = blocksInfo.ReadInt32();
+                    blockInfos[i] = new BlockInfo
-                    compressedSize = blocksInfo.ReadInt32();
+                    {
-                    flag = blocksInfo.ReadInt16();
+                        uncompressedSize = blocksInfoReader.ReadUInt32(),
-                    var compressedBytes = bundleReader.ReadBytes(compressedSize);
+                        compressedSize = blocksInfoReader.ReadUInt32(),
-                    switch (flag & 0x3F)
+                        flag = blocksInfoReader.ReadInt16()
+                    };
+                }
+                Stream dataStream;
+                var uncompressedSizeSum = blockInfos.Sum(x => x.uncompressedSize);
+                if (uncompressedSizeSum > int.MaxValue)
+                {
+                    /*var memoryMappedFile = MemoryMappedFile.CreateNew(Path.GetFileName(path), uncompressedSizeSum);
+                    assetsDataStream = memoryMappedFile.CreateViewStream();*/
+                    dataStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose);
+                }
+                else
+                {
+                    dataStream = new MemoryStream();
+                }
+                foreach (var blockInfo in blockInfos)
+                {
+                    switch (blockInfo.flag & 0x3F)
                     {
                         default://None
                             {
-                                assetsDataStream.Write(compressedBytes, 0, compressedSize);
+                                bundleReader.BaseStream.CopyTo(dataStream, blockInfo.compressedSize);
                                 break;
                             }
                         case 1://LZMA
                             {
-                                var uncompressedBytes = new byte[uncompressedSize];
+                                SevenZipHelper.StreamDecompress(bundleReader.BaseStream, dataStream, blockInfo.compressedSize, blockInfo.uncompressedSize);
-                                using (var mstream = new MemoryStream(compressedBytes))
-                                {
-                                    var decoder = SevenZipHelper.StreamDecompress(mstream, uncompressedSize);
-                                    decoder.Read(uncompressedBytes, 0, uncompressedSize);
-                                    decoder.Dispose();
-                                }
-                                assetsDataStream.Write(uncompressedBytes, 0, uncompressedSize);
                                 break;
                             }
                         case 2://LZ4
                         case 3://LZ4HC
                             {
-                                var uncompressedBytes = new byte[uncompressedSize];
+                                var lz4Stream = new Lz4DecoderStream(bundleReader.BaseStream, blockInfo.compressedSize);
-                                using (var decoder = new Lz4DecoderStream(new MemoryStream(compressedBytes)))
+                                lz4Stream.CopyTo(dataStream, blockInfo.uncompressedSize);
-                                {
-                                    decoder.Read(uncompressedBytes, 0, uncompressedSize);
-                                }
-                                assetsDataStream.Write(uncompressedBytes, 0, uncompressedSize);
                                 break;
                             }
                             //case 4:LZHAM?
                     }
                 }
-                using (var assetsDataReader = new EndianBinaryReader(assetsDataStream))
+                dataStream.Position = 0;
+                using (dataStream)
                 {
-                    var entryinfo_count = blocksInfo.ReadInt32();
+                    var entryinfo_count = blocksInfoReader.ReadInt32();
                     for (int i = 0; i < entryinfo_count; i++)
                     {
-                        var file = new MemoryFile();
+                        var file = new StreamFile();
-                        var entryinfo_offset = blocksInfo.ReadInt64();
+                        var entryinfo_offset = blocksInfoReader.ReadInt64();
-                        var entryinfo_size = blocksInfo.ReadInt64();
+                        var entryinfo_size = blocksInfoReader.ReadInt64();
-                        flag = blocksInfo.ReadInt32();
+                        flag = blocksInfoReader.ReadInt32();
-                        file.fileName = Path.GetFileName(blocksInfo.ReadStringToNull());
+                        file.fileName = Path.GetFileName(blocksInfoReader.ReadStringToNull());
-                        assetsDataReader.Position = entryinfo_offset;
+                        if (entryinfo_size > int.MaxValue)
-                        var buffer = assetsDataReader.ReadBytes((int)entryinfo_size);
+                        {
-                        file.stream = new MemoryStream(buffer);
+                            /*var memoryMappedFile = MemoryMappedFile.CreateNew(file.fileName, entryinfo_size);
+                            file.stream = memoryMappedFile.CreateViewStream();*/
+                            var extractPath = path + "_unpacked\\";
+                            Directory.CreateDirectory(extractPath);
+                            file.stream = new FileStream(extractPath + file.fileName, FileMode.Create);
+                        }
+                        else
+                        {
+                            file.stream = new MemoryStream();
+                        }
+                        dataStream.Position = entryinfo_offset;
+                        dataStream.CopyTo(file.stream, entryinfo_size);
+                        file.stream.Position = 0;
                         fileList.Add(file);
                     }
                 }

AssetStudio/StudioClasses/Importer.cs

@@ -100,7 +100,7 @@ namespace AssetStudio
         {
             var fileName = Path.GetFileName(fullName);
             StatusStripUpdate("Decompressing " + fileName);
-            var bundleFile = new BundleFile(reader);
+            var bundleFile = new BundleFile(reader, fullName);
             reader.Dispose();
             foreach (var file in bundleFile.fileList)
             {

AssetStudio/StudioClasses/SevenZipHelper.cs

@@ -0,0 +1,48 @@
+using System;
+using System.IO;
+using SevenZip.Compression.LZMA;
+
+
+namespace AssetStudio
+{
+    public static class SevenZipHelper
+    {
+        public static MemoryStream StreamDecompress(MemoryStream inStream)
+        {
+            var decoder = new Decoder();
+
+            inStream.Seek(0, 0);
+            var newOutStream = new MemoryStream();
+
+            var properties = new byte[5];
+            if (inStream.Read(properties, 0, 5) != 5)
+                throw (new Exception("input .lzma is too short"));
+            long outSize = 0;
+            for (var i = 0; i < 8; i++)
+            {
+                var v = inStream.ReadByte();
+                if (v < 0)
+                    throw (new Exception("Can't Read 1"));
+                outSize |= ((long)(byte)v) << (8 * i);
+            }
+            decoder.SetDecoderProperties(properties);
+
+            var compressedSize = inStream.Length - inStream.Position;
+            decoder.Code(inStream, newOutStream, compressedSize, outSize, null);
+
+            newOutStream.Position = 0;
+            return newOutStream;
+        }
+
+        public static void StreamDecompress(Stream inStream, Stream outStream, long inSize, long outSize)
+        {
+            var decoder = new Decoder();
+            var properties = new byte[5];
+            if (inStream.Read(properties, 0, 5) != 5)
+                throw new Exception("input .lzma is too short");
+            decoder.SetDecoderProperties(properties);
+            inSize -= 5L;
+            decoder.Code(inStream, outStream, inSize, outSize, null);
+        }
+    }
+}

AssetStudio/StudioClasses/StreamExtensions.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace AssetStudio
+{
+    public static class StreamExtensions
+    {
+        private const int BufferSize = 81920;
+
+        public static void CopyTo(this Stream source, Stream destination, long size)
+        {
+            var buffer = new byte[BufferSize];
+            for (var left = size; left > 0; left -= BufferSize)
+            {
+                int toRead = BufferSize < left ? BufferSize : (int)left;
+                int read = source.Read(buffer, 0, toRead);
+                destination.Write(buffer, 0, read);
+                if (read != toRead)
+                {
+                    return;
+                }
+            }
+        }
+    }
+}

AssetStudio/StudioClasses/Studio.cs

@@ -36,7 +36,7 @@ namespace AssetStudio
             WebFile
         }
 
-        public static FileType CheckFileType(MemoryStream stream, out EndianBinaryReader reader)
+        public static FileType CheckFileType(Stream stream, out EndianBinaryReader reader)
         {
             reader = new EndianBinaryReader(stream);
             return CheckFileType(reader);
@@ -103,48 +103,48 @@ namespace AssetStudio
 
         private static int ExtractBundleFile(string bundleFileName, EndianBinaryReader reader)
         {
-            var bundleFile = new BundleFile(reader);
+            StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFileName)} ...");
+            var bundleFile = new BundleFile(reader, bundleFileName);
             reader.Dispose();
             if (bundleFile.fileList.Count > 0)
             {
-                StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFileName)} ...");
                 var extractPath = bundleFileName + "_unpacked\\";
                 Directory.CreateDirectory(extractPath);
-                return ExtractMemoryFile(extractPath, bundleFile.fileList);
+                return ExtractStreamFile(extractPath, bundleFile.fileList);
             }
             return 0;
         }
 
         private static int ExtractWebDataFile(string webFileName, EndianBinaryReader reader)
         {
+            StatusStripUpdate($"Decompressing {Path.GetFileName(webFileName)} ...");
             var webFile = new WebFile(reader);
             reader.Dispose();
             if (webFile.fileList.Count > 0)
             {
-                StatusStripUpdate($"Decompressing {Path.GetFileName(webFileName)} ...");
                 var extractPath = webFileName + "_unpacked\\";
                 Directory.CreateDirectory(extractPath);
-                return ExtractMemoryFile(extractPath, webFile.fileList);
+                return ExtractStreamFile(extractPath, webFile.fileList);
             }
             return 0;
         }
 
-        private static int ExtractMemoryFile(string extractPath, List<MemoryFile> fileList)
+        private static int ExtractStreamFile(string extractPath, List<StreamFile> fileList)
         {
             int extractedCount = 0;
-            foreach (var memFile in fileList)
+            foreach (var file in fileList)
             {
-                var filePath = extractPath + memFile.fileName;
+                var filePath = extractPath + file.fileName;
                 if (!Directory.Exists(extractPath))
                 {
                     Directory.CreateDirectory(extractPath);
                 }
-                if (!File.Exists(filePath))
+                if (!File.Exists(filePath) && file.stream is MemoryStream stream)
                 {
-                    File.WriteAllBytes(filePath, memFile.stream.ToArray());
+                    File.WriteAllBytes(filePath, stream.ToArray());
-                    memFile.stream.Dispose();
                     extractedCount += 1;
                 }
+                file.stream.Dispose();
             }
             return extractedCount;
         }

AssetStudio/StudioClasses/WebFile.cs

@@ -12,7 +12,7 @@ namespace AssetStudio
     {
         public static byte[] gzipMagic = { 0x1f, 0x8b };
         public static byte[] brotliMagic = { 0x62, 0x72, 0x6F, 0x74, 0x6C, 0x69 };
-        public List<MemoryFile> fileList = new List<MemoryFile>();
+        public List<StreamFile> fileList = new List<StreamFile>();
 
 
         public class WebData
@@ -82,7 +82,7 @@ namespace AssetStudio
 
             foreach (var data in dataList)
             {
-                var file = new MemoryFile();
+                var file = new StreamFile();
                 file.fileName = Path.GetFileName(data.path);
                 reader.Position = data.dataOffset;
                 file.stream = new MemoryStream(reader.ReadBytes(data.dataLength));