support load over 2GB file, Fixed #149

This commit is contained in:
Perfare 2018-07-14 23:46:32 +08:00
parent a3e430d98d
commit d39e24246e
9 changed files with 172 additions and 210 deletions

View File

@ -1,144 +0,0 @@
using System;
using System.IO;
namespace SevenZip.Compression.LZMA
{
public static class SevenZipHelper
{
static int dictionary = 1 << 23;
// static Int32 posStateBits = 2;
// static Int32 litContextBits = 3; // for normal files
// UInt32 litContextBits = 0; // for 32-bit data
// static Int32 litPosBits = 0;
// UInt32 litPosBits = 2; // for 32-bit data
// static Int32 algorithm = 2;
// static Int32 numFastBytes = 128;
static bool eos = false;
static CoderPropID[] propIDs =
{
CoderPropID.DictionarySize,
CoderPropID.PosStateBits,
CoderPropID.LitContextBits,
CoderPropID.LitPosBits,
CoderPropID.Algorithm,
CoderPropID.NumFastBytes,
CoderPropID.MatchFinder,
CoderPropID.EndMarker
};
// these are the default properties, keeping it simple for now:
static object[] properties =
{
(Int32)(dictionary),
(Int32)(2),
(Int32)(3),
(Int32)(0),
(Int32)(2),
(Int32)(128),
"bt4",
eos
};
public static byte[] Compress(byte[] inputBytes)
{
MemoryStream inStream = new MemoryStream(inputBytes);
MemoryStream outStream = new MemoryStream();
Encoder encoder = new Encoder();
encoder.SetCoderProperties(propIDs, properties);
encoder.WriteCoderProperties(outStream);
long fileSize = inStream.Length;
for (int i = 0; i < 8; i++)
outStream.WriteByte((Byte)(fileSize >> (8 * i)));
encoder.Code(inStream, outStream, -1, -1, null);
return outStream.ToArray();
}
public static byte[] Decompress(byte[] inputBytes)
{
MemoryStream newInStream = new MemoryStream(inputBytes);
Decoder decoder = new Decoder();
newInStream.Seek(0, 0);
MemoryStream newOutStream = new MemoryStream();
byte[] properties2 = new byte[5];
if (newInStream.Read(properties2, 0, 5) != 5)
throw (new Exception("input .lzma is too short"));
long outSize = 0;
for (int i = 0; i < 8; i++)
{
int v = newInStream.ReadByte();
if (v < 0)
throw (new Exception("Can't Read 1"));
outSize |= ((long)(byte)v) << (8 * i);
}
decoder.SetDecoderProperties(properties2);
long compressedSize = newInStream.Length - newInStream.Position;
decoder.Code(newInStream, newOutStream, compressedSize, outSize, null);
byte[] b = newOutStream.ToArray();
return b;
}
public static MemoryStream StreamDecompress(MemoryStream newInStream)
{
Decoder decoder = new Decoder();
newInStream.Seek(0, 0);
MemoryStream newOutStream = new MemoryStream();
byte[] properties2 = new byte[5];
if (newInStream.Read(properties2, 0, 5) != 5)
throw (new Exception("input .lzma is too short"));
long outSize = 0;
for (int i = 0; i < 8; i++)
{
int v = newInStream.ReadByte();
if (v < 0)
throw (new Exception("Can't Read 1"));
outSize |= ((long)(byte)v) << (8 * i);
}
decoder.SetDecoderProperties(properties2);
long compressedSize = newInStream.Length - newInStream.Position;
decoder.Code(newInStream, newOutStream, compressedSize, outSize, null);
newOutStream.Position = 0;
return newOutStream;
}
public static MemoryStream StreamDecompress(MemoryStream newInStream, long outSize)
{
Decoder decoder = new Decoder();
newInStream.Seek(0, 0);
MemoryStream newOutStream = new MemoryStream();
byte[] properties2 = new byte[5];
if (newInStream.Read(properties2, 0, 5) != 5)
throw (new Exception("input .lzma is too short"));
decoder.SetDecoderProperties(properties2);
long compressedSize = newInStream.Length - newInStream.Position;
decoder.Code(newInStream, newOutStream, compressedSize, outSize, null);
newOutStream.Position = 0;
return newOutStream;
}
}
}

View File

@ -136,9 +136,6 @@
<Compile Include="7zip\ICoder.cs"> <Compile Include="7zip\ICoder.cs">
<SubType>Code</SubType> <SubType>Code</SubType>
</Compile> </Compile>
<Compile Include="7zip\SevenZipHelper.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="AssetStudioForm.cs"> <Compile Include="AssetStudioForm.cs">
<SubType>Form</SubType> <SubType>Form</SubType>
</Compile> </Compile>
@ -170,6 +167,7 @@
<Compile Include="StudioClasses\BuildTarget.cs" /> <Compile Include="StudioClasses\BuildTarget.cs" />
<Compile Include="StudioClasses\ModelConverter.cs" /> <Compile Include="StudioClasses\ModelConverter.cs" />
<Compile Include="StudioClasses\ResourcesHelper.cs" /> <Compile Include="StudioClasses\ResourcesHelper.cs" />
<Compile Include="StudioClasses\SevenZipHelper.cs" />
<Compile Include="StudioClasses\ShaderResource.Designer.cs"> <Compile Include="StudioClasses\ShaderResource.Designer.cs">
<DependentUpon>ShaderResource.resx</DependentUpon> <DependentUpon>ShaderResource.resx</DependentUpon>
<AutoGen>True</AutoGen> <AutoGen>True</AutoGen>
@ -179,6 +177,7 @@
<Compile Include="StudioClasses\EndianBinaryReader.cs" /> <Compile Include="StudioClasses\EndianBinaryReader.cs" />
<Compile Include="StudioClasses\Exporter.cs" /> <Compile Include="StudioClasses\Exporter.cs" />
<Compile Include="StudioClasses\Importer.cs" /> <Compile Include="StudioClasses\Importer.cs" />
<Compile Include="StudioClasses\StreamExtensions.cs" />
<Compile Include="StudioClasses\Studio.cs" /> <Compile Include="StudioClasses\Studio.cs" />
<Compile Include="StudioClasses\ClassIDReference.cs" /> <Compile Include="StudioClasses\ClassIDReference.cs" />
<Compile Include="ExportOptions.cs"> <Compile Include="ExportOptions.cs">

View File

@ -136,7 +136,7 @@
<Compile Include="7zip\ICoder.cs"> <Compile Include="7zip\ICoder.cs">
<SubType>Code</SubType> <SubType>Code</SubType>
</Compile> </Compile>
<Compile Include="7zip\SevenZipHelper.cs"> <Compile Include="StudioClasses\SevenZipHelper.cs">
<SubType>Code</SubType> <SubType>Code</SubType>
</Compile> </Compile>
<Compile Include="Classes\Animation.cs" /> <Compile Include="Classes\Animation.cs" />
@ -172,6 +172,7 @@
<Compile Include="StudioClasses\SpriteHelper.cs" /> <Compile Include="StudioClasses\SpriteHelper.cs" />
<Compile Include="StudioClasses\Exporter.cs" /> <Compile Include="StudioClasses\Exporter.cs" />
<Compile Include="StudioClasses\Importer.cs" /> <Compile Include="StudioClasses\Importer.cs" />
<Compile Include="StudioClasses\StreamExtensions.cs" />
<Compile Include="StudioClasses\Studio.cs" /> <Compile Include="StudioClasses\Studio.cs" />
<Compile Include="StudioClasses\EndianBinaryReader.cs" /> <Compile Include="StudioClasses\EndianBinaryReader.cs" />
<Compile Include="ExportOptions.cs"> <Compile Include="ExportOptions.cs">

View File

@ -1,25 +1,34 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq;
using Lz4; using Lz4;
using SevenZip.Compression.LZMA; using SevenZip.Compression.LZMA;
namespace AssetStudio namespace AssetStudio
{ {
public class MemoryFile public class StreamFile
{ {
public string fileName; public string fileName;
public MemoryStream stream; public Stream stream;
}
public class BlockInfo
{
public uint compressedSize;
public uint uncompressedSize;
public short flag;
} }
public class BundleFile public class BundleFile
{ {
public int format; private string path;
public string versionPlayer; public string versionPlayer;
public string versionEngine; public string versionEngine;
public List<MemoryFile> fileList = new List<MemoryFile>(); public List<StreamFile> fileList = new List<StreamFile>();
public BundleFile(EndianBinaryReader bundleReader) public BundleFile(EndianBinaryReader bundleReader, string path)
{ {
this.path = path;
var signature = bundleReader.ReadStringToNull(); var signature = bundleReader.ReadStringToNull();
switch (signature) switch (signature)
{ {
@ -27,7 +36,7 @@ namespace AssetStudio
case "UnityRaw": case "UnityRaw":
case "\xFA\xFA\xFA\xFA\xFA\xFA\xFA\xFA": case "\xFA\xFA\xFA\xFA\xFA\xFA\xFA\xFA":
{ {
format = bundleReader.ReadInt32(); var format = bundleReader.ReadInt32();
versionPlayer = bundleReader.ReadStringToNull(); versionPlayer = bundleReader.ReadStringToNull();
versionEngine = bundleReader.ReadStringToNull(); versionEngine = bundleReader.ReadStringToNull();
if (format < 6) if (format < 6)
@ -75,7 +84,8 @@ namespace AssetStudio
break; break;
} }
case "UnityFS": case "UnityFS":
format = bundleReader.ReadInt32(); {
var format = bundleReader.ReadInt32();
versionPlayer = bundleReader.ReadStringToNull(); versionPlayer = bundleReader.ReadStringToNull();
versionEngine = bundleReader.ReadStringToNull(); versionEngine = bundleReader.ReadStringToNull();
if (format == 6) if (format == 6)
@ -85,13 +95,14 @@ namespace AssetStudio
break; break;
} }
} }
}
private void GetAssetsFiles(EndianBinaryReader reader, int offset) private void GetAssetsFiles(EndianBinaryReader reader, int offset)
{ {
int fileCount = reader.ReadInt32(); int fileCount = reader.ReadInt32();
for (int i = 0; i < fileCount; i++) for (int i = 0; i < fileCount; i++)
{ {
var file = new MemoryFile(); var file = new StreamFile();
file.fileName = reader.ReadStringToNull(); file.fileName = reader.ReadStringToNull();
int fileOffset = reader.ReadInt32(); int fileOffset = reader.ReadInt32();
fileOffset += offset; fileOffset += offset;
@ -151,63 +162,82 @@ namespace AssetStudio
} }
//case 4:LZHAM? //case 4:LZHAM?
} }
using (var blocksInfo = new EndianBinaryReader(blocksInfoStream)) using (var blocksInfoReader = new EndianBinaryReader(blocksInfoStream))
{ {
blocksInfo.Position = 0x10; blocksInfoReader.Position = 0x10;
int blockcount = blocksInfo.ReadInt32(); int blockcount = blocksInfoReader.ReadInt32();
var assetsDataStream = new MemoryStream(); var blockInfos = new BlockInfo[blockcount];
for (int i = 0; i < blockcount; i++) for (int i = 0; i < blockcount; i++)
{ {
uncompressedSize = blocksInfo.ReadInt32(); blockInfos[i] = new BlockInfo
compressedSize = blocksInfo.ReadInt32(); {
flag = blocksInfo.ReadInt16(); uncompressedSize = blocksInfoReader.ReadUInt32(),
var compressedBytes = bundleReader.ReadBytes(compressedSize); compressedSize = blocksInfoReader.ReadUInt32(),
switch (flag & 0x3F) flag = blocksInfoReader.ReadInt16()
};
}
Stream dataStream;
var uncompressedSizeSum = blockInfos.Sum(x => x.uncompressedSize);
if (uncompressedSizeSum > int.MaxValue)
{
/*var memoryMappedFile = MemoryMappedFile.CreateNew(Path.GetFileName(path), uncompressedSizeSum);
assetsDataStream = memoryMappedFile.CreateViewStream();*/
dataStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose);
}
else
{
dataStream = new MemoryStream();
}
foreach (var blockInfo in blockInfos)
{
switch (blockInfo.flag & 0x3F)
{ {
default://None default://None
{ {
assetsDataStream.Write(compressedBytes, 0, compressedSize); bundleReader.BaseStream.CopyTo(dataStream, blockInfo.compressedSize);
break; break;
} }
case 1://LZMA case 1://LZMA
{ {
var uncompressedBytes = new byte[uncompressedSize]; SevenZipHelper.StreamDecompress(bundleReader.BaseStream, dataStream, blockInfo.compressedSize, blockInfo.uncompressedSize);
using (var mstream = new MemoryStream(compressedBytes))
{
var decoder = SevenZipHelper.StreamDecompress(mstream, uncompressedSize);
decoder.Read(uncompressedBytes, 0, uncompressedSize);
decoder.Dispose();
}
assetsDataStream.Write(uncompressedBytes, 0, uncompressedSize);
break; break;
} }
case 2://LZ4 case 2://LZ4
case 3://LZ4HC case 3://LZ4HC
{ {
var uncompressedBytes = new byte[uncompressedSize]; var lz4Stream = new Lz4DecoderStream(bundleReader.BaseStream, blockInfo.compressedSize);
using (var decoder = new Lz4DecoderStream(new MemoryStream(compressedBytes))) lz4Stream.CopyTo(dataStream, blockInfo.uncompressedSize);
{
decoder.Read(uncompressedBytes, 0, uncompressedSize);
}
assetsDataStream.Write(uncompressedBytes, 0, uncompressedSize);
break; break;
} }
//case 4:LZHAM? //case 4:LZHAM?
} }
} }
using (var assetsDataReader = new EndianBinaryReader(assetsDataStream)) dataStream.Position = 0;
using (dataStream)
{ {
var entryinfo_count = blocksInfo.ReadInt32(); var entryinfo_count = blocksInfoReader.ReadInt32();
for (int i = 0; i < entryinfo_count; i++) for (int i = 0; i < entryinfo_count; i++)
{ {
var file = new MemoryFile(); var file = new StreamFile();
var entryinfo_offset = blocksInfo.ReadInt64(); var entryinfo_offset = blocksInfoReader.ReadInt64();
var entryinfo_size = blocksInfo.ReadInt64(); var entryinfo_size = blocksInfoReader.ReadInt64();
flag = blocksInfo.ReadInt32(); flag = blocksInfoReader.ReadInt32();
file.fileName = Path.GetFileName(blocksInfo.ReadStringToNull()); file.fileName = Path.GetFileName(blocksInfoReader.ReadStringToNull());
assetsDataReader.Position = entryinfo_offset; if (entryinfo_size > int.MaxValue)
var buffer = assetsDataReader.ReadBytes((int)entryinfo_size); {
file.stream = new MemoryStream(buffer); /*var memoryMappedFile = MemoryMappedFile.CreateNew(file.fileName, entryinfo_size);
file.stream = memoryMappedFile.CreateViewStream();*/
var extractPath = path + "_unpacked\\";
Directory.CreateDirectory(extractPath);
file.stream = new FileStream(extractPath + file.fileName, FileMode.Create);
}
else
{
file.stream = new MemoryStream();
}
dataStream.Position = entryinfo_offset;
dataStream.CopyTo(file.stream, entryinfo_size);
file.stream.Position = 0;
fileList.Add(file); fileList.Add(file);
} }
} }

View File

@ -100,7 +100,7 @@ namespace AssetStudio
{ {
var fileName = Path.GetFileName(fullName); var fileName = Path.GetFileName(fullName);
StatusStripUpdate("Decompressing " + fileName); StatusStripUpdate("Decompressing " + fileName);
var bundleFile = new BundleFile(reader); var bundleFile = new BundleFile(reader, fullName);
reader.Dispose(); reader.Dispose();
foreach (var file in bundleFile.fileList) foreach (var file in bundleFile.fileList)
{ {

View File

@ -0,0 +1,48 @@
using System;
using System.IO;
using SevenZip.Compression.LZMA;
namespace AssetStudio
{
public static class SevenZipHelper
{
public static MemoryStream StreamDecompress(MemoryStream inStream)
{
var decoder = new Decoder();
inStream.Seek(0, 0);
var newOutStream = new MemoryStream();
var properties = new byte[5];
if (inStream.Read(properties, 0, 5) != 5)
throw (new Exception("input .lzma is too short"));
long outSize = 0;
for (var i = 0; i < 8; i++)
{
var v = inStream.ReadByte();
if (v < 0)
throw (new Exception("Can't Read 1"));
outSize |= ((long)(byte)v) << (8 * i);
}
decoder.SetDecoderProperties(properties);
var compressedSize = inStream.Length - inStream.Position;
decoder.Code(inStream, newOutStream, compressedSize, outSize, null);
newOutStream.Position = 0;
return newOutStream;
}
public static void StreamDecompress(Stream inStream, Stream outStream, long inSize, long outSize)
{
var decoder = new Decoder();
var properties = new byte[5];
if (inStream.Read(properties, 0, 5) != 5)
throw new Exception("input .lzma is too short");
decoder.SetDecoderProperties(properties);
inSize -= 5L;
decoder.Code(inStream, outStream, inSize, outSize, null);
}
}
}

View File

@ -0,0 +1,28 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace AssetStudio
{
public static class StreamExtensions
{
private const int BufferSize = 81920;
public static void CopyTo(this Stream source, Stream destination, long size)
{
var buffer = new byte[BufferSize];
for (var left = size; left > 0; left -= BufferSize)
{
int toRead = BufferSize < left ? BufferSize : (int)left;
int read = source.Read(buffer, 0, toRead);
destination.Write(buffer, 0, read);
if (read != toRead)
{
return;
}
}
}
}
}

View File

@ -36,7 +36,7 @@ namespace AssetStudio
WebFile WebFile
} }
public static FileType CheckFileType(MemoryStream stream, out EndianBinaryReader reader) public static FileType CheckFileType(Stream stream, out EndianBinaryReader reader)
{ {
reader = new EndianBinaryReader(stream); reader = new EndianBinaryReader(stream);
return CheckFileType(reader); return CheckFileType(reader);
@ -103,48 +103,48 @@ namespace AssetStudio
private static int ExtractBundleFile(string bundleFileName, EndianBinaryReader reader) private static int ExtractBundleFile(string bundleFileName, EndianBinaryReader reader)
{ {
var bundleFile = new BundleFile(reader); StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFileName)} ...");
var bundleFile = new BundleFile(reader, bundleFileName);
reader.Dispose(); reader.Dispose();
if (bundleFile.fileList.Count > 0) if (bundleFile.fileList.Count > 0)
{ {
StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFileName)} ...");
var extractPath = bundleFileName + "_unpacked\\"; var extractPath = bundleFileName + "_unpacked\\";
Directory.CreateDirectory(extractPath); Directory.CreateDirectory(extractPath);
return ExtractMemoryFile(extractPath, bundleFile.fileList); return ExtractStreamFile(extractPath, bundleFile.fileList);
} }
return 0; return 0;
} }
private static int ExtractWebDataFile(string webFileName, EndianBinaryReader reader) private static int ExtractWebDataFile(string webFileName, EndianBinaryReader reader)
{ {
StatusStripUpdate($"Decompressing {Path.GetFileName(webFileName)} ...");
var webFile = new WebFile(reader); var webFile = new WebFile(reader);
reader.Dispose(); reader.Dispose();
if (webFile.fileList.Count > 0) if (webFile.fileList.Count > 0)
{ {
StatusStripUpdate($"Decompressing {Path.GetFileName(webFileName)} ...");
var extractPath = webFileName + "_unpacked\\"; var extractPath = webFileName + "_unpacked\\";
Directory.CreateDirectory(extractPath); Directory.CreateDirectory(extractPath);
return ExtractMemoryFile(extractPath, webFile.fileList); return ExtractStreamFile(extractPath, webFile.fileList);
} }
return 0; return 0;
} }
private static int ExtractMemoryFile(string extractPath, List<MemoryFile> fileList) private static int ExtractStreamFile(string extractPath, List<StreamFile> fileList)
{ {
int extractedCount = 0; int extractedCount = 0;
foreach (var memFile in fileList) foreach (var file in fileList)
{ {
var filePath = extractPath + memFile.fileName; var filePath = extractPath + file.fileName;
if (!Directory.Exists(extractPath)) if (!Directory.Exists(extractPath))
{ {
Directory.CreateDirectory(extractPath); Directory.CreateDirectory(extractPath);
} }
if (!File.Exists(filePath)) if (!File.Exists(filePath) && file.stream is MemoryStream stream)
{ {
File.WriteAllBytes(filePath, memFile.stream.ToArray()); File.WriteAllBytes(filePath, stream.ToArray());
memFile.stream.Dispose();
extractedCount += 1; extractedCount += 1;
} }
file.stream.Dispose();
} }
return extractedCount; return extractedCount;
} }

View File

@ -12,7 +12,7 @@ namespace AssetStudio
{ {
public static byte[] gzipMagic = { 0x1f, 0x8b }; public static byte[] gzipMagic = { 0x1f, 0x8b };
public static byte[] brotliMagic = { 0x62, 0x72, 0x6F, 0x74, 0x6C, 0x69 }; public static byte[] brotliMagic = { 0x62, 0x72, 0x6F, 0x74, 0x6C, 0x69 };
public List<MemoryFile> fileList = new List<MemoryFile>(); public List<StreamFile> fileList = new List<StreamFile>();
public class WebData public class WebData
@ -82,7 +82,7 @@ namespace AssetStudio
foreach (var data in dataList) foreach (var data in dataList)
{ {
var file = new MemoryFile(); var file = new StreamFile();
file.fileName = Path.GetFileName(data.path); file.fileName = Path.GetFileName(data.path);
reader.Position = data.dataOffset; reader.Position = data.dataOffset;
file.stream = new MemoryStream(reader.ReadBytes(data.dataLength)); file.stream = new MemoryStream(reader.ReadBytes(data.dataLength));