mirror/AssetStudio
1
0
Fork 0
mirror of https://github.com/aelurum/AssetStudio.git synced 2025-05-25 05:40:21 -04:00
Code Issues Packages Projects Releases Wiki Activity

support load over 2GB file, Fixed #149

Browse Source
This commit is contained in:
Perfare 2018-07-14 23:46:32 +08:00
parent a3e430d98d
commit d39e24246e
9 changed files with 172 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
AssetStudio/7zip/SevenZipHelper.cs
View File

@ -1,144 +0,0 @@
using System;
using System.IO;
namespace SevenZip.Compression.LZMA
{
public static class SevenZipHelper
{
static int dictionary = 1 << 23;
// static Int32 posStateBits = 2;
// static Int32 litContextBits = 3; // for normal files
// UInt32 litContextBits = 0; // for 32-bit data
// static Int32 litPosBits = 0;
// UInt32 litPosBits = 2; // for 32-bit data
// static Int32 algorithm = 2;
// static Int32 numFastBytes = 128;
static bool eos = false;
static CoderPropID[] propIDs =
{
CoderPropID.DictionarySize,
CoderPropID.PosStateBits,
CoderPropID.LitContextBits,
CoderPropID.LitPosBits,
CoderPropID.Algorithm,
CoderPropID.NumFastBytes,
CoderPropID.MatchFinder,
CoderPropID.EndMarker
};
// these are the default properties, keeping it simple for now:
static object[] properties =
{
(Int32)(dictionary),
(Int32)(2),
(Int32)(3),
(Int32)(0),
(Int32)(2),
(Int32)(128),
"bt4",
eos
};
public static byte[] Compress(byte[] inputBytes)
{
MemoryStream inStream = new MemoryStream(inputBytes);
MemoryStream outStream = new MemoryStream();
Encoder encoder = new Encoder();
encoder.SetCoderProperties(propIDs, properties);
encoder.WriteCoderProperties(outStream);
long fileSize = inStream.Length;
for (int i = 0; i < 8; i++)
outStream.WriteByte((Byte)(fileSize >> (8 * i)));
encoder.Code(inStream, outStream, -1, -1, null);
return outStream.ToArray();
}
public static byte[] Decompress(byte[] inputBytes)
{
MemoryStream newInStream = new MemoryStream(inputBytes);
Decoder decoder = new Decoder();
newInStream.Seek(0, 0);
MemoryStream newOutStream = new MemoryStream();
byte[] properties2 = new byte[5];
if (newInStream.Read(properties2, 0, 5) != 5)
throw (new Exception("input .lzma is too short"));
long outSize = 0;
for (int i = 0; i < 8; i++)
{
int v = newInStream.ReadByte();
if (v < 0)
throw (new Exception("Can't Read 1"));
outSize |= ((long)(byte)v) << (8 * i);
}
decoder.SetDecoderProperties(properties2);
long compressedSize = newInStream.Length - newInStream.Position;
decoder.Code(newInStream, newOutStream, compressedSize, outSize, null);
byte[] b = newOutStream.ToArray();
return b;
}
public static MemoryStream StreamDecompress(MemoryStream newInStream)
{
Decoder decoder = new Decoder();
newInStream.Seek(0, 0);
MemoryStream newOutStream = new MemoryStream();
byte[] properties2 = new byte[5];
if (newInStream.Read(properties2, 0, 5) != 5)
throw (new Exception("input .lzma is too short"));
long outSize = 0;
for (int i = 0; i < 8; i++)
{
int v = newInStream.ReadByte();
if (v < 0)
throw (new Exception("Can't Read 1"));
outSize |= ((long)(byte)v) << (8 * i);
}
decoder.SetDecoderProperties(properties2);
long compressedSize = newInStream.Length - newInStream.Position;
decoder.Code(newInStream, newOutStream, compressedSize, outSize, null);
newOutStream.Position = 0;
return newOutStream;
}
public static MemoryStream StreamDecompress(MemoryStream newInStream, long outSize)
{
Decoder decoder = new Decoder();
newInStream.Seek(0, 0);
MemoryStream newOutStream = new MemoryStream();
byte[] properties2 = new byte[5];
if (newInStream.Read(properties2, 0, 5) != 5)
throw (new Exception("input .lzma is too short"));
decoder.SetDecoderProperties(properties2);
long compressedSize = newInStream.Length - newInStream.Position;
decoder.Code(newInStream, newOutStream, compressedSize, outSize, null);
newOutStream.Position = 0;
return newOutStream;
}
}
}

5
AssetStudio/AssetStudio-x86.csproj
View File

@ -136,9 +136,6 @@
<Compile Include="7zip\ICoder.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="7zip\SevenZipHelper.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="AssetStudioForm.cs">
<SubType>Form</SubType>
</Compile>
@ -170,6 +167,7 @@
<Compile Include="StudioClasses\BuildTarget.cs" />
<Compile Include="StudioClasses\ModelConverter.cs" />
<Compile Include="StudioClasses\ResourcesHelper.cs" />
<Compile Include="StudioClasses\SevenZipHelper.cs" />
<Compile Include="StudioClasses\ShaderResource.Designer.cs">
<DependentUpon>ShaderResource.resx</DependentUpon>
<AutoGen>True</AutoGen>
@ -179,6 +177,7 @@
<Compile Include="StudioClasses\EndianBinaryReader.cs" />
<Compile Include="StudioClasses\Exporter.cs" />
<Compile Include="StudioClasses\Importer.cs" />
<Compile Include="StudioClasses\StreamExtensions.cs" />
<Compile Include="StudioClasses\Studio.cs" />
<Compile Include="StudioClasses\ClassIDReference.cs" />
<Compile Include="ExportOptions.cs">

3
AssetStudio/AssetStudio.csproj
View File

@ -136,7 +136,7 @@
<Compile Include="7zip\ICoder.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="7zip\SevenZipHelper.cs">
<Compile Include="StudioClasses\SevenZipHelper.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="Classes\Animation.cs" />
@ -172,6 +172,7 @@
<Compile Include="StudioClasses\SpriteHelper.cs" />
<Compile Include="StudioClasses\Exporter.cs" />
<Compile Include="StudioClasses\Importer.cs" />
<Compile Include="StudioClasses\StreamExtensions.cs" />
<Compile Include="StudioClasses\Studio.cs" />
<Compile Include="StudioClasses\EndianBinaryReader.cs" />
<Compile Include="ExportOptions.cs">

124
AssetStudio/StudioClasses/BundleFile.cs
View File

@ -1,25 +1,34 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Lz4;
using SevenZip.Compression.LZMA;
namespace AssetStudio
{
public class MemoryFile
public class StreamFile
{
public string fileName;
public MemoryStream stream;
public Stream stream;
}
public class BlockInfo
{
public uint compressedSize;
public uint uncompressedSize;
public short flag;
}
public class BundleFile
{
public int format;
private string path;
public string versionPlayer;
public string versionEngine;
public List<MemoryFile> fileList = new List<MemoryFile>();
public List<StreamFile> fileList = new List<StreamFile>();
public BundleFile(EndianBinaryReader bundleReader)
public BundleFile(EndianBinaryReader bundleReader, string path)
{
this.path = path;
var signature = bundleReader.ReadStringToNull();
switch (signature)
{
@ -27,7 +36,7 @@ namespace AssetStudio
case "UnityRaw":
case "\xFA\xFA\xFA\xFA\xFA\xFA\xFA\xFA":
{
format = bundleReader.ReadInt32();
var format = bundleReader.ReadInt32();
versionPlayer = bundleReader.ReadStringToNull();
versionEngine = bundleReader.ReadStringToNull();
if (format < 6)
@ -75,14 +84,16 @@ namespace AssetStudio
break;
}
case "UnityFS":
format = bundleReader.ReadInt32();
versionPlayer = bundleReader.ReadStringToNull();
versionEngine = bundleReader.ReadStringToNull();
if (format == 6)
{
ReadFormat6(bundleReader);
var format = bundleReader.ReadInt32();
versionPlayer = bundleReader.ReadStringToNull();
versionEngine = bundleReader.ReadStringToNull();
if (format == 6)
{
ReadFormat6(bundleReader);
}
break;
}
break;
}
}
@ -91,7 +102,7 @@ namespace AssetStudio
int fileCount = reader.ReadInt32();
for (int i = 0; i < fileCount; i++)
{
var file = new MemoryFile();
var file = new StreamFile();
file.fileName = reader.ReadStringToNull();
int fileOffset = reader.ReadInt32();
fileOffset += offset;
@ -151,63 +162,82 @@ namespace AssetStudio
}
//case 4:LZHAM?
}
using (var blocksInfo = new EndianBinaryReader(blocksInfoStream))
using (var blocksInfoReader = new EndianBinaryReader(blocksInfoStream))
{
blocksInfo.Position = 0x10;
int blockcount = blocksInfo.ReadInt32();
var assetsDataStream = new MemoryStream();
blocksInfoReader.Position = 0x10;
int blockcount = blocksInfoReader.ReadInt32();
var blockInfos = new BlockInfo[blockcount];
for (int i = 0; i < blockcount; i++)
{
uncompressedSize = blocksInfo.ReadInt32();
compressedSize = blocksInfo.ReadInt32();
flag = blocksInfo.ReadInt16();
var compressedBytes = bundleReader.ReadBytes(compressedSize);
switch (flag & 0x3F)
blockInfos[i] = new BlockInfo
{
uncompressedSize = blocksInfoReader.ReadUInt32(),
compressedSize = blocksInfoReader.ReadUInt32(),
flag = blocksInfoReader.ReadInt16()
};
}
Stream dataStream;
var uncompressedSizeSum = blockInfos.Sum(x => x.uncompressedSize);
if (uncompressedSizeSum > int.MaxValue)
{
/*var memoryMappedFile = MemoryMappedFile.CreateNew(Path.GetFileName(path), uncompressedSizeSum);
assetsDataStream = memoryMappedFile.CreateViewStream();*/
dataStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose);
}
else
{
dataStream = new MemoryStream();
}
foreach (var blockInfo in blockInfos)
{
switch (blockInfo.flag & 0x3F)
{
default://None
{
assetsDataStream.Write(compressedBytes, 0, compressedSize);
bundleReader.BaseStream.CopyTo(dataStream, blockInfo.compressedSize);
break;
}
case 1://LZMA
{
var uncompressedBytes = new byte[uncompressedSize];
using (var mstream = new MemoryStream(compressedBytes))
{
var decoder = SevenZipHelper.StreamDecompress(mstream, uncompressedSize);
decoder.Read(uncompressedBytes, 0, uncompressedSize);
decoder.Dispose();
}
assetsDataStream.Write(uncompressedBytes, 0, uncompressedSize);
SevenZipHelper.StreamDecompress(bundleReader.BaseStream, dataStream, blockInfo.compressedSize, blockInfo.uncompressedSize);
break;
}
case 2://LZ4
case 3://LZ4HC
{
var uncompressedBytes = new byte[uncompressedSize];
using (var decoder = new Lz4DecoderStream(new MemoryStream(compressedBytes)))
{
decoder.Read(uncompressedBytes, 0, uncompressedSize);
}
assetsDataStream.Write(uncompressedBytes, 0, uncompressedSize);
var lz4Stream = new Lz4DecoderStream(bundleReader.BaseStream, blockInfo.compressedSize);
lz4Stream.CopyTo(dataStream, blockInfo.uncompressedSize);
break;
}
//case 4:LZHAM?
}
}
using (var assetsDataReader = new EndianBinaryReader(assetsDataStream))
dataStream.Position = 0;
using (dataStream)
{
var entryinfo_count = blocksInfo.ReadInt32();
var entryinfo_count = blocksInfoReader.ReadInt32();
for (int i = 0; i < entryinfo_count; i++)
{
var file = new MemoryFile();
var entryinfo_offset = blocksInfo.ReadInt64();
var entryinfo_size = blocksInfo.ReadInt64();
flag = blocksInfo.ReadInt32();
file.fileName = Path.GetFileName(blocksInfo.ReadStringToNull());
assetsDataReader.Position = entryinfo_offset;
var buffer = assetsDataReader.ReadBytes((int)entryinfo_size);
file.stream = new MemoryStream(buffer);
var file = new StreamFile();
var entryinfo_offset = blocksInfoReader.ReadInt64();
var entryinfo_size = blocksInfoReader.ReadInt64();
flag = blocksInfoReader.ReadInt32();
file.fileName = Path.GetFileName(blocksInfoReader.ReadStringToNull());
if (entryinfo_size > int.MaxValue)
{
/*var memoryMappedFile = MemoryMappedFile.CreateNew(file.fileName, entryinfo_size);
file.stream = memoryMappedFile.CreateViewStream();*/
var extractPath = path + "_unpacked\\";
Directory.CreateDirectory(extractPath);
file.stream = new FileStream(extractPath + file.fileName, FileMode.Create);
}
else
{
file.stream = new MemoryStream();
}
dataStream.Position = entryinfo_offset;
dataStream.CopyTo(file.stream, entryinfo_size);
file.stream.Position = 0;
fileList.Add(file);
}
}

2
AssetStudio/StudioClasses/Importer.cs
View File

@ -100,7 +100,7 @@ namespace AssetStudio
{
var fileName = Path.GetFileName(fullName);
StatusStripUpdate("Decompressing " + fileName);
var bundleFile = new BundleFile(reader);
var bundleFile = new BundleFile(reader, fullName);
reader.Dispose();
foreach (var file in bundleFile.fileList)
{

48
AssetStudio/StudioClasses/SevenZipHelper.cs Normal file
View File

@ -0,0 +1,48 @@
using System;
using System.IO;
using SevenZip.Compression.LZMA;
namespace AssetStudio
{
public static class SevenZipHelper
{
public static MemoryStream StreamDecompress(MemoryStream inStream)
{
var decoder = new Decoder();
inStream.Seek(0, 0);
var newOutStream = new MemoryStream();
var properties = new byte[5];
if (inStream.Read(properties, 0, 5) != 5)
throw (new Exception("input .lzma is too short"));
long outSize = 0;
for (var i = 0; i < 8; i++)
{
var v = inStream.ReadByte();
if (v < 0)
throw (new Exception("Can't Read 1"));
outSize |= ((long)(byte)v) << (8 * i);
}
decoder.SetDecoderProperties(properties);
var compressedSize = inStream.Length - inStream.Position;
decoder.Code(inStream, newOutStream, compressedSize, outSize, null);
newOutStream.Position = 0;
return newOutStream;
}
public static void StreamDecompress(Stream inStream, Stream outStream, long inSize, long outSize)
{
var decoder = new Decoder();
var properties = new byte[5];
if (inStream.Read(properties, 0, 5) != 5)
throw new Exception("input .lzma is too short");
decoder.SetDecoderProperties(properties);
inSize -= 5L;
decoder.Code(inStream, outStream, inSize, outSize, null);
}
}
}

28
AssetStudio/StudioClasses/StreamExtensions.cs Normal file
View File

@ -0,0 +1,28 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace AssetStudio
{
public static class StreamExtensions
{
private const int BufferSize = 81920;
public static void CopyTo(this Stream source, Stream destination, long size)
{
var buffer = new byte[BufferSize];
for (var left = size; left > 0; left -= BufferSize)
{
int toRead = BufferSize < left ? BufferSize : (int)left;
int read = source.Read(buffer, 0, toRead);
destination.Write(buffer, 0, read);
if (read != toRead)
{
return;
}
}
}
}
}

24
AssetStudio/StudioClasses/Studio.cs
View File

@ -36,7 +36,7 @@ namespace AssetStudio
WebFile
}
public static FileType CheckFileType(MemoryStream stream, out EndianBinaryReader reader)
public static FileType CheckFileType(Stream stream, out EndianBinaryReader reader)
{
reader = new EndianBinaryReader(stream);
return CheckFileType(reader);
@ -103,48 +103,48 @@ namespace AssetStudio
private static int ExtractBundleFile(string bundleFileName, EndianBinaryReader reader)
{
var bundleFile = new BundleFile(reader);
StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFileName)} ...");
var bundleFile = new BundleFile(reader, bundleFileName);
reader.Dispose();
if (bundleFile.fileList.Count > 0)
{
StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFileName)} ...");
var extractPath = bundleFileName + "_unpacked\\";
Directory.CreateDirectory(extractPath);
return ExtractMemoryFile(extractPath, bundleFile.fileList);
return ExtractStreamFile(extractPath, bundleFile.fileList);
}
return 0;
}
private static int ExtractWebDataFile(string webFileName, EndianBinaryReader reader)
{
StatusStripUpdate($"Decompressing {Path.GetFileName(webFileName)} ...");
var webFile = new WebFile(reader);
reader.Dispose();
if (webFile.fileList.Count > 0)
{
StatusStripUpdate($"Decompressing {Path.GetFileName(webFileName)} ...");
var extractPath = webFileName + "_unpacked\\";
Directory.CreateDirectory(extractPath);
return ExtractMemoryFile(extractPath, webFile.fileList);
return ExtractStreamFile(extractPath, webFile.fileList);
}
return 0;
}
private static int ExtractMemoryFile(string extractPath, List<MemoryFile> fileList)
private static int ExtractStreamFile(string extractPath, List<StreamFile> fileList)
{
int extractedCount = 0;
foreach (var memFile in fileList)
foreach (var file in fileList)
{
var filePath = extractPath + memFile.fileName;
var filePath = extractPath + file.fileName;
if (!Directory.Exists(extractPath))
{
Directory.CreateDirectory(extractPath);
}
if (!File.Exists(filePath))
if (!File.Exists(filePath) && file.stream is MemoryStream stream)
{
File.WriteAllBytes(filePath, memFile.stream.ToArray());
memFile.stream.Dispose();
File.WriteAllBytes(filePath, stream.ToArray());
extractedCount += 1;
}
file.stream.Dispose();
}
return extractedCount;
}

4
AssetStudio/StudioClasses/WebFile.cs
View File

@ -12,7 +12,7 @@ namespace AssetStudio
{
public static byte[] gzipMagic = { 0x1f, 0x8b };
public static byte[] brotliMagic = { 0x62, 0x72, 0x6F, 0x74, 0x6C, 0x69 };
public List<MemoryFile> fileList = new List<MemoryFile>();
public List<StreamFile> fileList = new List<StreamFile>();
public class WebData
@ -82,7 +82,7 @@ namespace AssetStudio
foreach (var data in dataList)
{
var file = new MemoryFile();
var file = new StreamFile();
file.fileName = Path.GetFileName(data.path);
reader.Position = data.dataOffset;
file.stream = new MemoryStream(reader.ReadBytes(data.dataLength));