refactor the file reading part

This commit is contained in:
Perfare 2021-07-02 02:17:59 +08:00
parent 4345885cc9
commit 7ab2cda120
7 changed files with 175 additions and 159 deletions

View File

@ -11,6 +11,7 @@ namespace AssetStudio
{ {
public string SpecifyUnityVersion; public string SpecifyUnityVersion;
public List<SerializedFile> assetsFileList = new List<SerializedFile>(); public List<SerializedFile> assetsFileList = new List<SerializedFile>();
internal Dictionary<string, int> assetsFileIndexCache = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase); internal Dictionary<string, int> assetsFileIndexCache = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
internal Dictionary<string, BinaryReader> resourceFileReaders = new Dictionary<string, BinaryReader>(StringComparer.OrdinalIgnoreCase); internal Dictionary<string, BinaryReader> resourceFileReaders = new Dictionary<string, BinaryReader>(StringComparer.OrdinalIgnoreCase);
@ -60,43 +61,43 @@ namespace AssetStudio
private void LoadFile(string fullName) private void LoadFile(string fullName)
{ {
switch (CheckFileType(fullName, out var reader)) var reader = new FileReader(fullName);
switch (reader.FileType)
{ {
case FileType.AssetsFile: case FileType.AssetsFile:
LoadAssetsFile(fullName, reader); LoadAssetsFile(reader);
break; break;
case FileType.BundleFile: case FileType.BundleFile:
LoadBundleFile(fullName, reader); LoadBundleFile(reader);
break; break;
case FileType.WebFile: case FileType.WebFile:
LoadWebFile(fullName, reader); LoadWebFile(reader);
break; break;
} }
} }
private void LoadAssetsFile(string fullName, EndianBinaryReader reader) private void LoadAssetsFile(FileReader reader)
{ {
var fileName = Path.GetFileName(fullName); if (!assetsFileListHash.Contains(reader.FileName))
if (!assetsFileListHash.Contains(fileName))
{ {
Logger.Info($"Loading {fileName}"); Logger.Info($"Loading {reader.FileName}");
try try
{ {
var assetsFile = new SerializedFile(this, fullName, reader); var assetsFile = new SerializedFile(reader, this);
CheckStrippedVersion(assetsFile); CheckStrippedVersion(assetsFile);
assetsFileList.Add(assetsFile); assetsFileList.Add(assetsFile);
assetsFileListHash.Add(assetsFile.fileName); assetsFileListHash.Add(assetsFile.fileName);
foreach (var sharedFile in assetsFile.m_Externals) foreach (var sharedFile in assetsFile.m_Externals)
{ {
var sharedFilePath = Path.Combine(Path.GetDirectoryName(fullName), sharedFile.fileName);
var sharedFileName = sharedFile.fileName; var sharedFileName = sharedFile.fileName;
if (!importFilesHash.Contains(sharedFileName)) if (!importFilesHash.Contains(sharedFileName))
{ {
var sharedFilePath = Path.Combine(Path.GetDirectoryName(reader.FullPath), sharedFileName);
if (!File.Exists(sharedFilePath)) if (!File.Exists(sharedFilePath))
{ {
var findFiles = Directory.GetFiles(Path.GetDirectoryName(fullName), sharedFileName, SearchOption.AllDirectories); var findFiles = Directory.GetFiles(Path.GetDirectoryName(reader.FullPath), sharedFileName, SearchOption.AllDirectories);
if (findFiles.Length > 0) if (findFiles.Length > 0)
{ {
sharedFilePath = findFiles[0]; sharedFilePath = findFiles[0];
@ -113,7 +114,7 @@ namespace AssetStudio
} }
catch (Exception e) catch (Exception e)
{ {
Logger.Error($"Error while reading assets file {fileName}", e); Logger.Error($"Error while reading assets file {reader.FileName}", e);
reader.Dispose(); reader.Dispose();
} }
} }
@ -123,14 +124,13 @@ namespace AssetStudio
} }
} }
private void LoadAssetsFromMemory(string fullName, EndianBinaryReader reader, string originalPath, string unityVersion = null) private void LoadAssetsFromMemory(FileReader reader, string originalPath, string unityVersion = null)
{ {
var fileName = Path.GetFileName(fullName); if (!assetsFileListHash.Contains(reader.FileName))
if (!assetsFileListHash.Contains(fileName))
{ {
try try
{ {
var assetsFile = new SerializedFile(this, fullName, reader); var assetsFile = new SerializedFile(reader, this);
assetsFile.originalPath = originalPath; assetsFile.originalPath = originalPath;
if (!string.IsNullOrEmpty(unityVersion) && assetsFile.header.m_Version < SerializedFileFormatVersion.kUnknown_7) if (!string.IsNullOrEmpty(unityVersion) && assetsFile.header.m_Version < SerializedFileFormatVersion.kUnknown_7)
{ {
@ -142,26 +142,25 @@ namespace AssetStudio
} }
catch (Exception e) catch (Exception e)
{ {
Logger.Error($"Error while reading assets file {fileName} from {Path.GetFileName(originalPath)}", e); Logger.Error($"Error while reading assets file {reader.FileName} from {Path.GetFileName(originalPath)}", e);
resourceFileReaders.Add(fileName, reader); resourceFileReaders.Add(reader.FileName, reader);
} }
} }
} }
private void LoadBundleFile(string fullName, EndianBinaryReader reader, string parentPath = null) private void LoadBundleFile(FileReader reader, string originalPath = null)
{ {
var fileName = Path.GetFileName(fullName); Logger.Info("Loading " + reader.FileName);
Logger.Info("Loading " + fileName);
try try
{ {
var bundleFile = new BundleFile(reader, fullName); var bundleFile = new BundleFile(reader);
foreach (var file in bundleFile.fileList) foreach (var file in bundleFile.fileList)
{ {
var subReader = new EndianBinaryReader(file.stream); var dummyPath = Path.Combine(Path.GetDirectoryName(reader.FullPath), file.fileName);
if (SerializedFile.IsSerializedFile(subReader)) var subReader = new FileReader(dummyPath, file.stream);
if (subReader.FileType == FileType.AssetsFile)
{ {
var dummyPath = Path.GetDirectoryName(fullName) + Path.DirectorySeparatorChar + file.fileName; LoadAssetsFromMemory(subReader, originalPath ?? reader.FullPath, bundleFile.m_Header.unityRevision);
LoadAssetsFromMemory(dummyPath, subReader, parentPath ?? fullName, bundleFile.m_Header.unityRevision);
} }
else else
{ {
@ -171,10 +170,10 @@ namespace AssetStudio
} }
catch (Exception e) catch (Exception e)
{ {
var str = $"Error while reading bundle file {fileName}"; var str = $"Error while reading bundle file {reader.FileName}";
if (parentPath != null) if (originalPath != null)
{ {
str += $" from {Path.GetFileName(parentPath)}"; str += $" from {Path.GetFileName(originalPath)}";
} }
Logger.Error(str, e); Logger.Error(str, e);
} }
@ -184,36 +183,36 @@ namespace AssetStudio
} }
} }
private void LoadWebFile(string fullName, EndianBinaryReader reader) private void LoadWebFile(FileReader reader)
{ {
var fileName = Path.GetFileName(fullName); Logger.Info("Loading " + reader.FileName);
Logger.Info("Loading " + fileName);
try try
{ {
var webFile = new WebFile(reader); var webFile = new WebFile(reader);
foreach (var file in webFile.fileList) foreach (var file in webFile.fileList)
{ {
var dummyPath = Path.Combine(Path.GetDirectoryName(fullName), file.fileName); var dummyPath = Path.Combine(Path.GetDirectoryName(reader.FullPath), file.fileName);
switch (CheckFileType(file.stream, out var fileReader)) var subReader = new FileReader(dummyPath, file.stream);
switch (subReader.FileType)
{ {
case FileType.AssetsFile: case FileType.AssetsFile:
LoadAssetsFromMemory(dummyPath, fileReader, fullName); LoadAssetsFromMemory(subReader, reader.FullPath);
break; break;
case FileType.BundleFile: case FileType.BundleFile:
LoadBundleFile(dummyPath, fileReader, fullName); LoadBundleFile(subReader, reader.FullPath);
break; break;
case FileType.WebFile: case FileType.WebFile:
LoadWebFile(dummyPath, fileReader); LoadWebFile(subReader);
break; break;
case FileType.ResourceFile: case FileType.ResourceFile:
resourceFileReaders[file.fileName] = fileReader; //TODO resourceFileReaders[file.fileName] = subReader; //TODO
break; break;
} }
} }
} }
catch (Exception e) catch (Exception e)
{ {
Logger.Error($"Error while reading web file {fileName}", e); Logger.Error($"Error while reading web file {reader.FileName}", e);
} }
finally finally
{ {

View File

@ -41,7 +41,7 @@ namespace AssetStudio
public StreamFile[] fileList; public StreamFile[] fileList;
public BundleFile(EndianBinaryReader reader, string path) public BundleFile(FileReader reader)
{ {
m_Header = new Header(); m_Header = new Header();
m_Header.signature = reader.ReadStringToNull(); m_Header.signature = reader.ReadStringToNull();
@ -59,19 +59,19 @@ namespace AssetStudio
goto case "UnityFS"; goto case "UnityFS";
} }
ReadHeaderAndBlocksInfo(reader); ReadHeaderAndBlocksInfo(reader);
using (var blocksStream = CreateBlocksStream(path)) using (var blocksStream = CreateBlocksStream(reader.FullPath))
{ {
ReadBlocksAndDirectory(reader, blocksStream); ReadBlocksAndDirectory(reader, blocksStream);
ReadFiles(blocksStream, path); ReadFiles(blocksStream, reader.FullPath);
} }
break; break;
case "UnityFS": case "UnityFS":
ReadHeader(reader); ReadHeader(reader);
ReadBlocksInfoAndDirectory(reader); ReadBlocksInfoAndDirectory(reader);
using (var blocksStream = CreateBlocksStream(path)) using (var blocksStream = CreateBlocksStream(reader.FullPath))
{ {
ReadBlocks(reader, blocksStream); ReadBlocks(reader, blocksStream);
ReadFiles(blocksStream, path); ReadFiles(blocksStream, reader.FullPath);
} }
break; break;
} }
@ -120,7 +120,7 @@ namespace AssetStudio
var uncompressedSizeSum = m_BlocksInfo.Sum(x => x.uncompressedSize); var uncompressedSizeSum = m_BlocksInfo.Sum(x => x.uncompressedSize);
if (uncompressedSizeSum >= int.MaxValue) if (uncompressedSizeSum >= int.MaxValue)
{ {
/*var memoryMappedFile = MemoryMappedFile.CreateNew(Path.GetFileName(path), uncompressedSizeSum); /*var memoryMappedFile = MemoryMappedFile.CreateNew(null, uncompressedSizeSum);
assetsDataStream = memoryMappedFile.CreateViewStream();*/ assetsDataStream = memoryMappedFile.CreateViewStream();*/
blocksStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose); blocksStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose);
} }
@ -175,7 +175,7 @@ namespace AssetStudio
file.fileName = Path.GetFileName(node.path); file.fileName = Path.GetFileName(node.path);
if (node.size >= int.MaxValue) if (node.size >= int.MaxValue)
{ {
/*var memoryMappedFile = MemoryMappedFile.CreateNew(file.fileName, entryinfo_size); /*var memoryMappedFile = MemoryMappedFile.CreateNew(null, entryinfo_size);
file.stream = memoryMappedFile.CreateViewStream();*/ file.stream = memoryMappedFile.CreateViewStream();*/
var extractPath = path + "_unpacked" + Path.DirectorySeparatorChar; var extractPath = path + "_unpacked" + Path.DirectorySeparatorChar;
Directory.CreateDirectory(extractPath); Directory.CreateDirectory(extractPath);

98
AssetStudio/FileReader.cs Normal file
View File

@ -0,0 +1,98 @@
using System.IO;
using System.Linq;
namespace AssetStudio
{
public class FileReader : EndianBinaryReader
{
public string FullPath;
public string FileName;
public FileType FileType;
public FileReader(string path) : this(path, File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { }
public FileReader(string path, Stream stream) : base(stream, EndianType.BigEndian)
{
FullPath = Path.GetFullPath(path);
FileName = Path.GetFileName(path);
CheckFileType();
}
private void CheckFileType()
{
var signature = this.ReadStringToNull(20);
Position = 0;
switch (signature)
{
case "UnityWeb":
case "UnityRaw":
case "UnityArchive":
case "UnityFS":
FileType = FileType.BundleFile;
break;
case "UnityWebData1.0":
FileType = FileType.WebFile;
break;
default:
var magic = ReadBytes(2);
Position = 0;
if (WebFile.gzipMagic.SequenceEqual(magic))
{
FileType = FileType.WebFile;
}
Position = 0x20;
magic = ReadBytes(6);
Position = 0;
if (WebFile.brotliMagic.SequenceEqual(magic))
{
FileType = FileType.WebFile;
}
if (IsSerializedFile())
{
FileType = FileType.AssetsFile;
}
else
{
FileType = FileType.ResourceFile;
}
break;
}
}
private bool IsSerializedFile()
{
var fileSize = BaseStream.Length;
if (fileSize < 20)
{
return false;
}
var m_MetadataSize = ReadUInt32();
long m_FileSize = ReadUInt32();
var m_Version = ReadUInt32();
long m_DataOffset = ReadUInt32();
var m_Endianess = ReadByte();
var m_Reserved = ReadBytes(3);
if (m_Version >= 22)
{
if (fileSize < 48)
{
Position = 0;
return false;
}
m_MetadataSize = ReadUInt32();
m_FileSize = ReadInt64();
m_DataOffset = ReadInt64();
}
Position = 0;
if (m_FileSize != fileSize)
{
return false;
}
if (m_DataOffset > fileSize)
{
return false;
}
return true;
}
}
}

16
AssetStudio/FileType.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace AssetStudio
{
public enum FileType
{
AssetsFile,
BundleFile,
WebFile,
ResourceFile
}
}

View File

@ -4,14 +4,6 @@ using System.Linq;
namespace AssetStudio namespace AssetStudio
{ {
public enum FileType
{
AssetsFile,
BundleFile,
WebFile,
ResourceFile
}
public static class ImportHelper public static class ImportHelper
{ {
public static void MergeSplitAssets(string path, bool allDirectories = false) public static void MergeSplitAssets(string path, bool allDirectories = false)
@ -56,57 +48,5 @@ namespace AssetStudio
} }
return selectFile.Distinct().ToArray(); return selectFile.Distinct().ToArray();
} }
public static FileType CheckFileType(Stream stream, out EndianBinaryReader reader)
{
reader = new EndianBinaryReader(stream);
return CheckFileType(reader);
}
public static FileType CheckFileType(string fileName, out EndianBinaryReader reader)
{
reader = new EndianBinaryReader(File.Open(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
return CheckFileType(reader);
}
private static FileType CheckFileType(EndianBinaryReader reader)
{
var signature = reader.ReadStringToNull(20);
reader.Position = 0;
switch (signature)
{
case "UnityWeb":
case "UnityRaw":
case "UnityArchive":
case "UnityFS":
return FileType.BundleFile;
case "UnityWebData1.0":
return FileType.WebFile;
default:
{
var magic = reader.ReadBytes(2);
reader.Position = 0;
if (WebFile.gzipMagic.SequenceEqual(magic))
{
return FileType.WebFile;
}
reader.Position = 0x20;
magic = reader.ReadBytes(6);
reader.Position = 0;
if (WebFile.brotliMagic.SequenceEqual(magic))
{
return FileType.WebFile;
}
if (SerializedFile.IsSerializedFile(reader))
{
return FileType.AssetsFile;
}
else
{
return FileType.ResourceFile;
}
}
}
}
} }
} }

View File

@ -9,7 +9,7 @@ namespace AssetStudio
public class SerializedFile public class SerializedFile
{ {
public AssetsManager assetsManager; public AssetsManager assetsManager;
public EndianBinaryReader reader; public FileReader reader;
public string fullName; public string fullName;
public string originalPath; public string originalPath;
public string fileName; public string fileName;
@ -31,12 +31,12 @@ namespace AssetStudio
public List<SerializedType> m_RefTypes; public List<SerializedType> m_RefTypes;
public string userInformation; public string userInformation;
public SerializedFile(AssetsManager assetsManager, string fullName, EndianBinaryReader reader) public SerializedFile(FileReader reader, AssetsManager assetsManager)
{ {
this.assetsManager = assetsManager; this.assetsManager = assetsManager;
this.reader = reader; this.reader = reader;
this.fullName = fullName; fullName = reader.FullPath;
fileName = Path.GetFileName(fullName); fileName = reader.FileName;
// ReadHeader // ReadHeader
header = new SerializedFileHeader(); header = new SerializedFileHeader();
@ -377,42 +377,5 @@ namespace AssetStudio
public bool IsVersionStripped => unityVersion == strippedVersion; public bool IsVersionStripped => unityVersion == strippedVersion;
private const string strippedVersion = "0.0.0"; private const string strippedVersion = "0.0.0";
public static bool IsSerializedFile(EndianBinaryReader reader)
{
var fileSize = reader.BaseStream.Length;
if (fileSize < 20)
{
return false;
}
var m_MetadataSize = reader.ReadUInt32();
long m_FileSize = reader.ReadUInt32();
var m_Version = reader.ReadUInt32();
long m_DataOffset = reader.ReadUInt32();
var m_Endianess = reader.ReadByte();
var m_Reserved = reader.ReadBytes(3);
if (m_Version >= 22)
{
if (fileSize < 48)
{
return false;
}
m_MetadataSize = reader.ReadUInt32();
m_FileSize = reader.ReadInt64();
m_DataOffset = reader.ReadInt64();
}
if (m_FileSize != fileSize)
{
reader.Position = 0;
return false;
}
if (m_DataOffset > fileSize)
{
reader.Position = 0;
return false;
}
reader.Position = 0;
return true;
}
} }
} }

View File

@ -72,37 +72,37 @@ namespace AssetStudioGUI
public static int ExtractFile(string fileName, string savePath) public static int ExtractFile(string fileName, string savePath)
{ {
int extractedCount = 0; int extractedCount = 0;
var type = ImportHelper.CheckFileType(fileName, out var reader); var reader = new FileReader(fileName);
if (type == FileType.BundleFile) if (reader.FileType == FileType.BundleFile)
extractedCount += ExtractBundleFile(fileName, reader, savePath); extractedCount += ExtractBundleFile(reader, savePath);
else if (type == FileType.WebFile) else if (reader.FileType == FileType.WebFile)
extractedCount += ExtractWebDataFile(fileName, reader, savePath); extractedCount += ExtractWebDataFile(reader, savePath);
else else
reader.Dispose(); reader.Dispose();
return extractedCount; return extractedCount;
} }
private static int ExtractBundleFile(string bundleFilePath, EndianBinaryReader reader, string savePath) private static int ExtractBundleFile(FileReader reader, string savePath)
{ {
StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFilePath)} ..."); StatusStripUpdate($"Decompressing {reader.FileName} ...");
var bundleFile = new BundleFile(reader, bundleFilePath); var bundleFile = new BundleFile(reader);
reader.Dispose(); reader.Dispose();
if (bundleFile.fileList.Length > 0) if (bundleFile.fileList.Length > 0)
{ {
var extractPath = Path.Combine(savePath, Path.GetFileName(bundleFilePath) + "_unpacked"); var extractPath = Path.Combine(savePath, reader.FileName + "_unpacked");
return ExtractStreamFile(extractPath, bundleFile.fileList); return ExtractStreamFile(extractPath, bundleFile.fileList);
} }
return 0; return 0;
} }
private static int ExtractWebDataFile(string webFilePath, EndianBinaryReader reader, string savePath) private static int ExtractWebDataFile(FileReader reader, string savePath)
{ {
StatusStripUpdate($"Decompressing {Path.GetFileName(webFilePath)} ..."); StatusStripUpdate($"Decompressing {reader.FileName} ...");
var webFile = new WebFile(reader); var webFile = new WebFile(reader);
reader.Dispose(); reader.Dispose();
if (webFile.fileList.Length > 0) if (webFile.fileList.Length > 0)
{ {
var extractPath = Path.Combine(savePath, Path.GetFileName(webFilePath) + "_unpacked"); var extractPath = Path.Combine(savePath, reader.FileName + "_unpacked");
return ExtractStreamFile(extractPath, webFile.fileList); return ExtractStreamFile(extractPath, webFile.fileList);
} }
return 0; return 0;