From c9e9bc840c46b0800e2fc9d03d0ee3567d711a02 Mon Sep 17 00:00:00 2001 From: VaDiM Date: Mon, 25 Mar 2024 05:00:58 +0300 Subject: [PATCH] Add parallel export support for some asset types --- AssetStudio/AssetsManager.cs | 16 +- AssetStudio/ResourceReader.cs | 20 +- AssetStudioCLI/CLILogger.cs | 41 +++- AssetStudioCLI/Exporter.cs | 194 ++------------- AssetStudioCLI/Options/CLIOptions.cs | 62 ++++- AssetStudioCLI/ParallelExporter.cs | 223 +++++++++++++++++ AssetStudioCLI/Program.cs | 2 +- AssetStudioCLI/Studio.cs | 97 ++++++-- AssetStudioGUI/AssetStudioGUIForm.cs | 2 +- AssetStudioGUI/AssetStudioGUIForm.resx | 6 +- AssetStudioGUI/ExportOptions.Designer.cs | 58 +++++ AssetStudioGUI/ExportOptions.cs | 13 + AssetStudioGUI/ExportOptions.resx | 6 + AssetStudioGUI/Exporter.cs | 114 +-------- AssetStudioGUI/GUILogger.cs | 89 +++++-- AssetStudioGUI/ParallelExport.cs | 226 ++++++++++++++++++ .../Properties/Settings.Designer.cs | 26 +- AssetStudioGUI/Properties/Settings.settings | 6 + AssetStudioGUI/Studio.cs | 125 ++++++++-- AssetStudioUtility/AudioClipConverter.cs | 54 +++-- .../CubismLive2DExtractor/Live2DExtractor.cs | 21 +- 21 files changed, 994 insertions(+), 407 deletions(-) create mode 100644 AssetStudioCLI/ParallelExporter.cs create mode 100644 AssetStudioGUI/ParallelExport.cs diff --git a/AssetStudio/AssetsManager.cs b/AssetStudio/AssetsManager.cs index 9668c0b..805b356 100644 --- a/AssetStudio/AssetsManager.cs +++ b/AssetStudio/AssetsManager.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.IO.Compression; @@ -17,7 +18,7 @@ namespace AssetStudio private HashSet filteredAssetTypesList = new HashSet(); internal Dictionary assetsFileIndexCache = new Dictionary(StringComparer.OrdinalIgnoreCase); - internal Dictionary resourceFileReaders = new Dictionary(StringComparer.OrdinalIgnoreCase); + internal ConcurrentDictionary resourceFileReaders = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); private List importFiles = new List(); private HashSet importFilesHash = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -248,13 +249,13 @@ namespace AssetStudio catch (NotSupportedException e) { Logger.Error(e.Message); - resourceFileReaders.Add(reader.FileName, reader); + resourceFileReaders.TryAdd(reader.FileName, reader); return false; } catch (Exception e) { Logger.Warning($"Failed to read assets file {reader.FullPath} from {Path.GetFileName(originalPath)}\r\n{e}"); - resourceFileReaders.Add(reader.FileName, reader); + resourceFileReaders.TryAdd(reader.FileName, reader); } } else @@ -279,9 +280,9 @@ namespace AssetStudio if (!LoadAssetsFromMemory(subReader, originalPath ?? reader.FullPath, bundleFile.m_Header.unityRevision)) return false; } - else if (!resourceFileReaders.ContainsKey(file.fileName)) + else { - resourceFileReaders.Add(file.fileName, subReader); + resourceFileReaders.TryAdd(file.fileName, subReader); } } return true; @@ -424,10 +425,7 @@ namespace AssetStudio if (entryReader.FileType == FileType.ResourceFile) { entryReader.Position = 0; - if (!resourceFileReaders.ContainsKey(entry.Name)) - { - resourceFileReaders.Add(entry.Name, entryReader); - } + resourceFileReaders.TryAdd(entry.Name, entryReader); } Progress.Report(++k, progressCount); } diff --git a/AssetStudio/ResourceReader.cs b/AssetStudio/ResourceReader.cs index 85403c1..9cf2e4c 100644 --- a/AssetStudio/ResourceReader.cs +++ b/AssetStudio/ResourceReader.cs @@ -63,8 +63,12 @@ namespace AssetStudio if (File.Exists(resourceFilePath)) { needSearch = false; + if (assetsFile.assetsManager.resourceFileReaders.TryGetValue(resourceFileName, out reader)) + { + return reader; + } reader = new BinaryReader(File.OpenRead(resourceFilePath)); - assetsFile.assetsManager.resourceFileReaders.Add(resourceFileName, reader); + assetsFile.assetsManager.resourceFileReaders.TryAdd(resourceFileName, reader); return reader; } throw new FileNotFoundException($"Can't find the resource file {resourceFileName}"); @@ -78,15 +82,21 @@ namespace AssetStudio public byte[] GetData() { var binaryReader = GetReader(); - binaryReader.BaseStream.Position = offset; - return binaryReader.ReadBytes((int)size); + lock (binaryReader) + { + binaryReader.BaseStream.Position = offset; + return binaryReader.ReadBytes((int) size); + } } public void GetData(byte[] buff) { var binaryReader = GetReader(); - binaryReader.BaseStream.Position = offset; - binaryReader.Read(buff, 0, (int)size); + lock (binaryReader) + { + binaryReader.BaseStream.Position = offset; + binaryReader.Read(buff, 0, (int) size); + } } public void WriteData(string path) diff --git a/AssetStudioCLI/CLILogger.cs b/AssetStudioCLI/CLILogger.cs index 3c199a0..c1863c5 100644 --- a/AssetStudioCLI/CLILogger.cs +++ b/AssetStudioCLI/CLILogger.cs @@ -1,9 +1,10 @@ using AssetStudio; using AssetStudioCLI.Options; using System; +using System.Collections.Concurrent; using System.IO; -using System.Linq; using System.Text.RegularExpressions; +using System.Threading.Tasks; namespace AssetStudioCLI { @@ -16,21 +17,29 @@ namespace AssetStudioCLI internal class CLILogger : ILogger { - private readonly LogOutputMode logOutput; - private readonly LoggerEvent logMinLevel; public string LogName; public string LogPath; + private static BlockingCollection logMessageCollection = new BlockingCollection(); + private readonly LogOutputMode logOutput; + private readonly LoggerEvent logMinLevel; + public CLILogger() { logOutput = CLIOptions.o_logOutput.Value; logMinLevel = CLIOptions.o_logLevel.Value; + var appAssembly = typeof(Program).Assembly.GetName(); + var arch = Environment.Is64BitProcess ? "x64" : "x32"; LogName = $"{appAssembly.Name}_{DateTime.Now:yyyy-MM-dd_HH-mm-ss}.log"; LogPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, LogName); - var arch = Environment.Is64BitProcess ? "x64" : "x32"; Console.OutputEncoding = System.Text.Encoding.UTF8; + if (logOutput != LogOutputMode.Console) + { + ConcurrentFileWriter(); + } + LogToFile(LoggerEvent.Verbose, $"---{appAssembly.Name} v{appAssembly.Version} [{arch}] | Logger launched---\n" + $"CMD Args: {string.Join(" ", CLIOptions.cliArgs)}"); } @@ -55,7 +64,7 @@ namespace AssetStudioCLI { var curTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"); message = message.TrimEnd(); - var multiLine = message.Contains('\n'); + var multiLine = message.Contains("\n"); string formattedMessage; if (consoleMode) @@ -64,7 +73,7 @@ namespace AssetStudioCLI formattedMessage = $"{colorLogLevel} {message}"; if (multiLine) { - formattedMessage = formattedMessage.Replace("\n", $"\n{colorLogLevel} "); + formattedMessage = formattedMessage.Replace("\n", $"\n{colorLogLevel} ") + $"\n{colorLogLevel}"; } } else @@ -74,7 +83,7 @@ namespace AssetStudioCLI formattedMessage = $"{curTime} | {logLevel} | {message}"; if (multiLine) { - formattedMessage = formattedMessage.Replace("\n", $"\n{curTime} | {logLevel} | "); + formattedMessage = formattedMessage.Replace("\n", $"\n{curTime} | {logLevel} | ") + $"\n{curTime} | {logLevel} |"; } } return formattedMessage; @@ -88,15 +97,27 @@ namespace AssetStudioCLI } } - public async void LogToFile(LoggerEvent logMsgLevel, string message) + public void LogToFile(LoggerEvent logMsgLevel, string message) { if (logOutput != LogOutputMode.Console) + { + logMessageCollection.Add(FormatMessage(logMsgLevel, message)); + } + } + + private void ConcurrentFileWriter() + { + Task.Run(() => { using (var sw = new StreamWriter(LogPath, append: true, System.Text.Encoding.UTF8)) { - await sw.WriteLineAsync(FormatMessage(logMsgLevel, message)); + sw.AutoFlush = true; + foreach (var msg in logMessageCollection.GetConsumingEnumerable()) + { + sw.WriteLine(msg); + } } - } + }); } public void Log(LoggerEvent logMsgLevel, string message, bool ignoreLevel) diff --git a/AssetStudioCLI/Exporter.cs b/AssetStudioCLI/Exporter.cs index e273c10..5751daf 100644 --- a/AssetStudioCLI/Exporter.cs +++ b/AssetStudioCLI/Exporter.cs @@ -10,141 +10,6 @@ namespace AssetStudioCLI { internal static class Exporter { - public static bool ExportTexture2D(AssetItem item, string exportPath) - { - var m_Texture2D = (Texture2D)item.Asset; - if (CLIOptions.convertTexture) - { - var type = CLIOptions.o_imageFormat.Value; - if (!TryExportFile(exportPath, item, "." + type.ToString().ToLower(), out var exportFullPath)) - return false; - - if (CLIOptions.o_logLevel.Value <= LoggerEvent.Debug) - { - var sb = new StringBuilder(); - sb.AppendLine($"Converting \"{m_Texture2D.m_Name}\" to {type}.."); - sb.AppendLine($"Width: {m_Texture2D.m_Width}"); - sb.AppendLine($"Height: {m_Texture2D.m_Height}"); - sb.AppendLine($"Format: {m_Texture2D.m_TextureFormat}"); - switch (m_Texture2D.m_TextureSettings.m_FilterMode) - { - case 0: sb.AppendLine("Filter Mode: Point "); break; - case 1: sb.AppendLine("Filter Mode: Bilinear "); break; - case 2: sb.AppendLine("Filter Mode: Trilinear "); break; - } - sb.AppendLine($"Anisotropic level: {m_Texture2D.m_TextureSettings.m_Aniso}"); - sb.AppendLine($"Mip map bias: {m_Texture2D.m_TextureSettings.m_MipBias}"); - switch (m_Texture2D.m_TextureSettings.m_WrapMode) - { - case 0: sb.AppendLine($"Wrap mode: Repeat"); break; - case 1: sb.AppendLine($"Wrap mode: Clamp"); break; - } - Logger.Debug(sb.ToString()); - } - - var image = m_Texture2D.ConvertToImage(flip: true); - if (image == null) - { - Logger.Error($"Export error. Failed to convert texture \"{m_Texture2D.m_Name}\" into image"); - return false; - } - using (image) - { - using (var file = File.OpenWrite(exportFullPath)) - { - image.WriteToStream(file, type); - } - Logger.Debug($"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""); - return true; - } - } - else - { - if (!TryExportFile(exportPath, item, ".tex", out var exportFullPath)) - return false; - File.WriteAllBytes(exportFullPath, m_Texture2D.image_data.GetData()); - Logger.Debug($"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""); - return true; - } - } - - public static bool ExportTexture2DArray(AssetItem item, string exportPath) - { - var m_Texture2DArray = (Texture2DArray)item.Asset; - var count = 0; - foreach (var texture in m_Texture2DArray.TextureList) - { - var fakeItem = new AssetItem(texture) - { - Text = texture.m_Name, - Container = item.Container, - }; - if (ExportTexture2D(fakeItem, exportPath)) - { - count++; - } - } - Logger.Debug($"{item.TypeString} \"{item.Text}\" exported to \"{exportPath}\""); - return count > 0; - } - - public static bool ExportAudioClip(AssetItem item, string exportPath) - { - string exportFullPath; - var m_AudioClip = (AudioClip)item.Asset; - var m_AudioData = m_AudioClip.m_AudioData.GetData(); - if (m_AudioData == null || m_AudioData.Length == 0) - { - Logger.Error($"Export error. \"{item.Text}\": AudioData was not found"); - return false; - } - var converter = new AudioClipConverter(m_AudioClip); - if (CLIOptions.o_audioFormat.Value != AudioFormat.None && converter.IsSupport) - { - if (!TryExportFile(exportPath, item, ".wav", out exportFullPath)) - return false; - - if (CLIOptions.o_logLevel.Value <= LoggerEvent.Debug) - { - var sb = new StringBuilder(); - sb.AppendLine($"Converting \"{m_AudioClip.m_Name}\" to wav.."); - sb.AppendLine(m_AudioClip.version[0] < 5 ? $"AudioClip type: {m_AudioClip.m_Type}" : $"AudioClip compression format: {m_AudioClip.m_CompressionFormat}"); - sb.AppendLine($"AudioClip channel count: {m_AudioClip.m_Channels}"); - sb.AppendLine($"AudioClip sample rate: {m_AudioClip.m_Frequency}"); - sb.AppendLine($"AudioClip bit depth: {m_AudioClip.m_BitsPerSample}"); - Logger.Debug(sb.ToString()); - } - - var buffer = converter.ConvertToWav(m_AudioData); - if (buffer == null) - { - Logger.Error($"Export error. \"{item.Text}\": Failed to convert fmod audio to Wav"); - return false; - } - File.WriteAllBytes(exportFullPath, buffer); - } - else - { - if (!TryExportFile(exportPath, item, converter.GetExtensionName(), out exportFullPath)) - return false; - - if (CLIOptions.o_logLevel.Value <= LoggerEvent.Debug) - { - var sb = new StringBuilder(); - sb.AppendLine($"Exporting non-fmod {item.TypeString} \"{m_AudioClip.m_Name}\".."); - sb.AppendLine(m_AudioClip.version[0] < 5 ? $"AudioClip type: {m_AudioClip.m_Type}" : $"AudioClip compression format: {m_AudioClip.m_CompressionFormat}"); - sb.AppendLine($"AudioClip channel count: {m_AudioClip.m_Channels}"); - sb.AppendLine($"AudioClip sample rate: {m_AudioClip.m_Frequency}"); - sb.AppendLine($"AudioClip bit depth: {m_AudioClip.m_BitsPerSample}"); - Logger.Debug(sb.ToString()); - } - File.WriteAllBytes(exportFullPath, m_AudioData); - } - - Logger.Debug($"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""); - return true; - } - public static bool ExportVideoClip(AssetItem item, string exportPath) { var m_VideoClip = (VideoClip)item.Asset; @@ -264,38 +129,6 @@ namespace AssetStudioCLI return false; } - public static bool ExportSprite(AssetItem item, string exportPath) - { - var type = CLIOptions.o_imageFormat.Value; - var alphaMask = SpriteMaskMode.On; - if (!TryExportFile(exportPath, item, "." + type.ToString().ToLower(), out var exportFullPath)) - return false; - var image = ((Sprite)item.Asset).GetImage(alphaMask); - if (image != null) - { - using (image) - { - using (var file = File.OpenWrite(exportFullPath)) - { - image.WriteToStream(file, type); - } - Logger.Debug($"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""); - return true; - } - } - return false; - } - - public static bool ExportRawFile(AssetItem item, string exportPath) - { - if (!TryExportFile(exportPath, item, ".dat", out var exportFullPath)) - return false; - File.WriteAllBytes(exportFullPath, item.Asset.GetRawData()); - - Logger.Debug($"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""); - return true; - } - public static void ExportGameObject(GameObject gameObject, string exportPath, List animationList = null) { var convert = animationList != null @@ -323,10 +156,19 @@ namespace AssetStudioCLI exportAllNodes, exportSkins, exportAnimations, exportBlendShape, castToBone, boneSize, exportAllUvsAsDiffuseMaps, scaleFactor, fbxVersion, fbxFormat == 1); } + public static bool ExportRawFile(AssetItem item, string exportPath) + { + if (!TryExportFile(exportPath, item, ".dat", out var exportFullPath, mode: "ExportRaw")) + return false; + File.WriteAllBytes(exportFullPath, item.Asset.GetRawData()); + + Logger.Debug($"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""); + return true; + } public static bool ExportDumpFile(AssetItem item, string exportPath) { - if (!TryExportFile(exportPath, item, ".txt", out var exportFullPath)) + if (!TryExportFile(exportPath, item, ".txt", out var exportFullPath, mode: "Dump")) return false; var str = item.Asset.Dump(); if (str == null && item.Asset is MonoBehaviour m_MonoBehaviour) @@ -347,7 +189,7 @@ namespace AssetStudioCLI return false; } - private static bool TryExportFile(string dir, AssetItem item, string extension, out string fullPath) + private static bool TryExportFile(string dir, AssetItem item, string extension, out string fullPath, string mode = "Export") { var fileName = FixFileName(item.Text); var filenameFormat = CLIOptions.o_filenameFormat.Value; @@ -375,7 +217,7 @@ namespace AssetStudioCLI return true; } } - Logger.Error($"Export error. File \"{fullPath.Color(ColorConsole.BrightRed)}\" already exist"); + Logger.Error($"{mode} error. File \"{fullPath.Color(ColorConsole.BrightRed)}\" already exist"); return false; } @@ -481,11 +323,10 @@ namespace AssetStudioCLI switch (item.Type) { case ClassIDType.Texture2D: - return ExportTexture2D(item, exportPath); case ClassIDType.Texture2DArray: - return ExportTexture2DArray(item, exportPath); + case ClassIDType.Sprite: case ClassIDType.AudioClip: - return ExportAudioClip(item, exportPath); + throw new System.NotImplementedException(); case ClassIDType.VideoClip: return ExportVideoClip(item, exportPath); case ClassIDType.MovieTexture: @@ -498,8 +339,6 @@ namespace AssetStudioCLI return ExportMonoBehaviour(item, exportPath); case ClassIDType.Font: return ExportFont(item, exportPath); - case ClassIDType.Sprite: - return ExportSprite(item, exportPath); case ClassIDType.Mesh: return ExportMesh(item, exportPath); default: @@ -509,8 +348,9 @@ namespace AssetStudioCLI public static string FixFileName(string str) { - if (str.Length >= 260) return Path.GetRandomFileName(); - return Path.GetInvalidFileNameChars().Aggregate(str, (current, c) => current.Replace(c, '_')); + return str.Length >= 260 + ? Path.GetRandomFileName() + : Path.GetInvalidFileNameChars().Aggregate(str, (current, c) => current.Replace(c, '_')); } } } diff --git a/AssetStudioCLI/Options/CLIOptions.cs b/AssetStudioCLI/Options/CLIOptions.cs index 0669c5d..12f24b5 100644 --- a/AssetStudioCLI/Options/CLIOptions.cs +++ b/AssetStudioCLI/Options/CLIOptions.cs @@ -24,7 +24,7 @@ namespace AssetStudioCLI.Options ExportRaw, Dump, Info, - ExportLive2D, + Live2D, SplitObjects, } @@ -112,6 +112,7 @@ namespace AssetStudioCLI.Options public static Option> o_filterByText; //advanced public static Option o_customCompressionType; + public static Option o_maxParallelExportTasks; public static Option o_exportAssetList; public static Option o_assemblyPath; public static Option o_unityVersion; @@ -190,7 +191,7 @@ namespace AssetStudioCLI.Options "ExportRaw - Exports raw data\n" + "Dump - Makes asset dumps\n" + "Info - Loads file(s), shows the number of available for export assets and exits\n" + - "Live2D - Exports Live2D Cubism 3 models\n" + + "Live2D - Exports Live2D Cubism models\n" + "SplitObjects - Exports split objects (fbx)\n", optionExample: "Example: \"-m info\"\n", optionHelpGroup: HelpGroups.General @@ -329,7 +330,7 @@ namespace AssetStudioCLI.Options optionDefaultValue: 1f, optionName: "--fbx-scale-factor ", optionDescription: "Specify the FBX Scale Factor\n" + - "\n", optionExample: "Example: \"--fbx-scale-factor 50\"\n", optionHelpGroup: HelpGroups.FBX ); @@ -338,7 +339,7 @@ namespace AssetStudioCLI.Options optionDefaultValue: 10, optionName: "--fbx-bone-size ", optionDescription: "Specify the FBX Bone Size\n" + - "\n", optionExample: "Example: \"--fbx-bone-size 10\"", optionHelpGroup: HelpGroups.FBX ); @@ -397,6 +398,17 @@ namespace AssetStudioCLI.Options optionHelpGroup: HelpGroups.Advanced ); + o_maxParallelExportTasks = new GroupedOption + ( + optionDefaultValue: Environment.ProcessorCount - 1, + optionName: "--max-export-tasks ", + optionDescription: "Specify the number of parallel tasks for asset export\n" + + "\n" + + "Max - Number of cores in your CPU\n", + optionExample: "Example: \"--max-export-tasks 8\"\n", + optionHelpGroup: HelpGroups.Advanced + ); + o_exportAssetList = new GroupedOption ( optionDefaultValue: ExportListType.None, @@ -528,7 +540,7 @@ namespace AssetStudioCLI.Options break; case "l2d": case "live2d": - o_workMode.Value = WorkMode.ExportLive2D; + o_workMode.Value = WorkMode.Live2D; o_exportAssetTypes.Value = new List() { ClassIDType.AnimationClip, @@ -564,7 +576,7 @@ namespace AssetStudioCLI.Options switch(flag) { case "--l2d-force-bezier": - if (o_workMode.Value != WorkMode.ExportLive2D) + if (o_workMode.Value != WorkMode.Live2D) { Console.WriteLine($"{"Error".Color(brightRed)} during parsing [{flag.Color(brightYellow)}] flag. This flag is not suitable for the current working mode [{o_workMode.Value}].\n"); ShowOptionDescription(o_workMode); @@ -611,7 +623,7 @@ namespace AssetStudioCLI.Options { case "-t": case "--asset-type": - if (o_workMode.Value == WorkMode.ExportLive2D || o_workMode.Value == WorkMode.SplitObjects) + if (o_workMode.Value == WorkMode.Live2D || o_workMode.Value == WorkMode.SplitObjects) { i++; continue; @@ -820,7 +832,7 @@ namespace AssetStudioCLI.Options } break; case "--l2d-motion-mode": - if (o_workMode.Value != WorkMode.ExportLive2D) + if (o_workMode.Value != WorkMode.Live2D) { Console.WriteLine($"{"Error".Color(brightRed)} during parsing [{option.Color(brightYellow)}] option. This option is not suitable for the current working mode [{o_workMode.Value}].\n"); ShowOptionDescription(o_workMode); @@ -846,7 +858,8 @@ namespace AssetStudioCLI.Options } break; case "--fbx-scale-factor": - var isFloat = float.TryParse(value, out float floatValue); + { + var isFloat = float.TryParse(value, out var floatValue); if (isFloat && floatValue >= 0 && floatValue <= 100) { o_fbxScaleFactor.Value = floatValue; @@ -858,8 +871,10 @@ namespace AssetStudioCLI.Options return; } break; + } case "--fbx-bone-size": - var isInt = int.TryParse(value, out int intValue); + { + var isInt = int.TryParse(value, out var intValue); if (isInt && intValue >= 0 && intValue <= 100) { o_fbxBoneSize.Value = intValue; @@ -871,6 +886,7 @@ namespace AssetStudioCLI.Options return; } break; + } case "--custom-compression": switch (value.ToLower()) { @@ -887,6 +903,29 @@ namespace AssetStudioCLI.Options return; } break; + case "--max-export-tasks": + { + var processorCount = Environment.ProcessorCount; + if (value.ToLower() == "max") + { + o_maxParallelExportTasks.Value = processorCount - 1; + } + else + { + var isInt = int.TryParse(value, out var intValue); + if (isInt && intValue >= 0 && intValue <= processorCount) + { + o_maxParallelExportTasks.Value = Math.Min(intValue, processorCount - 1); + } + else + { + Console.WriteLine($"{"Error".Color(brightRed)} during parsing [{option.Color(brightYellow)}] option. Unsupported number of parallel tasks: [{value.Color(brightRed)}].\n"); + ShowOptionDescription(o_maxParallelExportTasks); + return; + } + } + break; + } case "--export-asset-list": switch (value.ToLower()) { @@ -1125,6 +1164,7 @@ namespace AssetStudioCLI.Options sb.AppendLine($"# Unity Version: \"{o_unityVersion}\""); if (o_workMode.Value == WorkMode.Export) { + sb.AppendLine($"# Max Parallel Export Tasks: {o_maxParallelExportTasks}"); sb.AppendLine($"# Restore TextAsset Extension: {!f_notRestoreExtensionName.Value}"); } break; @@ -1137,7 +1177,7 @@ namespace AssetStudioCLI.Options sb.AppendLine(ShowCurrentFilter()); sb.AppendLine($"# Unity Version: \"{o_unityVersion}\""); break; - case WorkMode.ExportLive2D: + case WorkMode.Live2D: case WorkMode.SplitObjects: sb.AppendLine($"# Output Path: \"{o_outputFolder}\""); sb.AppendLine($"# Log Level: {o_logLevel}"); diff --git a/AssetStudioCLI/ParallelExporter.cs b/AssetStudioCLI/ParallelExporter.cs new file mode 100644 index 0000000..a5c49ae --- /dev/null +++ b/AssetStudioCLI/ParallelExporter.cs @@ -0,0 +1,223 @@ +using AssetStudio; +using AssetStudioCLI.Options; +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Text; + +namespace AssetStudioCLI +{ + internal static class ParallelExporter + { + private static ConcurrentDictionary savePathHash = new ConcurrentDictionary(); + + public static bool ExportTexture2D(AssetItem item, string exportPath, out string debugLog) + { + debugLog = ""; + var m_Texture2D = (Texture2D)item.Asset; + if (CLIOptions.convertTexture) + { + var type = CLIOptions.o_imageFormat.Value; + if (!TryExportFile(exportPath, item, "." + type.ToString().ToLower(), out var exportFullPath)) + return false; + + if (CLIOptions.o_logLevel.Value <= LoggerEvent.Debug) + { + var sb = new StringBuilder(); + sb.AppendLine($"Converting {item.TypeString} \"{m_Texture2D.m_Name}\" to {type}.."); + sb.AppendLine($"Width: {m_Texture2D.m_Width}"); + sb.AppendLine($"Height: {m_Texture2D.m_Height}"); + sb.AppendLine($"Format: {m_Texture2D.m_TextureFormat}"); + switch (m_Texture2D.m_TextureSettings.m_FilterMode) + { + case 0: sb.AppendLine("Filter Mode: Point "); break; + case 1: sb.AppendLine("Filter Mode: Bilinear "); break; + case 2: sb.AppendLine("Filter Mode: Trilinear "); break; + } + sb.AppendLine($"Anisotropic level: {m_Texture2D.m_TextureSettings.m_Aniso}"); + sb.AppendLine($"Mip map bias: {m_Texture2D.m_TextureSettings.m_MipBias}"); + switch (m_Texture2D.m_TextureSettings.m_WrapMode) + { + case 0: sb.AppendLine($"Wrap mode: Repeat"); break; + case 1: sb.AppendLine($"Wrap mode: Clamp"); break; + } + debugLog += sb.ToString(); + } + + var image = m_Texture2D.ConvertToImage(flip: true); + if (image == null) + { + Logger.Error($"{debugLog}Export error. Failed to convert texture \"{m_Texture2D.m_Name}\" into image"); + return false; + } + using (image) + { + using (var file = File.OpenWrite(exportFullPath)) + { + image.WriteToStream(file, type); + } + debugLog += $"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""; + return true; + } + } + else + { + if (!TryExportFile(exportPath, item, ".tex", out var exportFullPath)) + return false; + File.WriteAllBytes(exportFullPath, m_Texture2D.image_data.GetData()); + debugLog += $"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""; + return true; + } + } + + public static bool ExportSprite(AssetItem item, string exportPath, out string debugLog) + { + debugLog = ""; + var type = CLIOptions.o_imageFormat.Value; + var alphaMask = SpriteMaskMode.On; + if (!TryExportFile(exportPath, item, "." + type.ToString().ToLower(), out var exportFullPath)) + return false; + var image = ((Sprite)item.Asset).GetImage(alphaMask); + if (image != null) + { + using (image) + { + using (var file = File.OpenWrite(exportFullPath)) + { + image.WriteToStream(file, type); + } + debugLog += $"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""; + return true; + } + } + return false; + } + + public static bool ExportAudioClip(AssetItem item, string exportPath, out string debugLog) + { + debugLog = ""; + string exportFullPath; + var m_AudioClip = (AudioClip)item.Asset; + var m_AudioData = BigArrayPool.Shared.Rent(m_AudioClip.m_AudioData.Size); + try + { + m_AudioClip.m_AudioData.GetData(m_AudioData); + if (m_AudioData == null || m_AudioData.Length == 0) + { + Logger.Error($"Export error. \"{item.Text}\": AudioData was not found"); + return false; + } + var converter = new AudioClipConverter(m_AudioClip); + if (CLIOptions.o_audioFormat.Value != AudioFormat.None && converter.IsSupport) + { + if (!TryExportFile(exportPath, item, ".wav", out exportFullPath)) + return false; + + if (CLIOptions.o_logLevel.Value <= LoggerEvent.Debug) + { + var sb = new StringBuilder(); + sb.AppendLine($"Converting {item.TypeString} \"{m_AudioClip.m_Name}\" to wav.."); + sb.AppendLine(m_AudioClip.version[0] < 5 ? $"AudioClip type: {m_AudioClip.m_Type}" : $"AudioClip compression format: {m_AudioClip.m_CompressionFormat}"); + sb.AppendLine($"AudioClip channel count: {m_AudioClip.m_Channels}"); + sb.AppendLine($"AudioClip sample rate: {m_AudioClip.m_Frequency}"); + sb.AppendLine($"AudioClip bit depth: {m_AudioClip.m_BitsPerSample}"); + debugLog += sb.ToString(); + } + + var buffer = converter.ConvertToWav(m_AudioData, out var debugLogConverter); + debugLog += debugLogConverter; + if (buffer == null) + { + Logger.Error($"{debugLog}Export error. \"{item.Text}\": Failed to convert fmod audio to Wav"); + return false; + } + File.WriteAllBytes(exportFullPath, buffer); + } + else + { + if (!TryExportFile(exportPath, item, converter.GetExtensionName(), out exportFullPath)) + return false; + + if (CLIOptions.o_logLevel.Value <= LoggerEvent.Debug) + { + var sb = new StringBuilder(); + sb.AppendLine($"Exporting non-fmod {item.TypeString} \"{m_AudioClip.m_Name}\".."); + sb.AppendLine(m_AudioClip.version[0] < 5 ? $"AudioClip type: {m_AudioClip.m_Type}" : $"AudioClip compression format: {m_AudioClip.m_CompressionFormat}"); + sb.AppendLine($"AudioClip channel count: {m_AudioClip.m_Channels}"); + sb.AppendLine($"AudioClip sample rate: {m_AudioClip.m_Frequency}"); + sb.AppendLine($"AudioClip bit depth: {m_AudioClip.m_BitsPerSample}"); + debugLog += sb.ToString(); + } + File.WriteAllBytes(exportFullPath, m_AudioData); + } + debugLog += $"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""; + return true; + } + finally + { + BigArrayPool.Shared.Return(m_AudioData, clearArray: true); + } + } + + private static bool TryExportFile(string dir, AssetItem item, string extension, out string fullPath) + { + var fileName = FixFileName(item.Text); + var filenameFormat = CLIOptions.o_filenameFormat.Value; + switch (filenameFormat) + { + case FilenameFormat.AssetName_PathID: + fileName = $"{fileName} @{item.m_PathID}"; + break; + case FilenameFormat.PathID: + fileName = item.m_PathID.ToString(); + break; + } + fullPath = Path.Combine(dir, fileName + extension); + if (savePathHash.TryAdd(fullPath.ToLower(), true) && !File.Exists(fullPath)) + { + Directory.CreateDirectory(dir); + return true; + } + if (filenameFormat == FilenameFormat.AssetName) + { + fullPath = Path.Combine(dir, fileName + item.UniqueID + extension); + if (!File.Exists(fullPath)) + { + Directory.CreateDirectory(dir); + return true; + } + } + Logger.Error($"Export error. File \"{fullPath.Color(ColorConsole.BrightRed)}\" already exist"); + return false; + } + + public static bool ParallelExportConvertFile(AssetItem item, string exportPath, out string debugLog) + { + switch (item.Type) + { + case ClassIDType.Texture2D: + case ClassIDType.Texture2DArrayImage: + return ExportTexture2D(item, exportPath, out debugLog); + case ClassIDType.Sprite: + return ExportSprite(item, exportPath, out debugLog); + case ClassIDType.AudioClip: + return ExportAudioClip(item, exportPath, out debugLog); + default: + throw new NotImplementedException(); + } + } + + private static string FixFileName(string str) + { + return str.Length >= 260 + ? Path.GetRandomFileName() + : Path.GetInvalidFileNameChars().Aggregate(str, (current, c) => current.Replace(c, '_')); + } + + public static void ClearHash() + { + savePathHash.Clear(); + } + } +} diff --git a/AssetStudioCLI/Program.cs b/AssetStudioCLI/Program.cs index 40de9d5..4c61fd4 100644 --- a/AssetStudioCLI/Program.cs +++ b/AssetStudioCLI/Program.cs @@ -48,7 +48,7 @@ namespace AssetStudioCLI case WorkMode.Info: Studio.ShowExportableAssetsInfo(); break; - case WorkMode.ExportLive2D: + case WorkMode.Live2D: Studio.ExportLive2D(); break; case WorkMode.SplitObjects: diff --git a/AssetStudioCLI/Studio.cs b/AssetStudioCLI/Studio.cs index 66f84f0..d697e29 100644 --- a/AssetStudioCLI/Studio.cs +++ b/AssetStudioCLI/Studio.cs @@ -2,9 +2,12 @@ using AssetStudioCLI.Options; using CubismLive2DExtractor; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using System.Xml.Linq; using static AssetStudioCLI.Exporter; using Ansi = AssetStudio.ColorConsole; @@ -191,7 +194,7 @@ namespace AssetStudioCLI parsedAssetsList.AddRange(fileAssetsList); fileAssetsList.Clear(); tex2dArrayAssetList.Clear(); - if (CLIOptions.o_workMode.Value != WorkMode.ExportLive2D) + if (CLIOptions.o_workMode.Value != WorkMode.Live2D) { containers.Clear(); } @@ -364,7 +367,7 @@ namespace AssetStudioCLI { switch (CLIOptions.o_workMode.Value) { - case WorkMode.ExportLive2D: + case WorkMode.Live2D: case WorkMode.SplitObjects: break; default: @@ -434,7 +437,10 @@ namespace AssetStudioCLI var exportedCount = 0; var groupOption = CLIOptions.o_groupAssetsBy.Value; - foreach (var asset in parsedAssetsList) + var parallelExportCount = CLIOptions.o_maxParallelExportTasks.Value; + var toExportAssetDict = new ConcurrentDictionary(); + var toParallelExportAssetDict = new ConcurrentDictionary(); + Parallel.ForEach(parsedAssetsList, asset => { string exportPath; switch (groupOption) @@ -481,32 +487,61 @@ namespace AssetStudioCLI exportPath = savePath; break; } - exportPath += Path.DirectorySeparatorChar; + + if (CLIOptions.o_workMode.Value == WorkMode.Export) + { + switch (asset.Type) + { + case ClassIDType.Texture2D: + case ClassIDType.Sprite: + case ClassIDType.AudioClip: + toParallelExportAssetDict.TryAdd(asset, exportPath); + break; + case ClassIDType.Texture2DArray: + var m_Texture2DArray = (Texture2DArray)asset.Asset; + toExportCount += m_Texture2DArray.TextureList.Count - 1; + foreach (var texture in m_Texture2DArray.TextureList) + { + var fakeItem = new AssetItem(texture) + { + Text = texture.m_Name, + Container = asset.Container, + }; + toParallelExportAssetDict.TryAdd(fakeItem, exportPath); + } + break; + default: + toExportAssetDict.TryAdd(asset, exportPath); + break; + } + } + else + { + toExportAssetDict.TryAdd(asset, exportPath); + } + }); + + foreach (var toExportAsset in toExportAssetDict) + { + var asset = toExportAsset.Key; + var exportPath = toExportAsset.Value; + var isExported = false; try { switch (CLIOptions.o_workMode.Value) { case WorkMode.ExportRaw: Logger.Debug($"{CLIOptions.o_workMode}: {asset.Type} : {asset.Container} : {asset.Text}"); - if (ExportRawFile(asset, exportPath)) - { - exportedCount++; - } + isExported = ExportRawFile(asset, exportPath); break; case WorkMode.Dump: Logger.Debug($"{CLIOptions.o_workMode}: {asset.Type} : {asset.Container} : {asset.Text}"); - if (ExportDumpFile(asset, exportPath)) - { - exportedCount++; - } + isExported = ExportDumpFile(asset, exportPath); break; case WorkMode.Export: Logger.Debug($"{CLIOptions.o_workMode}: {asset.Type} : {asset.Container} : {asset.Text}"); - if (ExportConvertFile(asset, exportPath)) - { - exportedCount++; - } + isExported = ExportConvertFile(asset, exportPath); break; } } @@ -514,8 +549,33 @@ namespace AssetStudioCLI { Logger.Error($"{asset.SourceFile.originalPath}: [{$"{asset.Type}: {asset.Text}".Color(Ansi.BrightRed)}] : Export error\n{ex}"); } + + if (isExported) + { + exportedCount++; + } Console.Write($"Exported [{exportedCount}/{toExportCount}]\r"); } + + Parallel.ForEach(toParallelExportAssetDict, new ParallelOptions { MaxDegreeOfParallelism = parallelExportCount }, toExportAsset => + { + var asset = toExportAsset.Key; + var exportPath = toExportAsset.Value; + try + { + if (ParallelExporter.ParallelExportConvertFile(asset, exportPath, out var debugLog)) + { + Interlocked.Increment(ref exportedCount); + Logger.Debug(debugLog); + Console.Write($"Exported [{exportedCount}/{toExportCount}]\r"); + } + } + catch (Exception ex) + { + Logger.Error($"{asset.SourceFile.originalPath}: [{$"{asset.Type}: {asset.Text}".Color(Ansi.BrightRed)}] : Export error\n{ex}"); + } + }); + ParallelExporter.ClearHash(); Console.WriteLine(""); if (exportedCount == 0) @@ -708,7 +768,7 @@ namespace AssetStudioCLI } basePathSet.Clear(); - var lookup = containers.ToLookup( + var lookup = containers.AsParallel().ToLookup( x => mocPathList.Find(b => x.Value.Contains(b) && x.Value.Split('/').Any(y => y == b.Substring(b.LastIndexOf("/") + 1))), x => x.Key ); @@ -720,6 +780,7 @@ namespace AssetStudioCLI var totalModelCount = lookup.LongCount(x => x.Key != null); Logger.Info($"Found {totalModelCount} model(s)."); + var parallelTaskCount = CLIOptions.o_maxParallelExportTasks.Value; var modelCounter = 0; foreach (var assets in lookup) { @@ -740,7 +801,7 @@ namespace AssetStudioCLI var destPath = Path.Combine(baseDestPath, container) + Path.DirectorySeparatorChar; var modelExtractor = new Live2DExtractor(assets); - modelExtractor.ExtractCubismModel(destPath, modelName, motionMode, assemblyLoader, forceBezier); + modelExtractor.ExtractCubismModel(destPath, modelName, motionMode, assemblyLoader, forceBezier, parallelTaskCount); modelCounter++; } catch (Exception ex) diff --git a/AssetStudioGUI/AssetStudioGUIForm.cs b/AssetStudioGUI/AssetStudioGUIForm.cs index 98ca27f..1e03886 100644 --- a/AssetStudioGUI/AssetStudioGUIForm.cs +++ b/AssetStudioGUI/AssetStudioGUIForm.cs @@ -1966,7 +1966,7 @@ namespace AssetStudioGUI private void toolStripMenuItem15_Click(object sender, EventArgs e) { - logger.ShowDebugMessage = toolStripMenuItem15.Checked; + GUILogger.ShowDebugMessage = toolStripMenuItem15.Checked; } private void sceneTreeView_NodeMouseClick(object sender, TreeNodeMouseClickEventArgs e) diff --git a/AssetStudioGUI/AssetStudioGUIForm.resx b/AssetStudioGUI/AssetStudioGUIForm.resx index dcce5cf..095950b 100644 --- a/AssetStudioGUI/AssetStudioGUIForm.resx +++ b/AssetStudioGUI/AssetStudioGUIForm.resx @@ -120,9 +120,6 @@ 312, 17 - - 432, 17 - abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWYZ 1234567890.:,;'\"(!?)+-*/= @@ -141,6 +138,9 @@ The quick brown fox jumps over the lazy dog. 1234567890 The quick brown fox jumps over the lazy dog. 1234567890 + + 432, 17 + 775, 21 diff --git a/AssetStudioGUI/ExportOptions.Designer.cs b/AssetStudioGUI/ExportOptions.Designer.cs index ca13d11..cdd6a89 100644 --- a/AssetStudioGUI/ExportOptions.Designer.cs +++ b/AssetStudioGUI/ExportOptions.Designer.cs @@ -72,6 +72,9 @@ this.exportAllNodes = new System.Windows.Forms.CheckBox(); this.eulerFilter = new System.Windows.Forms.CheckBox(); this.optionTooltip = new System.Windows.Forms.ToolTip(this.components); + this.parallelExportUpDown = new System.Windows.Forms.NumericUpDown(); + this.parallelExportCheckBox = new System.Windows.Forms.CheckBox(); + this.parallelExportMaxLabel = new System.Windows.Forms.Label(); this.groupBox1.SuspendLayout(); this.panel1.SuspendLayout(); this.l2dGroupBox.SuspendLayout(); @@ -80,6 +83,7 @@ ((System.ComponentModel.ISupportInitialize)(this.scaleFactor)).BeginInit(); ((System.ComponentModel.ISupportInitialize)(this.boneSize)).BeginInit(); ((System.ComponentModel.ISupportInitialize)(this.filterPrecision)).BeginInit(); + ((System.ComponentModel.ISupportInitialize)(this.parallelExportUpDown)).BeginInit(); this.SuspendLayout(); // // OKbutton @@ -106,6 +110,9 @@ // groupBox1 // this.groupBox1.AutoSize = true; + this.groupBox1.Controls.Add(this.parallelExportMaxLabel); + this.groupBox1.Controls.Add(this.parallelExportCheckBox); + this.groupBox1.Controls.Add(this.parallelExportUpDown); this.groupBox1.Controls.Add(this.filenameFormatLabel); this.groupBox1.Controls.Add(this.filenameFormatComboBox); this.groupBox1.Controls.Add(this.exportSpriteWithAlphaMask); @@ -593,6 +600,53 @@ this.eulerFilter.Text = "EulerFilter"; this.eulerFilter.UseVisualStyleBackColor = true; // + // parallelExportUpDown + // + this.parallelExportUpDown.Location = new System.Drawing.Point(209, 218); + this.parallelExportUpDown.Maximum = new decimal(new int[] { + 8, + 0, + 0, + 0}); + this.parallelExportUpDown.Minimum = new decimal(new int[] { + 1, + 0, + 0, + 0}); + this.parallelExportUpDown.Name = "parallelExportUpDown"; + this.parallelExportUpDown.Size = new System.Drawing.Size(42, 20); + this.parallelExportUpDown.TabIndex = 13; + this.parallelExportUpDown.Value = new decimal(new int[] { + 1, + 0, + 0, + 0}); + // + // parallelExportCheckBox + // + this.parallelExportCheckBox.AutoSize = true; + this.parallelExportCheckBox.Checked = true; + this.parallelExportCheckBox.CheckState = System.Windows.Forms.CheckState.Checked; + this.parallelExportCheckBox.Location = new System.Drawing.Point(6, 219); + this.parallelExportCheckBox.Name = "parallelExportCheckBox"; + this.parallelExportCheckBox.Size = new System.Drawing.Size(203, 17); + this.parallelExportCheckBox.TabIndex = 15; + this.parallelExportCheckBox.Text = "Export in parallel with number of tasks"; + this.optionTooltip.SetToolTip(this.parallelExportCheckBox, "*Requires slightly more RAM than in single-task mode"); + this.parallelExportCheckBox.UseVisualStyleBackColor = true; + this.parallelExportCheckBox.CheckedChanged += new System.EventHandler(this.parallelExportCheckBox_CheckedChanged); + // + // parallelExportMaxLabel + // + this.parallelExportMaxLabel.AutoSize = true; + this.parallelExportMaxLabel.ForeColor = System.Drawing.SystemColors.ControlDark; + this.parallelExportMaxLabel.Location = new System.Drawing.Point(256, 221); + this.parallelExportMaxLabel.Name = "parallelExportMaxLabel"; + this.parallelExportMaxLabel.Size = new System.Drawing.Size(33, 13); + this.parallelExportMaxLabel.TabIndex = 16; + this.parallelExportMaxLabel.Text = "Max: "; + this.optionTooltip.SetToolTip(this.parallelExportMaxLabel, "*The maximum number matches the number of CPU cores"); + // // ExportOptions // this.AcceptButton = this.OKbutton; @@ -626,6 +680,7 @@ ((System.ComponentModel.ISupportInitialize)(this.scaleFactor)).EndInit(); ((System.ComponentModel.ISupportInitialize)(this.boneSize)).EndInit(); ((System.ComponentModel.ISupportInitialize)(this.filterPrecision)).EndInit(); + ((System.ComponentModel.ISupportInitialize)(this.parallelExportUpDown)).EndInit(); this.ResumeLayout(false); this.PerformLayout(); @@ -675,5 +730,8 @@ private System.Windows.Forms.Panel l2dMotionExportMethodPanel; private System.Windows.Forms.ComboBox filenameFormatComboBox; private System.Windows.Forms.Label filenameFormatLabel; + private System.Windows.Forms.NumericUpDown parallelExportUpDown; + private System.Windows.Forms.CheckBox parallelExportCheckBox; + private System.Windows.Forms.Label parallelExportMaxLabel; } } \ No newline at end of file diff --git a/AssetStudioGUI/ExportOptions.cs b/AssetStudioGUI/ExportOptions.cs index ec5242d..9dc0add 100644 --- a/AssetStudioGUI/ExportOptions.cs +++ b/AssetStudioGUI/ExportOptions.cs @@ -34,6 +34,12 @@ namespace AssetStudioGUI ((RadioButton)l2dMotionExportMethodPanel.Controls.Cast().First(x => x.AccessibleName == defaultMotionMode)).Checked = true; l2dForceBezierCheckBox.Checked = Properties.Settings.Default.l2dForceBezier; filenameFormatComboBox.SelectedIndex = Properties.Settings.Default.filenameFormat; + var maxParallelTasks = Environment.ProcessorCount; + var taskCount = Properties.Settings.Default.parallelExportCount; + parallelExportUpDown.Maximum = maxParallelTasks; + parallelExportUpDown.Value = taskCount <= 0 ? maxParallelTasks : Math.Min(taskCount, maxParallelTasks); + parallelExportMaxLabel.Text += maxParallelTasks; + parallelExportCheckBox.Checked = Properties.Settings.Default.parallelExport; } private void OKbutton_Click(object sender, EventArgs e) @@ -62,6 +68,8 @@ namespace AssetStudioGUI Properties.Settings.Default.l2dMotionMode = (CubismLive2DExtractor.Live2DMotionMode)Enum.Parse(typeof(CubismLive2DExtractor.Live2DMotionMode), checkedMotionMode.AccessibleName); Properties.Settings.Default.l2dForceBezier = l2dForceBezierCheckBox.Checked; Properties.Settings.Default.filenameFormat = filenameFormatComboBox.SelectedIndex; + Properties.Settings.Default.parallelExport = parallelExportCheckBox.Checked; + Properties.Settings.Default.parallelExportCount = (int)parallelExportUpDown.Value; Properties.Settings.Default.Save(); DialogResult = DialogResult.OK; Close(); @@ -72,5 +80,10 @@ namespace AssetStudioGUI DialogResult = DialogResult.Cancel; Close(); } + + private void parallelExportCheckBox_CheckedChanged(object sender, EventArgs e) + { + parallelExportUpDown.Enabled = parallelExportCheckBox.Checked; + } } } diff --git a/AssetStudioGUI/ExportOptions.resx b/AssetStudioGUI/ExportOptions.resx index 7da86a7..ef2cd3d 100644 --- a/AssetStudioGUI/ExportOptions.resx +++ b/AssetStudioGUI/ExportOptions.resx @@ -120,4 +120,10 @@ 17, 17 + + 17, 17 + + + 17, 17 + \ No newline at end of file diff --git a/AssetStudioGUI/Exporter.cs b/AssetStudioGUI/Exporter.cs index ba9c873..b638c36 100644 --- a/AssetStudioGUI/Exporter.cs +++ b/AssetStudioGUI/Exporter.cs @@ -9,79 +9,6 @@ namespace AssetStudioGUI { internal static class Exporter { - public static bool ExportTexture2D(AssetItem item, string exportPath) - { - var m_Texture2D = (Texture2D)item.Asset; - if (Properties.Settings.Default.convertTexture) - { - var type = Properties.Settings.Default.convertType; - if (!TryExportFile(exportPath, item, "." + type.ToString().ToLower(), out var exportFullPath)) - return false; - var image = m_Texture2D.ConvertToImage(true); - if (image == null) - return false; - using (image) - { - using (var file = File.OpenWrite(exportFullPath)) - { - image.WriteToStream(file, type); - } - return true; - } - } - else - { - if (!TryExportFile(exportPath, item, ".tex", out var exportFullPath)) - return false; - File.WriteAllBytes(exportFullPath, m_Texture2D.image_data.GetData()); - return true; - } - } - - public static bool ExportTexture2DArray(AssetItem item, string exportPath) - { - var m_Texture2DArray = (Texture2DArray)item.Asset; - var count = 0; - foreach(var texture in m_Texture2DArray.TextureList) - { - var fakeItem = new AssetItem(texture) - { - Text = texture.m_Name, - Container = item.Container, - }; - if (ExportTexture2D(fakeItem, exportPath)) - { - count++; - } - } - return count > 0; - } - - public static bool ExportAudioClip(AssetItem item, string exportPath) - { - var m_AudioClip = (AudioClip)item.Asset; - var m_AudioData = m_AudioClip.m_AudioData.GetData(); - if (m_AudioData == null || m_AudioData.Length == 0) - return false; - var converter = new AudioClipConverter(m_AudioClip); - if (Properties.Settings.Default.convertAudio && converter.IsSupport) - { - if (!TryExportFile(exportPath, item, ".wav", out var exportFullPath)) - return false; - var buffer = converter.ConvertToWav(m_AudioData); - if (buffer == null) - return false; - File.WriteAllBytes(exportFullPath, buffer); - } - else - { - if (!TryExportFile(exportPath, item, converter.GetExtensionName(), out var exportFullPath)) - return false; - File.WriteAllBytes(exportFullPath, m_AudioData); - } - return true; - } - public static bool ExportShader(AssetItem item, string exportPath) { if (!TryExportFile(exportPath, item, ".shader", out var exportFullPath)) @@ -256,36 +183,15 @@ namespace AssetStudioGUI return true; } - public static bool ExportSprite(AssetItem item, string exportPath) - { - var type = Properties.Settings.Default.convertType; - var spriteMaskMode = Properties.Settings.Default.exportSpriteWithMask ? SpriteMaskMode.Export : SpriteMaskMode.Off; - if (!TryExportFile(exportPath, item, "." + type.ToString().ToLower(), out var exportFullPath)) - return false; - var image = ((Sprite)item.Asset).GetImage(spriteMaskMode: spriteMaskMode); - if (image != null) - { - using (image) - { - using (var file = File.OpenWrite(exportFullPath)) - { - image.WriteToStream(file, type); - } - return true; - } - } - return false; - } - public static bool ExportRawFile(AssetItem item, string exportPath) { - if (!TryExportFile(exportPath, item, ".dat", out var exportFullPath)) + if (!TryExportFile(exportPath, item, ".dat", out var exportFullPath, mode: "ExportRaw")) return false; File.WriteAllBytes(exportFullPath, item.Asset.GetRawData()); return true; } - private static bool TryExportFile(string dir, AssetItem item, string extension, out string fullPath) + private static bool TryExportFile(string dir, AssetItem item, string extension, out string fullPath, string mode = "Export") { var fileName = FixFileName(item.Text); var filenameFormatIndex = Properties.Settings.Default.filenameFormat; @@ -313,7 +219,7 @@ namespace AssetStudioGUI return true; } } - Logger.Warning($"Export error. File \"{fullPath.Color(ColorConsole.BrightYellow)}\" already exist"); + Logger.Warning($"{mode} failed. File \"{fullPath.Color(ColorConsole.BrightYellow)}\" already exist"); return false; } @@ -370,7 +276,7 @@ namespace AssetStudioGUI public static bool ExportDumpFile(AssetItem item, string exportPath) { - if (!TryExportFile(exportPath, item, ".txt", out var exportFullPath)) + if (!TryExportFile(exportPath, item, ".txt", out var exportFullPath, mode: "Dump")) return false; var str = item.Asset.Dump(); if (str == null && item.Asset is MonoBehaviour m_MonoBehaviour) @@ -396,11 +302,10 @@ namespace AssetStudioGUI { case ClassIDType.Texture2D: case ClassIDType.Texture2DArrayImage: - return ExportTexture2D(item, exportPath); case ClassIDType.Texture2DArray: - return ExportTexture2DArray(item, exportPath); case ClassIDType.AudioClip: - return ExportAudioClip(item, exportPath); + case ClassIDType.Sprite: + throw new System.NotImplementedException(); case ClassIDType.Shader: return ExportShader(item, exportPath); case ClassIDType.TextAsset: @@ -415,8 +320,6 @@ namespace AssetStudioGUI return ExportVideoClip(item, exportPath); case ClassIDType.MovieTexture: return ExportMovieTexture(item, exportPath); - case ClassIDType.Sprite: - return ExportSprite(item, exportPath); case ClassIDType.Animator: return ExportAnimator(item, exportPath); case ClassIDType.AnimationClip: @@ -428,8 +331,9 @@ namespace AssetStudioGUI public static string FixFileName(string str) { - if (str.Length >= 260) return Path.GetRandomFileName(); - return Path.GetInvalidFileNameChars().Aggregate(str, (current, c) => current.Replace(c, '_')); + return str.Length >= 260 + ? Path.GetRandomFileName() + : Path.GetInvalidFileNameChars().Aggregate(str, (current, c) => current.Replace(c, '_')); } } } diff --git a/AssetStudioGUI/GUILogger.cs b/AssetStudioGUI/GUILogger.cs index 64127c0..9860c81 100644 --- a/AssetStudioGUI/GUILogger.cs +++ b/AssetStudioGUI/GUILogger.cs @@ -1,20 +1,26 @@ using AssetStudio; using System; +using System.Collections.Concurrent; using System.IO; -using System.Linq; using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; using System.Windows.Forms; namespace AssetStudioGUI { class GUILogger : ILogger { - public bool ShowDebugMessage = false; - private bool IsFileLoggerRunning = false; - private string LoggerInitString; - private string FileLogName; - private string FileLogPath; + public static bool ShowDebugMessage = false; + + private bool isFileLoggerRunning = false; + private string loggerInitString; + private string fileLogName; + private string fileLogPath; private Action action; + private CancellationTokenSource tokenSource; + private BlockingCollection consoleLogMessageCollection = new BlockingCollection(); + private BlockingCollection fileLogMessageCollection = new BlockingCollection(); private bool _useFileLogger = false; public bool UseFileLogger @@ -23,19 +29,23 @@ namespace AssetStudioGUI set { _useFileLogger = value; - if (_useFileLogger && !IsFileLoggerRunning) + if (_useFileLogger && !isFileLoggerRunning) { var appAssembly = typeof(Program).Assembly.GetName(); - FileLogName = $"{appAssembly.Name}_{DateTime.Now:yyyy-MM-dd_HH-mm-ss}.log"; - FileLogPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, FileLogName); + fileLogName = $"{appAssembly.Name}_{DateTime.Now:yyyy-MM-dd_HH-mm-ss}.log"; + fileLogPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, fileLogName); + tokenSource = new CancellationTokenSource(); + isFileLoggerRunning = true; - LogToFile(LoggerEvent.Verbose, $"# {LoggerInitString} - Logger launched #"); - IsFileLoggerRunning = true; + ConcurrentFileWriter(tokenSource.Token); + LogToFile(LoggerEvent.Verbose, $"# {loggerInitString} - Logger launched #"); } - else if (!_useFileLogger && IsFileLoggerRunning) + else if (!_useFileLogger && isFileLoggerRunning) { LogToFile(LoggerEvent.Verbose, "# Logger closed #"); - IsFileLoggerRunning = false; + isFileLoggerRunning = false; + tokenSource.Cancel(); + tokenSource.Dispose(); } } } @@ -47,7 +57,7 @@ namespace AssetStudioGUI var appAssembly = typeof(Program).Assembly.GetName(); var arch = Environment.Is64BitProcess ? "x64" : "x32"; var frameworkName = AppDomain.CurrentDomain.SetupInformation.TargetFrameworkName; - LoggerInitString = $"{appAssembly.Name} v{appAssembly.Version} [{arch}] [{frameworkName}]"; + loggerInitString = $"{appAssembly.Name} v{appAssembly.Version} [{arch}] [{frameworkName}]"; try { Console.Title = $"Console Logger - {appAssembly.Name} v{appAssembly.Version}"; @@ -57,7 +67,9 @@ namespace AssetStudioGUI { // ignored } - Console.WriteLine($"# {LoggerInitString}"); + + ConcurrentConsoleWriter(); + Console.WriteLine($"# {loggerInitString}"); } private static string ColorLogLevel(LoggerEvent logLevel) @@ -79,7 +91,7 @@ namespace AssetStudioGUI private static string FormatMessage(LoggerEvent logMsgLevel, string message, bool toConsole) { message = message.TrimEnd(); - var multiLine = message.Contains('\n'); + var multiLine = message.Contains("\n"); string formattedMessage; if (toConsole) @@ -88,7 +100,7 @@ namespace AssetStudioGUI formattedMessage = $"{colorLogLevel} {message}"; if (multiLine) { - formattedMessage = formattedMessage.Replace("\n", $"\n{colorLogLevel} "); + formattedMessage = formattedMessage.Replace("\n", $"\n{colorLogLevel} ") + $"\n{colorLogLevel}"; } } else @@ -99,19 +111,48 @@ namespace AssetStudioGUI formattedMessage = $"{curTime} | {logLevel} | {message}"; if (multiLine) { - formattedMessage = formattedMessage.Replace("\n", $"\n{curTime} | {logLevel} | "); + formattedMessage = formattedMessage.Replace("\n", $"\n{curTime} | {logLevel} | ") + $"\n{curTime} | {logLevel} |"; } } - return formattedMessage; } - private async void LogToFile(LoggerEvent logMsgLevel, string message) + private void ConcurrentFileWriter(CancellationToken token) { - using (var sw = new StreamWriter(FileLogPath, append: true, System.Text.Encoding.UTF8)) + Task.Run(() => { - await sw.WriteLineAsync(FormatMessage(logMsgLevel, message, toConsole: false)); - } + using (var sw = new StreamWriter(fileLogPath, append: true, System.Text.Encoding.UTF8)) + { + sw.AutoFlush = true; + foreach (var msg in fileLogMessageCollection.GetConsumingEnumerable()) + { + sw.WriteLine(msg); + if (token.IsCancellationRequested) + break; + } + } + }, token); + } + + private void ConcurrentConsoleWriter() + { + Task.Run(() => + { + foreach (var msg in consoleLogMessageCollection.GetConsumingEnumerable()) + { + Console.WriteLine(msg); + } + }); + } + + private void LogToFile(LoggerEvent logMsgLevel, string message) + { + fileLogMessageCollection.Add(FormatMessage(logMsgLevel, message, toConsole: false)); + } + + private void LogToConsole(LoggerEvent logMsgLevel, string message) + { + consoleLogMessageCollection.Add(FormatMessage(logMsgLevel, message, toConsole: true)); } public void Log(LoggerEvent loggerEvent, string message, bool ignoreLevel) @@ -125,7 +166,7 @@ namespace AssetStudioGUI //Console logger if (!ShowDebugMessage && loggerEvent == LoggerEvent.Debug) return; - Console.WriteLine(FormatMessage(loggerEvent, message, toConsole: true)); + LogToConsole(loggerEvent, message); //GUI logger switch (loggerEvent) diff --git a/AssetStudioGUI/ParallelExport.cs b/AssetStudioGUI/ParallelExport.cs new file mode 100644 index 0000000..77a33f0 --- /dev/null +++ b/AssetStudioGUI/ParallelExport.cs @@ -0,0 +1,226 @@ +using AssetStudio; +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Text; + +namespace AssetStudioGUI +{ + internal static class ParallelExporter + { + private static ConcurrentDictionary savePathHash = new ConcurrentDictionary(); + + public static bool ExportTexture2D(AssetItem item, string exportPath, out string debugLog) + { + debugLog = ""; + var m_Texture2D = (Texture2D)item.Asset; + if (Properties.Settings.Default.convertTexture) + { + var type = Properties.Settings.Default.convertType; + if (!TryExportFile(exportPath, item, "." + type.ToString().ToLower(), out var exportFullPath)) + return false; + + if (GUILogger.ShowDebugMessage) + { + var sb = new StringBuilder(); + sb.AppendLine($"Converting {item.TypeString} \"{m_Texture2D.m_Name}\" to {type}.."); + sb.AppendLine($"Width: {m_Texture2D.m_Width}"); + sb.AppendLine($"Height: {m_Texture2D.m_Height}"); + sb.AppendLine($"Format: {m_Texture2D.m_TextureFormat}"); + switch (m_Texture2D.m_TextureSettings.m_FilterMode) + { + case 0: sb.AppendLine("Filter Mode: Point "); break; + case 1: sb.AppendLine("Filter Mode: Bilinear "); break; + case 2: sb.AppendLine("Filter Mode: Trilinear "); break; + } + sb.AppendLine($"Anisotropic level: {m_Texture2D.m_TextureSettings.m_Aniso}"); + sb.AppendLine($"Mip map bias: {m_Texture2D.m_TextureSettings.m_MipBias}"); + switch (m_Texture2D.m_TextureSettings.m_WrapMode) + { + case 0: sb.AppendLine($"Wrap mode: Repeat"); break; + case 1: sb.AppendLine($"Wrap mode: Clamp"); break; + } + debugLog += sb.ToString(); + } + + var image = m_Texture2D.ConvertToImage(flip: true); + if (image == null) + { + Logger.Warning($"Failed to convert texture \"{m_Texture2D.m_Name}\" into image"); + return false; + } + using (image) + { + using (var file = File.OpenWrite(exportFullPath)) + { + image.WriteToStream(file, type); + } + debugLog += $"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""; + return true; + } + } + else + { + if (!TryExportFile(exportPath, item, ".tex", out var exportFullPath)) + return false; + File.WriteAllBytes(exportFullPath, m_Texture2D.image_data.GetData()); + debugLog += $"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""; + return true; + } + } + + public static bool ExportSprite(AssetItem item, string exportPath, out string debugLog) + { + debugLog = ""; + var type = Properties.Settings.Default.convertType; + var spriteMaskMode = Properties.Settings.Default.exportSpriteWithMask ? SpriteMaskMode.Export : SpriteMaskMode.Off; + if (!TryExportFile(exportPath, item, "." + type.ToString().ToLower(), out var exportFullPath)) + return false; + var image = ((Sprite)item.Asset).GetImage(spriteMaskMode: spriteMaskMode); + if (image != null) + { + using (image) + { + using (var file = File.OpenWrite(exportFullPath)) + { + image.WriteToStream(file, type); + } + debugLog += $"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""; + return true; + } + } + return false; + } + + public static bool ExportAudioClip(AssetItem item, string exportPath, out string debugLog) + { + debugLog = ""; + string exportFullPath; + var m_AudioClip = (AudioClip)item.Asset; + var m_AudioData = BigArrayPool.Shared.Rent(m_AudioClip.m_AudioData.Size); + try + { + m_AudioClip.m_AudioData.GetData(m_AudioData); + if (m_AudioData == null || m_AudioData.Length == 0) + { + Logger.Warning($"Failed to export \"{item.Text}\": AudioData was not found"); + return false; + } + var converter = new AudioClipConverter(m_AudioClip); + if (Properties.Settings.Default.convertAudio && converter.IsSupport) + { + if (!TryExportFile(exportPath, item, ".wav", out exportFullPath)) + return false; + + if (GUILogger.ShowDebugMessage) + { + var sb = new StringBuilder(); + sb.AppendLine($"Converting {item.TypeString} \"{m_AudioClip.m_Name}\" to wav.."); + sb.AppendLine(m_AudioClip.version[0] < 5 + ? $"AudioClip type: {m_AudioClip.m_Type}" + : $"AudioClip compression format: {m_AudioClip.m_CompressionFormat}"); + sb.AppendLine($"AudioClip channel count: {m_AudioClip.m_Channels}"); + sb.AppendLine($"AudioClip sample rate: {m_AudioClip.m_Frequency}"); + sb.AppendLine($"AudioClip bit depth: {m_AudioClip.m_BitsPerSample}"); + debugLog += sb.ToString(); + } + + var buffer = converter.ConvertToWav(m_AudioData, out var debugLogConverter); + debugLog += debugLogConverter; + if (buffer == null) + { + Logger.Warning($"{debugLog}Failed to export \"{item.Text}\": Failed to convert fmod audio to Wav"); + return false; + } + File.WriteAllBytes(exportFullPath, buffer); + } + else + { + if (!TryExportFile(exportPath, item, converter.GetExtensionName(), out exportFullPath)) + return false; + + if (GUILogger.ShowDebugMessage) + { + var sb = new StringBuilder(); + sb.AppendLine($"Exporting non-fmod {item.TypeString} \"{m_AudioClip.m_Name}\".."); + sb.AppendLine(m_AudioClip.version[0] < 5 + ? $"AudioClip type: {m_AudioClip.m_Type}" + : $"AudioClip compression format: {m_AudioClip.m_CompressionFormat}"); + sb.AppendLine($"AudioClip channel count: {m_AudioClip.m_Channels}"); + sb.AppendLine($"AudioClip sample rate: {m_AudioClip.m_Frequency}"); + sb.AppendLine($"AudioClip bit depth: {m_AudioClip.m_BitsPerSample}"); + debugLog += sb.ToString(); + } + File.WriteAllBytes(exportFullPath, m_AudioData); + } + debugLog += $"{item.TypeString} \"{item.Text}\" exported to \"{exportFullPath}\""; + return true; + } + finally + { + BigArrayPool.Shared.Return(m_AudioData, clearArray: true); + } + } + + private static bool TryExportFile(string dir, AssetItem item, string extension, out string fullPath) + { + var fileName = FixFileName(item.Text); + var filenameFormatIndex = Properties.Settings.Default.filenameFormat; + switch (filenameFormatIndex) + { + case 1: //assetName@pathID + fileName = $"{fileName} @{item.m_PathID}"; + break; + case 2: //pathID + fileName = item.m_PathID.ToString(); + break; + } + fullPath = Path.Combine(dir, fileName + extension); + if (savePathHash.TryAdd(fullPath.ToLower(), true) && !File.Exists(fullPath)) + { + Directory.CreateDirectory(dir); + return true; + } + if (filenameFormatIndex == 0) //assetName + { + fullPath = Path.Combine(dir, fileName + item.UniqueID + extension); + if (!File.Exists(fullPath)) + { + Directory.CreateDirectory(dir); + return true; + } + } + Logger.Warning($"Export failed. File \"{fullPath.Color(ColorConsole.BrightYellow)}\" already exist"); + return false; + } + + public static bool ParallelExportConvertFile(AssetItem item, string exportPath, out string debugLog) + { + switch (item.Type) + { + case ClassIDType.Texture2D: + case ClassIDType.Texture2DArrayImage: + return ExportTexture2D(item, exportPath, out debugLog); + case ClassIDType.Sprite: + return ExportSprite(item, exportPath, out debugLog); + case ClassIDType.AudioClip: + return ExportAudioClip(item, exportPath, out debugLog); + default: + throw new NotImplementedException(); + } + } + + private static string FixFileName(string str) + { + return str.Length >= 260 + ? Path.GetRandomFileName() + : Path.GetInvalidFileNameChars().Aggregate(str, (current, c) => current.Replace(c, '_')); + } + + public static void ClearHash() + { + savePathHash.Clear(); + } + } +} diff --git a/AssetStudioGUI/Properties/Settings.Designer.cs b/AssetStudioGUI/Properties/Settings.Designer.cs index 9ce6082..58d204e 100644 --- a/AssetStudioGUI/Properties/Settings.Designer.cs +++ b/AssetStudioGUI/Properties/Settings.Designer.cs @@ -12,7 +12,7 @@ namespace AssetStudioGUI.Properties { [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "17.8.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "17.9.0.0")] internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase { private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings()))); @@ -358,5 +358,29 @@ namespace AssetStudioGUI.Properties { this["filenameFormat"] = value; } } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("True")] + public bool parallelExport { + get { + return ((bool)(this["parallelExport"])); + } + set { + this["parallelExport"] = value; + } + } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("-1")] + public int parallelExportCount { + get { + return ((int)(this["parallelExportCount"])); + } + set { + this["parallelExportCount"] = value; + } + } } } diff --git a/AssetStudioGUI/Properties/Settings.settings b/AssetStudioGUI/Properties/Settings.settings index 7594d73..183b71d 100644 --- a/AssetStudioGUI/Properties/Settings.settings +++ b/AssetStudioGUI/Properties/Settings.settings @@ -86,5 +86,11 @@ 0 + + True + + + -1 + \ No newline at end of file diff --git a/AssetStudioGUI/Studio.cs b/AssetStudioGUI/Studio.cs index 6db4f92..34542f2 100644 --- a/AssetStudioGUI/Studio.cs +++ b/AssetStudioGUI/Studio.cs @@ -1,12 +1,14 @@ using AssetStudio; using CubismLive2DExtractor; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; using System.Threading; +using System.Threading.Tasks; using System.Windows.Forms; using System.Xml.Linq; using static AssetStudioGUI.Exporter; @@ -468,13 +470,21 @@ namespace AssetStudioGUI { Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US"); - int toExportCount = toExportAssets.Count; - int exportedCount = 0; - int i = 0; - Progress.Reset(); var groupOption = (AssetGroupOption)Properties.Settings.Default.assetGroupOption; + var parallelExportCount = Properties.Settings.Default.parallelExportCount <= 0 + ? Environment.ProcessorCount - 1 + : Math.Min(Properties.Settings.Default.parallelExportCount, Environment.ProcessorCount - 1); + parallelExportCount = Properties.Settings.Default.parallelExport ? parallelExportCount : 1; + var toExportAssetDict = new ConcurrentDictionary(); + var toParallelExportAssetDict = new ConcurrentDictionary(); + var exceptionMsgs = new ConcurrentDictionary(); var mode = exportType == ExportType.Dump ? "Dump" : "Export"; - foreach (var asset in toExportAssets) + var toExportCount = toExportAssets.Count; + var exportedCount = 0; + var i = 0; + Progress.Reset(); + + Parallel.ForEach(toExportAssets, asset => { string exportPath; switch (groupOption) @@ -522,10 +532,49 @@ namespace AssetStudioGUI break; } exportPath += Path.DirectorySeparatorChar; - Logger.Info($"[{exportedCount + 1}/{toExportCount}] {mode}ing {asset.TypeString}: {asset.Text}"); + + if (exportType == ExportType.Convert) + { + switch (asset.Type) + { + case ClassIDType.Texture2D: + case ClassIDType.Texture2DArrayImage: + case ClassIDType.Sprite: + case ClassIDType.AudioClip: + toParallelExportAssetDict.TryAdd(asset, exportPath); + break; + case ClassIDType.Texture2DArray: + var m_Texture2DArray = (Texture2DArray)asset.Asset; + toExportCount += m_Texture2DArray.TextureList.Count - 1; + foreach (var texture in m_Texture2DArray.TextureList) + { + var fakeItem = new AssetItem(texture) + { + Text = texture.m_Name, + Container = asset.Container, + }; + toParallelExportAssetDict.TryAdd(fakeItem, exportPath); + } + break; + default: + toExportAssetDict.TryAdd(asset, exportPath); + break; + } + } + else + { + toExportAssetDict.TryAdd(asset, exportPath); + } + }); + + foreach (var toExportAsset in toExportAssetDict) + { + var asset = toExportAsset.Key; + var exportPath = toExportAsset.Value; var isExported = false; try { + Logger.Info($"[{exportedCount + 1}/{toExportCount}] {mode}ing {asset.TypeString}: {asset.Text}"); switch (exportType) { case ExportType.Raw: @@ -556,14 +605,58 @@ namespace AssetStudioGUI Progress.Report(++i, toExportCount); } - var statusText = exportedCount == 0 ? "Nothing exported." : $"Finished {mode.ToLower()}ing {exportedCount} assets."; - - if (toExportCount > exportedCount) + Parallel.ForEach(toParallelExportAssetDict, new ParallelOptions { MaxDegreeOfParallelism = parallelExportCount }, (toExportAsset, loopState) => { - statusText += $" {toExportCount - exportedCount} assets skipped (not extractable or files already exist)"; + var asset = toExportAsset.Key; + var exportPath = toExportAsset.Value; + try + { + if (ParallelExporter.ParallelExportConvertFile(asset, exportPath, out var debugLog)) + { + Interlocked.Increment(ref exportedCount); + if (GUILogger.ShowDebugMessage) + { + Logger.Debug(debugLog); + StatusStripUpdate($"[{exportedCount}/{toExportCount}] Exporting {asset.TypeString}: {asset.Text}"); + } + else + { + Logger.Info($"[{exportedCount}/{toExportCount}] Exporting {asset.TypeString}: {asset.Text}"); + } + } + Interlocked.Increment(ref i); + Progress.Report(i, toExportCount); + } + catch (Exception ex) + { + if (parallelExportCount == 1) + { + Logger.Error($"{mode} {asset.TypeString}: {asset.Text} error", ex); + } + else + { + loopState.Break(); + exceptionMsgs.TryAdd(ex, $"Exception occurred when exporting {asset.TypeString}: {asset.Text}\n{ex}\n"); + } + } + }); + ParallelExporter.ClearHash(); + + foreach (var ex in exceptionMsgs) + { + Logger.Error(ex.Value); } + var statusText = exportedCount == 0 ? "Nothing exported." : $"Finished {mode.ToLower()}ing [{exportedCount}/{toExportCount}] assets."; + if (toExportCount > exportedCount) + { + statusText += exceptionMsgs.IsEmpty + ? $" {toExportCount - exportedCount} assets skipped (not extractable or files already exist)." + : " Export process was stopped because one or more exceptions occurred."; + Progress.Report(toExportCount, toExportCount); + } Logger.Info(statusText); + exceptionMsgs.Clear(); if (Properties.Settings.Default.openAfterExport && exportedCount > 0) { @@ -607,11 +700,11 @@ namespace AssetStudioGUI break; } - var statusText = $"Finished exporting asset list with {toExportAssets.Count()} items."; + var statusText = $"Finished exporting asset list with {toExportAssets.Count} items."; Logger.Info(statusText); - if (Properties.Settings.Default.openAfterExport && toExportAssets.Count() > 0) + if (Properties.Settings.Default.openAfterExport && toExportAssets.Count > 0) { OpenFolderInExplorer(savePath); } @@ -884,7 +977,7 @@ namespace AssetStudioGUI } mocPathDict.Clear(); - var lookup = l2dResourceContainers.ToLookup( + var lookup = l2dResourceContainers.AsParallel().ToLookup( x => mocPathList.Find(b => x.Value.Contains(b) && x.Value.Split('/').Any(y => y == b.Substring(b.LastIndexOf("/") + 1))), x => x.Key ); @@ -897,6 +990,10 @@ namespace AssetStudioGUI var totalModelCount = lookup.LongCount(x => x.Key != null); var modelCounter = 0; + var parallelExportCount = Properties.Settings.Default.parallelExportCount <= 0 + ? Environment.ProcessorCount - 1 + : Math.Min(Properties.Settings.Default.parallelExportCount, Environment.ProcessorCount - 1); + parallelExportCount = Properties.Settings.Default.parallelExport ? parallelExportCount : 1; foreach (var assets in lookup) { var srcContainer = assets.Key; @@ -916,7 +1013,7 @@ namespace AssetStudioGUI var destPath = Path.Combine(baseDestPath, container) + Path.DirectorySeparatorChar; var modelExtractor = new Live2DExtractor(assets, selClipMotions, selFadeMotions, selFadeLst); - modelExtractor.ExtractCubismModel(destPath, modelName, motionMode, assemblyLoader, forceBezier); + modelExtractor.ExtractCubismModel(destPath, modelName, motionMode, assemblyLoader, forceBezier, parallelExportCount); modelCounter++; } catch (Exception ex) diff --git a/AssetStudioUtility/AudioClipConverter.cs b/AssetStudioUtility/AudioClipConverter.cs index c088e99..3dec1b7 100644 --- a/AssetStudioUtility/AudioClipConverter.cs +++ b/AssetStudioUtility/AudioClipConverter.cs @@ -10,24 +10,34 @@ namespace AssetStudio public bool IsSupport => m_AudioClip.IsConvertSupport(); private AudioClip m_AudioClip; + private static FMOD.System system; + + static AudioClipConverter() + { + var result = Factory.System_Create(out system); + if (result != RESULT.OK) + { + Logger.Error($"FMOD error! {result} - {Error.String(result)}"); + } + result = system.init(1, INITFLAGS.NORMAL, IntPtr.Zero); + if (result != RESULT.OK) + { + Logger.Error($"FMOD error! {result} - {Error.String(result)}"); + } + } public AudioClipConverter(AudioClip audioClip) { m_AudioClip = audioClip; } - public byte[] ConvertToWav(byte[] m_AudioData) + public byte[] ConvertToWav(byte[] m_AudioData, out string debugLog) { + debugLog = ""; var exinfo = new CREATESOUNDEXINFO(); - var result = Factory.System_Create(out var system); - if (result != RESULT.OK) - return null; - result = system.init(1, INITFLAGS.NORMAL, IntPtr.Zero); - if (result != RESULT.OK) - return null; exinfo.cbsize = Marshal.SizeOf(exinfo); exinfo.length = (uint)m_AudioClip.m_Size; - result = system.createSound(m_AudioData, MODE.OPENMEMORY, ref exinfo, out var sound); + var result = system.createSound(m_AudioData, MODE.OPENMEMORY, ref exinfo, out var sound); if (result != RESULT.OK) return null; result = sound.getNumSubSounds(out var numsubsounds); @@ -39,28 +49,29 @@ namespace AssetStudio result = sound.getSubSound(0, out var subsound); if (result != RESULT.OK) return null; - buff = SoundToWav(subsound); + buff = SoundToWav(subsound, out debugLog); subsound.release(); + subsound.clearHandle(); } else { - buff = SoundToWav(sound); + buff = SoundToWav(sound, out debugLog); } sound.release(); - system.release(); + sound.clearHandle(); return buff; } - public byte[] SoundToWav(Sound sound) + public byte[] SoundToWav(Sound sound, out string debugLog) { - Logger.Debug($"[Fmod] Detecting sound format..\n"); + debugLog = "[Fmod] Detecting sound format..\n"; var result = sound.getFormat(out SOUND_TYPE soundType, out SOUND_FORMAT soundFormat, out int channels, out int bits); if (result != RESULT.OK) return null; - Logger.Debug($"Detected sound type: {soundType}\n" + - $"Detected sound format: {soundFormat}\n" + - $"Detected channels: {channels}\n" + - $"Detected bit depth: {bits}"); + debugLog += $"Detected sound type: {soundType}\n" + + $"Detected sound format: {soundFormat}\n" + + $"Detected channels: {channels}\n" + + $"Detected bit depth: {bits}\n"; result = sound.getDefaults(out var frequency, out _); if (result != RESULT.OK) return null; @@ -71,11 +82,11 @@ namespace AssetStudio result = sound.@lock(0, length, out var ptr1, out var ptr2, out var len1, out var len2); if (result != RESULT.OK) return null; - byte[] buffer = new byte[len1 + 44]; + var buffer = new byte[len1 + 44]; //添加wav头 - Encoding.UTF8.GetBytes("RIFF").CopyTo(buffer, 0); + Encoding.ASCII.GetBytes("RIFF").CopyTo(buffer, 0); BitConverter.GetBytes(len1 + 36).CopyTo(buffer, 4); - Encoding.UTF8.GetBytes("WAVEfmt ").CopyTo(buffer, 8); + Encoding.ASCII.GetBytes("WAVEfmt ").CopyTo(buffer, 8); BitConverter.GetBytes(16).CopyTo(buffer, 16); BitConverter.GetBytes((short)1).CopyTo(buffer, 20); BitConverter.GetBytes((short)channels).CopyTo(buffer, 22); @@ -83,7 +94,7 @@ namespace AssetStudio BitConverter.GetBytes(sampleRate * channels * bits / 8).CopyTo(buffer, 28); BitConverter.GetBytes((short)(channels * bits / 8)).CopyTo(buffer, 32); BitConverter.GetBytes((short)bits).CopyTo(buffer, 34); - Encoding.UTF8.GetBytes("data").CopyTo(buffer, 36); + Encoding.ASCII.GetBytes("data").CopyTo(buffer, 36); BitConverter.GetBytes(len1).CopyTo(buffer, 40); Marshal.Copy(ptr1, buffer, 44, (int)len1); result = sound.unlock(ptr1, ptr2, len1, len2); @@ -151,7 +162,6 @@ namespace AssetStudio return ".fsb"; } } - return ".AudioClip"; } } diff --git a/AssetStudioUtility/CubismLive2DExtractor/Live2DExtractor.cs b/AssetStudioUtility/CubismLive2DExtractor/Live2DExtractor.cs index 9193b48..5d91063 100644 --- a/AssetStudioUtility/CubismLive2DExtractor/Live2DExtractor.cs +++ b/AssetStudioUtility/CubismLive2DExtractor/Live2DExtractor.cs @@ -4,10 +4,12 @@ //// using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; +using System.Threading.Tasks; using AssetStudio; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -47,7 +49,7 @@ namespace CubismLive2DExtractor ParametersCdi = new List(); PartsCdi = new List(); - Logger.Info("Sorting model assets.."); + Logger.Debug("Sorting model assets.."); foreach (var asset in assets) { switch (asset) @@ -133,7 +135,7 @@ namespace CubismLive2DExtractor } } - public void ExtractCubismModel(string destPath, string modelName, Live2DMotionMode motionMode, AssemblyLoader assemblyLoader, bool forceBezier = false) + public void ExtractCubismModel(string destPath, string modelName, Live2DMotionMode motionMode, AssemblyLoader assemblyLoader, bool forceBezier = false, int parallelTaskCount = 1) { Directory.CreateDirectory(destPath); @@ -174,17 +176,24 @@ namespace CubismLive2DExtractor Directory.CreateDirectory(destTexturePath); } - foreach (var texture2D in Texture2Ds) + var textureBag = new ConcurrentBag(); + var savePathHash = new ConcurrentDictionary(); + Parallel.ForEach(Texture2Ds, new ParallelOptions { MaxDegreeOfParallelism = parallelTaskCount }, texture2D => { + var savePath = $"{destTexturePath}{texture2D.m_Name}.png"; + if (!savePathHash.TryAdd(savePath, true)) + return; + using (var image = texture2D.ConvertToImage(flip: true)) { - using (var file = File.OpenWrite($"{destTexturePath}{texture2D.m_Name}.png")) + using (var file = File.OpenWrite(savePath)) { image.WriteToStream(file, ImageFormat.Png); } - textures.Add($"textures/{texture2D.m_Name}.png"); + textureBag.Add($"textures/{texture2D.m_Name}.png"); } - } + }); + textures.UnionWith(textureBag); #endregion #region physics3.json