improve the speed of loading a large number of files

Rewriting folder related functions
fixed some bug

Unity Studio/Unity Studio Classes/UnityStudio.cs

@@ -14,12 +14,13 @@ namespace Unity_Studio
     internal static class UnityStudio
     {
         public static List<string> unityFiles = new List<string>(); //files to load
-        public static HashSet<string> unityFilesHash = new HashSet<string>(); //improve performance
+        public static HashSet<string> unityFilesHash = new HashSet<string>(); //to improve the loading speed
         public static List<AssetsFile> assetsfileList = new List<AssetsFile>(); //loaded files
-        public static HashSet<string> assetsfileListHash = new HashSet<string>(); //improve performance
+        public static HashSet<string> assetsfileListHash = new HashSet<string>(); //to improve the loading speed
+        public static Dictionary<string, int> sharedFileIndex = new Dictionary<string, int>(); //to improve the loading speed
         public static Dictionary<string, EndianBinaryReader> assetsfileandstream = new Dictionary<string, EndianBinaryReader>(); //use for read res files
         public static List<AssetPreloadData> exportableAssets = new List<AssetPreloadData>(); //used to hold all assets while the ListView is filtered
-        private static HashSet<string> exportableAssetsHash = new HashSet<string>(); //improve performance
+        private static HashSet<string> exportableAssetsHash = new HashSet<string>(); //avoid the same name asset
         public static List<AssetPreloadData> visibleAssets = new List<AssetPreloadData>(); //used to build the ListView from all or filtered assets
 
         public static string productName = "";
@@ -89,7 +90,12 @@ namespace Unity_Studio
                     }
                     else
                     {
-                        sharedFile.Index = unityFiles.IndexOf(sharedFilePath);
+                        if (!sharedFileIndex.TryGetValue(sharedFilePath, out var index))
+                        {
+                            index = unityFiles.IndexOf(sharedFilePath);
+                            sharedFileIndex.Add(sharedFilePath, index);
+                        }
+                        sharedFile.Index = index;
                     }
                 }
                 if (value > 0)
@@ -124,7 +130,7 @@ namespace Unity_Studio
             }
         }
 
-        public static void LoadAssetsFromBundle()
+        public static void BuildSharedIndex()
         {
             foreach (var assetsFile in assetsfileList)
             {
@@ -162,35 +168,31 @@ namespace Unity_Studio
         public static int extractBundleFile(string bundleFileName)
         {
             int extractedCount = 0;
-
+            if (CheckBundleFile(bundleFileName))
+            {
                 StatusStripUpdate($"Decompressing {Path.GetFileName(bundleFileName)} ...");
-
+                var extractPath = bundleFileName + "_unpacked\\";
-            string extractPath = bundleFileName + "_unpacked\\";
                 Directory.CreateDirectory(extractPath);
-
+                var b_File = new BundleFile(bundleFileName);
-            BundleFile b_File = new BundleFile(bundleFileName);
-
                 foreach (var memFile in b_File.MemoryAssetsFileList)
                 {
-                string filePath = extractPath + memFile.fileName.Replace('/', '\\');
+                    var filePath = extractPath + memFile.fileName.Replace('/', '\\');
                     if (!Directory.Exists(Path.GetDirectoryName(filePath)))
                     {
                         Directory.CreateDirectory(Path.GetDirectoryName(filePath));
-
                     }
                     if (!File.Exists(filePath))
                     {
                         StatusStripUpdate($"Extracting {Path.GetFileName(memFile.fileName)}");
                         extractedCount += 1;
-
+                        using (var file = File.Create(filePath))
-                    using (FileStream file = new FileStream(filePath, FileMode.Create, FileAccess.Write))
                         {
                             memFile.memStream.WriteTo(file);
                             memFile.memStream.Close();
                         }
                     }
                 }
-
+            }
             return extractedCount;
         }
 
@@ -359,8 +361,6 @@ namespace Unity_Studio
                 }
 
                 visibleAssets = exportableAssets;
-
-                //will only work if ListView is visible
                 exportableAssetsHash.Clear();
             }
             #endregion
@@ -1867,5 +1867,22 @@ namespace Unity_Studio
                 }
             }
         }
+
+        public static string[] ProcessingSplitFiles(List<string> selectFile)
+        {
+            var splitFiles = selectFile.Where(x => x.Contains(".split"))
+                .Select(x => Path.GetDirectoryName(x) + "\\" + Path.GetFileNameWithoutExtension(x))
+                .Distinct()
+                .ToList();
+            selectFile.RemoveAll(x => x.Contains(".split"));
+            foreach (var file in splitFiles)
+            {
+                if (File.Exists(file))
+                {
+                    selectFile.Add(file);
+                }
+            }
+            return selectFile.Distinct().ToArray();
+        }
     }
 }

Unity Studio/UnityStudioForm.Designer.cs

@@ -153,14 +153,14 @@
             // 
             this.loadFileToolStripMenuItem.Name = "loadFileToolStripMenuItem";
             this.loadFileToolStripMenuItem.Size = new System.Drawing.Size(158, 22);
-            this.loadFileToolStripMenuItem.Text = "Load file...";
+            this.loadFileToolStripMenuItem.Text = "Load file";
             this.loadFileToolStripMenuItem.Click += new System.EventHandler(this.loadFile_Click);
             // 
             // loadFolderToolStripMenuItem
             // 
             this.loadFolderToolStripMenuItem.Name = "loadFolderToolStripMenuItem";
             this.loadFolderToolStripMenuItem.Size = new System.Drawing.Size(158, 22);
-            this.loadFolderToolStripMenuItem.Text = "Load folder...";
+            this.loadFolderToolStripMenuItem.Text = "Load folder";
             this.loadFolderToolStripMenuItem.Click += new System.EventHandler(this.loadFolder_Click);
             // 
             // toolStripMenuItem1
@@ -172,14 +172,14 @@
             // 
             this.extractBundleToolStripMenuItem.Name = "extractBundleToolStripMenuItem";
             this.extractBundleToolStripMenuItem.Size = new System.Drawing.Size(158, 22);
-            this.extractBundleToolStripMenuItem.Text = "Extract bundle...";
+            this.extractBundleToolStripMenuItem.Text = "Extract bundle";
             this.extractBundleToolStripMenuItem.Click += new System.EventHandler(this.extractBundleToolStripMenuItem_Click);
             // 
             // extractFolderToolStripMenuItem
             // 
             this.extractFolderToolStripMenuItem.Name = "extractFolderToolStripMenuItem";
             this.extractFolderToolStripMenuItem.Size = new System.Drawing.Size(158, 22);
-            this.extractFolderToolStripMenuItem.Text = "Extract folder...";
+            this.extractFolderToolStripMenuItem.Text = "Extract folder";
             this.extractFolderToolStripMenuItem.Click += new System.EventHandler(this.extractFolderToolStripMenuItem_Click);
             // 
             // debugMenuItem
@@ -310,7 +310,7 @@
             // 
             this.showExpOpt.Name = "showExpOpt";
             this.showExpOpt.Size = new System.Drawing.Size(252, 22);
-            this.showExpOpt.Text = "Export options...";
+            this.showExpOpt.Text = "Export options";
             this.showExpOpt.Click += new System.EventHandler(this.showExpOpt_Click);
             // 
             // exportToolStripMenuItem

Unity Studio/UnityStudioForm.cs

@@ -22,9 +22,6 @@ namespace Unity_Studio
         private AssetPreloadData lastSelectedItem;
         private AssetPreloadData lastLoadedAsset;
 
-        private string[] assetsFileTypes = { "globalgamemanagers", "maindata.", "level*.", "*.assets", "*.sharedAssets", "CustomAssetBundle-*", "CAB-*", "BuildPlayer-*" };
-        private string[] bundleFileTypes = { "*.unity3d", "*.unity3d.lz4", "*.assetbundle", "*.assetbundle-*", "*.bundle", "*.bytes" };
-
         private FMOD.System system;
         private FMOD.Sound sound;
         private FMOD.Channel channel;
@@ -87,7 +84,6 @@ namespace Unity_Studio
                 resetForm();
                 ThreadPool.QueueUserWorkItem(state =>
                 {
-                    mainPath = Path.GetDirectoryName(openFileDialog1.FileNames[0]);
                     var bundle = false;
                     if (openFileDialog1.FilterIndex == 1 || openFileDialog1.FilterIndex == 3)
                     {
@@ -98,20 +94,19 @@ namespace Unity_Studio
                                 MessageBox.Show($"{Path.GetFileName(openFileDialog1.FileNames[0])} is bundle file, please select bundle file type to load this file");
                                 return;
                             }
-                            else
-                            {
                             bundle = true;
                         }
                     }
-                    }
                     else
                     {
                         bundle = true;
                     }
                     if (!bundle)
                     {
+                        mainPath = Path.GetDirectoryName(openFileDialog1.FileNames[0]);
                         MergeSplitAssets(mainPath);
-                        foreach (var i in openFileDialog1.FileNames)
+                        var readFile = ProcessingSplitFiles(openFileDialog1.FileNames.ToList());
+                        foreach (var i in readFile)
                         {
                             unityFiles.Add(i);
                             unityFilesHash.Add(Path.GetFileName(i));
@@ -121,8 +116,9 @@ namespace Unity_Studio
                         //use a for loop because list size can change
                         for (int f = 0; f < unityFiles.Count; f++)
                         {
-                            StatusStripUpdate("Loading " + Path.GetFileName(unityFiles[f]));
+                            var fileName = unityFiles[f];
-                            LoadAssetsFile(unityFiles[f]);
+                            StatusStripUpdate("Loading " + Path.GetFileName(fileName));
+                            LoadAssetsFile(fileName);
                             ProgressBarPerformStep();
                         }
                     }
@@ -135,11 +131,12 @@ namespace Unity_Studio
                             LoadBundleFile(filename);
                             ProgressBarPerformStep();
                         }
-                        LoadAssetsFromBundle();
+                        BuildSharedIndex();
                     }
-                    BuildAssetStrucutres();
                     unityFilesHash.Clear();
                     assetsfileListHash.Clear();
+                    sharedFileIndex.Clear();
+                    BuildAssetStrucutres();
                 });
             }
         }
@@ -149,54 +146,20 @@ namespace Unity_Studio
             var openFolderDialog1 = new OpenFolderDialog();
             if (openFolderDialog1.ShowDialog(this) == DialogResult.OK)
             {
-                mainPath = openFolderDialog1.Folder;
                 resetForm();
-
+                ThreadPool.QueueUserWorkItem(state =>
+                {
+                    mainPath = openFolderDialog1.Folder;
                     MergeSplitAssets(mainPath);
-
+                    var files = Directory.GetFiles(mainPath, "*.*", SearchOption.AllDirectories).ToList();
-                for (int t = 0; t < assetsFileTypes.Length; t++)
+                    var readFile = ProcessingSplitFiles(files);
-                {
+                    foreach (var i in readFile)
-                    string[] fileNames = Directory.GetFiles(mainPath, assetsFileTypes[t], SearchOption.AllDirectories);
-                    #region  sort specific types alphanumerically
-                    if (fileNames.Length > 0 && (t == 1 || t == 2))
-                    {
-                        var sortedList = fileNames.ToList();
-                        sortedList.Sort((s1, s2) =>
-                        {
-                            string pattern = "([A-Za-z\\s]*)([0-9]*)";
-                            string h1 = Regex.Match(Path.GetFileNameWithoutExtension(s1), pattern).Groups[1].Value;
-                            string h2 = Regex.Match(Path.GetFileNameWithoutExtension(s2), pattern).Groups[1].Value;
-                            if (h1 != h2)
-                                return h1.CompareTo(h2);
-                            string t1 = Regex.Match(Path.GetFileNameWithoutExtension(s1), pattern).Groups[2].Value;
-                            string t2 = Regex.Match(Path.GetFileNameWithoutExtension(s2), pattern).Groups[2].Value;
-                            if (t1 != "" && t2 != "")
-                                return int.Parse(t1).CompareTo(int.Parse(t2));
-                            return 0;
-                        });
-                        foreach (var i in sortedList)
                     {
                         unityFiles.Add(i);
                         unityFilesHash.Add(Path.GetFileName(i));
                     }
-
+                    SetProgressBarValue(0);
-                    }
+                    SetProgressBarMaximum(unityFiles.Count);
-                    #endregion
-                    else
-                    {
-                        foreach (var i in fileNames)
-                        {
-                            unityFiles.Add(i);
-                            unityFilesHash.Add(Path.GetFileName(i));
-                        }
-                    }
-                }
-
-                unityFiles = unityFiles.Distinct().ToList();
-                progressBar1.Value = 0;
-                progressBar1.Maximum = unityFiles.Count;
-                ThreadPool.QueueUserWorkItem(delegate
-                {
                     //use a for loop because list size can change
                     for (int f = 0; f < unityFiles.Count; f++)
                     {
@@ -207,6 +170,7 @@ namespace Unity_Studio
                     }
                     unityFilesHash.Clear();
                     assetsfileListHash.Clear();
+                    sharedFileIndex.Clear();
                     BuildAssetStrucutres();
                 });
             }
@@ -240,17 +204,11 @@ namespace Unity_Studio
         private void extractFolderToolStripMenuItem_Click(object sender, EventArgs e)
         {
             int extractedCount = 0;
-            List<string> bundleFiles = new List<string>();
-
             var openFolderDialog1 = new OpenFolderDialog();
             if (openFolderDialog1.ShowDialog(this) == DialogResult.OK)
             {
                 string startPath = openFolderDialog1.Folder;
-                foreach (var fileType in bundleFileTypes)
+                var bundleFiles = Directory.GetFiles(startPath, "*.*", SearchOption.AllDirectories).ToList();
-                {
-                    string[] fileNames = Directory.GetFiles(startPath, fileType, SearchOption.AllDirectories);
-                    bundleFiles.AddRange(fileNames);
-                }
                 progressBar1.Value = 0;
                 progressBar1.Maximum = bundleFiles.Count;
                 ThreadPool.QueueUserWorkItem(delegate
@@ -287,6 +245,7 @@ namespace Unity_Studio
                 if (!dontLoadAssetsMenuItem.Checked)
                 {
                     assetListView.VirtualListSize = visibleAssets.Count;
+                    //will only work if ListView is visible
                     resizeAssetListColumns();
                 }
                 if (!dontBuildHierarchyMenuItem.Checked)
@@ -1427,9 +1386,9 @@ namespace Unity_Studio
                                                 //选中它和它的子节点
                                                 sceneTreeView.Invoke(new Action(() => j.Checked = true));
                                                 //处理非法文件名
-                                                var name = FixFileName(savePath + filename + ".fbx");
+                                                filename = FixFileName(filename);
                                                 //导出FBX
-                                                WriteFBX(name, false);
+                                                WriteFBX(savePath + filename + ".fbx", false);
                                                 //取消选中
                                                 sceneTreeView.Invoke(new Action(() => j.Checked = false));
                                             }