Improve server bundle loading

Speeds up SPT server initialization and reduces allocations when a lot of bundle mods are active

Libraries/SPTarkov.Server.Core/Loaders/BundleLoader.cs

@@ -1,4 +1,5 @@
-using System.Text.Json.Serialization;
+using System.Collections.Concurrent;
+using System.Text.Json.Serialization;
 using SPTarkov.DI.Annotations;
 using SPTarkov.Server.Core.Extensions;
 using SPTarkov.Server.Core.Models.Spt.Mod;
@@ -35,37 +36,43 @@ public class BundleInfo(string modPath, BundleManifestEntry bundle, uint bundleH
 [Injectable(InjectionType.Singleton)]
 public class BundleLoader(ISptLogger<BundleLoader> logger, JsonUtil jsonUtil, BundleHashCacheService bundleHashCacheService)
 {
-    private readonly Dictionary<string, BundleInfo> _bundles = [];
+    private readonly ConcurrentDictionary<string, BundleInfo> _bundles = [];
 
     public async Task LoadBundlesAsync(SptMod mod)
     {
         await bundleHashCacheService.HydrateCache();
 
         var modPath = mod.GetModPath();
-
         var modBundles = await jsonUtil.DeserializeFromFileAsync<BundleManifest>(
             Path.Join(Directory.GetCurrentDirectory(), modPath, "bundles.json")
         );
 
-        var bundleManifests = modBundles?.Manifest ?? [];
+        var relativeModPath = modPath.Replace('\\', '/');
+        var bundlesPath = Path.Join(relativeModPath, "bundles");
 
-        foreach (var bundleManifest in bundleManifests)
+        if (modBundles?.Manifest is null)
         {
-            var relativeModPath = modPath.Replace('\\', '/');
-
-            var bundleLocalPath = Path.Join(relativeModPath, "bundles", bundleManifest.Key).Replace('\\', '/');
-
-            if (!File.Exists(bundleLocalPath))
-            {
-                logger.Warning($"Could not find bundle {bundleManifest.Key} for mod {mod.ModMetadata.Name}");
-                continue;
-            }
-
-            var bundleHash = await bundleHashCacheService.CalculateMatchAndStoreHash(bundleLocalPath);
-
-            AddBundle(bundleManifest.Key, new BundleInfo(relativeModPath, bundleManifest, bundleHash));
+            logger.Warning($"Could not find manifest for mod {mod.ModMetadata.Name}, skipping!");
+            return;
         }
 
+        await Parallel.ForEachAsync(
+            modBundles.Manifest,
+            async (bundleManifest, ct) =>
+            {
+                var bundleLocalPath = Path.Join(bundlesPath, bundleManifest.Key).Replace('\\', '/');
+
+                if (!File.Exists(bundleLocalPath))
+                {
+                    logger.Warning($"Could not find bundle {bundleManifest.Key} for mod {mod.ModMetadata.Name}");
+                    return;
+                }
+
+                var bundleHash = await bundleHashCacheService.CalculateMatchAndStoreHash(bundleLocalPath);
+                AddBundle(bundleManifest.Key, new BundleInfo(relativeModPath, bundleManifest, bundleHash));
+            }
+        );
+
         await bundleHashCacheService.WriteCache();
     }
 

Libraries/SPTarkov.Server.Core/Services/BundleHashCacheService.cs

@@ -1,5 +1,4 @@
 using SPTarkov.DI.Annotations;
-using SPTarkov.Server.Core.Models.Eft.Profile;
 using SPTarkov.Server.Core.Models.Utils;
 using SPTarkov.Server.Core.Utils;
 
@@ -79,7 +78,7 @@ public class BundleHashCacheService(ISptLogger<BundleHashCacheService> logger, J
 
         if (!MatchWithStoredHash(BundlePath, hash))
         {
-            await StoreValue(BundlePath, await CalculateHash(BundlePath));
+            await StoreValue(BundlePath, hash);
         }
 
         return hash;
@@ -87,7 +86,7 @@ public class BundleHashCacheService(ISptLogger<BundleHashCacheService> logger, J
 
     protected async Task<uint> CalculateHash(string BundlePath)
     {
-        return hashUtil.GenerateCrc32ForData(await fileUtil.ReadFileAsBytesAsync(BundlePath));
+        return await hashUtil.GenerateCrc32ForFileAsync(BundlePath);
     }
 
     protected bool MatchWithStoredHash(string BundlePath, uint hash)

Libraries/SPTarkov.Server.Core/Utils/HashUtil.cs

@@ -1,3 +1,4 @@
+using System.Buffers;
 using System.IO.Hashing;
 using System.Security.Cryptography;
 using System.Text;
@@ -18,6 +19,34 @@ public class HashUtil(RandomUtil _randomUtil)
         return Crc32.HashToUInt32(data);
     }
 
+    /// <summary>
+    /// Generates a CRC32 hash for a file, reading in chunks using a pooled buffer to reduce allocations.
+    /// </summary>
+    /// <param name="filePath">The path to the file</param>
+    /// <returns>The CRC32 hash as a uint</returns>
+    public async Task<uint> GenerateCrc32ForFileAsync(string filePath)
+    {
+        var crc = new Crc32();
+        await using var stream = File.OpenRead(filePath);
+
+        // Rent from pool to avoid repeated allocations for each file read
+        var buffer = ArrayPool<byte>.Shared.Rent(81920);
+        try
+        {
+            int bytesRead;
+            while ((bytesRead = await stream.ReadAsync(buffer)) > 0)
+            {
+                crc.Append(buffer.AsSpan(0, bytesRead));
+            }
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(buffer);
+        }
+
+        return crc.GetCurrentHashAsUInt32();
+    }
+
     /// <summary>
     ///     Create a hash for the data parameter
     /// </summary>