Use file streams to deserialize files

- Improves load speed
- Lowers memory overhead

Libraries/Core/Helpers/ModHelper.cs

@@ -30,7 +30,7 @@ namespace Core.Helpers
             var rawContent = _fileUtil.ReadFile(Path.Combine(pathToFile, fileName));
 
             // Take the string above and deserialise it into a file with a type (defined between the diamond brackets)
-            return _jsonUtil.Deserialize<T>(rawContent);
+            return _jsonUtil.DeserializeFromFile<T>(rawContent);
         }
     }
 }

Libraries/Core/Servers/ConfigServer.cs

@@ -71,9 +71,8 @@ public class ConfigServer
         {
             if (acceptableFileExtensions.Contains(_fileUtil.GetFileExtension(file)))
             {
-                var fileContent = _fileUtil.ReadFile(file);
                 var type = GetConfigTypeByFilename(file);
-                var deserializedContent = _jsonUtil.Deserialize(fileContent, type);
+                var deserializedContent = _jsonUtil.DeserializeFromFile(file, type);
 
                 if (deserializedContent == null)
                 {

Libraries/Core/Servers/SaveServer.cs

@@ -216,7 +216,7 @@ public class SaveServer(
             if (_fileUtil.FileExists(filePath))
                 // File found, store in profiles[]
             {
-                profiles[sessionID] = _jsonUtil.Deserialize<SptProfile>(_fileUtil.ReadFile(filePath));
+                profiles[sessionID] = _jsonUtil.DeserializeFromFile<SptProfile>(filePath);
             }
 
             // Run callbacks

Libraries/Core/Services/I18nService.cs

@@ -48,7 +48,7 @@ public class I18nService
             _loadedLocales.Add(
                 _fileUtil.StripExtension(file),
                 new LazyLoad<Dictionary<string, string>>(
-                    () => _jsonUtil.Deserialize<Dictionary<string, string>>(_fileUtil.ReadFile(file)) ??
+                    () => _jsonUtil.DeserializeFromFile<Dictionary<string, string>>(file) ??
                           new Dictionary<string, string>()
                 )
             );

Libraries/Core/Utils/DatabaseImporter.cs

@@ -172,8 +172,10 @@ public class DatabaseImporter : IOnLoad
         _databaseServer.SetTables(dataToImport);
     }
 
-    protected void OnReadValidate(string fileWithPath, string data)
+    protected void OnReadValidate(string fileWithPath)
     {
+
+
         // Validate files
         //if (ProgramStatics.COMPILED && hashedFile && !ValidateFile(fileWithPath, data)) {
         //    this.valid = ValidationResult.FAILED;

Libraries/Core/Utils/ImporterUtil.cs

@@ -28,7 +28,7 @@ public class ImporterUtil
 
     public Task<T> LoadRecursiveAsync<T>(
         string filepath,
-        Action<string, string>? onReadCallback = null,
+        Action<string>? onReadCallback = null,
         Action<string, object>? onObjectDeserialized = null
     )
     {
@@ -45,7 +45,7 @@ public class ImporterUtil
     protected async Task<object> LoadRecursiveAsync(
         string filepath,
         Type loadedType,
-        Action<string, string>? onReadCallback = null,
+        Action<string>? onReadCallback = null,
         Action<string, object>? onObjectDeserialized = null
     )
     {
@@ -88,7 +88,7 @@ public class ImporterUtil
     private async Task ProcessFileAsync(
         string file,
         Type loadedType,
-        Action<string, string>? onReadCallback,
+        Action<string>? onReadCallback,
         Action<string, object>? onObjectDeserialized,
         object result,
         object dictionaryLock
@@ -96,28 +96,31 @@ public class ImporterUtil
     {
         try
         {
-            var fileData = _fileUtil.ReadFile(file);
-            onReadCallback?.Invoke(file, fileData);
-
-            var setMethod = GetSetMethod(
-                _fileUtil.StripExtension(file).ToLower(),
-                loadedType,
-                out var propertyType,
-                out var isDictionary
-            );
-
-            var fileDeserialized = await DeserializeFileAsync(fileData, propertyType);
-
-            onObjectDeserialized?.Invoke(file, fileDeserialized);
-
-            lock (dictionaryLock)
+            using (var fs = new FileStream(file, FileMode.Open, FileAccess.Read))
             {
-                setMethod.Invoke(
-                    result,
-                    isDictionary
-                        ? [_fileUtil.StripExtension(file), fileDeserialized]
-                        : new object[] { fileDeserialized }
+                onReadCallback?.Invoke(file);
+
+                // Get the set method to update the object
+                var setMethod = GetSetMethod(
+                    _fileUtil.StripExtension(file).ToLower(),
+                    loadedType,
+                    out var propertyType,
+                    out var isDictionary
                 );
+
+                var fileDeserialized = await DeserializeFileAsync(fs, file, propertyType);
+
+                onObjectDeserialized?.Invoke(file, fileDeserialized);
+
+                lock (dictionaryLock)
+                {
+                    setMethod.Invoke(
+                        result,
+                        isDictionary
+                            ? [_fileUtil.StripExtension(file), fileDeserialized]
+                            : new object[] { fileDeserialized }
+                    );
+                }
             }
         }
         catch (Exception ex)
@@ -155,50 +158,25 @@ public class ImporterUtil
         }
     }
 
-    private async Task<object> DeserializeFileAsync(string fileData, Type propertyType)
+    private async Task<object> DeserializeFileAsync(FileStream fs, string file, Type propertyType)
     {
         if (propertyType.IsGenericType && propertyType.GetGenericTypeDefinition() == typeof(LazyLoad<>))
         {
-            return CreateLazyLoadDeserialization(fileData, propertyType);
+            return CreateLazyLoadDeserialization(file, propertyType);
         }
 
-        return await Task.Run(() => _jsonUtil.Deserialize(fileData, propertyType));
+        return await Task.Run(() => _jsonUtil.DeserializeFromFileStream(fs, propertyType));
     }
 
-    private object CreateLazyLoadDeserialization(string fileData, Type propertyType)
+    private object CreateLazyLoadDeserialization(string file, Type propertyType)
     {
         var genericArgument = propertyType.GetGenericArguments()[0];
 
-        /*
-        if (!lazyLoadDeserializationCache.TryGetValue(genericArgument, out var cachedDelegate))
-        {
-            // Create the expression for deserialization
-            var deserializeCall = Expression.Call(
-                Expression.Constant(_jsonUtil),
-                "Deserialize",
-                Type.EmptyTypes,
-                Expression.Constant(fileData),
-                Expression.Constant(genericArgument)
-            );
-
-            var typeAsExpression = Expression.TypeAs(deserializeCall, genericArgument);
-
-            var expression = Expression.Lambda(
-                typeof(Func<>).MakeGenericType(genericArgument),
-                typeAsExpression
-            );
-
-            // Compile the expression and store it in the cache
-            cachedDelegate = expression.Compile();
-            lazyLoadDeserializationCache.TryAdd(genericArgument, cachedDelegate);
-        }
-        */
-
         var deserializeCall = Expression.Call(
                 Expression.Constant(_jsonUtil),
-                "Deserialize",
+                "DeserializeFromFile",
                 Type.EmptyTypes,
-                Expression.Constant(fileData),
+                Expression.Constant(file),
                 Expression.Constant(genericArgument)
             );
 

Libraries/Core/Utils/JsonUtil.cs

@@ -83,6 +83,54 @@ public class JsonUtil
         return string.IsNullOrEmpty(json) ? null : JsonSerializer.Deserialize(json, type, jsonSerializerOptionsNoIndent);
     }
 
+    /// <summary>
+    ///     Convert JSON into an object from a file
+    /// </summary>
+    /// <param name="file">The JSON File to read</param>
+    /// <returns></returns>
+    public T? DeserializeFromFile<T>(string file)
+    {
+        if (!File.Exists(file))
+        {
+            return default;
+        }
+
+        using (FileStream fs = new(file, FileMode.Open, FileAccess.Read))
+        {
+            return JsonSerializer.Deserialize<T>(fs, jsonSerializerOptionsNoIndent);
+        }
+    }
+
+    /// <summary>
+    ///     Convert JSON into an object from a file
+    /// </summary>
+    /// <param name="file">The JSON File to read</param>
+    /// <param name="type">The type of the object to deserialize to</param>
+    /// <returns></returns>
+    public object? DeserializeFromFile(string file, Type type)
+    {
+        if (!File.Exists(file))
+        {
+            return default;
+        }
+
+        using (FileStream fs = new(file, FileMode.Open, FileAccess.Read))
+        {
+            return JsonSerializer.Deserialize(fs, type, jsonSerializerOptionsNoIndent);
+        }
+    }
+
+    /// <summary>
+    ///     Convert JSON into an object from a FileStream
+    /// </summary>
+    /// <param name="fs">The file stream to deserialize</param>
+    /// <param name="type">The type of the object to deserialize to</param>
+    /// <returns></returns>
+    public object? DeserializeFromFileStream(FileStream fs, Type type)
+    {
+        return JsonSerializer.Deserialize(fs, type, jsonSerializerOptionsNoIndent);
+    }
+
     /// <summary>
     ///     Convert an object into JSON
     /// </summary>