From d3990c12192117eaf615bf59025ba30948950c11 Mon Sep 17 00:00:00 2001 From: Archangel Date: Fri, 14 Feb 2025 12:52:18 +0100 Subject: [PATCH] Use file streams to deserialize files - Improves load speed - Lowers memory overhead --- Libraries/Core/Helpers/ModHelper.cs | 2 +- Libraries/Core/Servers/ConfigServer.cs | 3 +- Libraries/Core/Servers/SaveServer.cs | 2 +- Libraries/Core/Services/I18nService.cs | 2 +- Libraries/Core/Utils/DatabaseImporter.cs | 4 +- Libraries/Core/Utils/ImporterUtil.cs | 86 +++++++++--------------- Libraries/Core/Utils/JsonUtil.cs | 48 +++++++++++++ 7 files changed, 87 insertions(+), 60 deletions(-) diff --git a/Libraries/Core/Helpers/ModHelper.cs b/Libraries/Core/Helpers/ModHelper.cs index e2ac3324..76f13aa4 100644 --- a/Libraries/Core/Helpers/ModHelper.cs +++ b/Libraries/Core/Helpers/ModHelper.cs @@ -30,7 +30,7 @@ namespace Core.Helpers var rawContent = _fileUtil.ReadFile(Path.Combine(pathToFile, fileName)); // Take the string above and deserialise it into a file with a type (defined between the diamond brackets) - return _jsonUtil.Deserialize(rawContent); + return _jsonUtil.DeserializeFromFile(rawContent); } } } diff --git a/Libraries/Core/Servers/ConfigServer.cs b/Libraries/Core/Servers/ConfigServer.cs index c7a64139..f6644a37 100644 --- a/Libraries/Core/Servers/ConfigServer.cs +++ b/Libraries/Core/Servers/ConfigServer.cs @@ -71,9 +71,8 @@ public class ConfigServer { if (acceptableFileExtensions.Contains(_fileUtil.GetFileExtension(file))) { - var fileContent = _fileUtil.ReadFile(file); var type = GetConfigTypeByFilename(file); - var deserializedContent = _jsonUtil.Deserialize(fileContent, type); + var deserializedContent = _jsonUtil.DeserializeFromFile(file, type); if (deserializedContent == null) { diff --git a/Libraries/Core/Servers/SaveServer.cs b/Libraries/Core/Servers/SaveServer.cs index d6a97d23..8197b2f4 100644 --- a/Libraries/Core/Servers/SaveServer.cs +++ b/Libraries/Core/Servers/SaveServer.cs @@ -216,7 +216,7 @@ public class SaveServer( if (_fileUtil.FileExists(filePath)) // File found, store in profiles[] { - profiles[sessionID] = _jsonUtil.Deserialize(_fileUtil.ReadFile(filePath)); + profiles[sessionID] = _jsonUtil.DeserializeFromFile(filePath); } // Run callbacks diff --git a/Libraries/Core/Services/I18nService.cs b/Libraries/Core/Services/I18nService.cs index 87d87ddf..aa284964 100644 --- a/Libraries/Core/Services/I18nService.cs +++ b/Libraries/Core/Services/I18nService.cs @@ -48,7 +48,7 @@ public class I18nService _loadedLocales.Add( _fileUtil.StripExtension(file), new LazyLoad>( - () => _jsonUtil.Deserialize>(_fileUtil.ReadFile(file)) ?? + () => _jsonUtil.DeserializeFromFile>(file) ?? new Dictionary() ) ); diff --git a/Libraries/Core/Utils/DatabaseImporter.cs b/Libraries/Core/Utils/DatabaseImporter.cs index 62b157c5..e50828ba 100644 --- a/Libraries/Core/Utils/DatabaseImporter.cs +++ b/Libraries/Core/Utils/DatabaseImporter.cs @@ -172,8 +172,10 @@ public class DatabaseImporter : IOnLoad _databaseServer.SetTables(dataToImport); } - protected void OnReadValidate(string fileWithPath, string data) + protected void OnReadValidate(string fileWithPath) { + + // Validate files //if (ProgramStatics.COMPILED && hashedFile && !ValidateFile(fileWithPath, data)) { // this.valid = ValidationResult.FAILED; diff --git a/Libraries/Core/Utils/ImporterUtil.cs b/Libraries/Core/Utils/ImporterUtil.cs index ffce3e58..b2d2fa03 100644 --- a/Libraries/Core/Utils/ImporterUtil.cs +++ b/Libraries/Core/Utils/ImporterUtil.cs @@ -28,7 +28,7 @@ public class ImporterUtil public Task LoadRecursiveAsync( string filepath, - Action? onReadCallback = null, + Action? onReadCallback = null, Action? onObjectDeserialized = null ) { @@ -45,7 +45,7 @@ public class ImporterUtil protected async Task LoadRecursiveAsync( string filepath, Type loadedType, - Action? onReadCallback = null, + Action? onReadCallback = null, Action? onObjectDeserialized = null ) { @@ -88,7 +88,7 @@ public class ImporterUtil private async Task ProcessFileAsync( string file, Type loadedType, - Action? onReadCallback, + Action? onReadCallback, Action? onObjectDeserialized, object result, object dictionaryLock @@ -96,28 +96,31 @@ public class ImporterUtil { try { - var fileData = _fileUtil.ReadFile(file); - onReadCallback?.Invoke(file, fileData); - - var setMethod = GetSetMethod( - _fileUtil.StripExtension(file).ToLower(), - loadedType, - out var propertyType, - out var isDictionary - ); - - var fileDeserialized = await DeserializeFileAsync(fileData, propertyType); - - onObjectDeserialized?.Invoke(file, fileDeserialized); - - lock (dictionaryLock) + using (var fs = new FileStream(file, FileMode.Open, FileAccess.Read)) { - setMethod.Invoke( - result, - isDictionary - ? [_fileUtil.StripExtension(file), fileDeserialized] - : new object[] { fileDeserialized } + onReadCallback?.Invoke(file); + + // Get the set method to update the object + var setMethod = GetSetMethod( + _fileUtil.StripExtension(file).ToLower(), + loadedType, + out var propertyType, + out var isDictionary ); + + var fileDeserialized = await DeserializeFileAsync(fs, file, propertyType); + + onObjectDeserialized?.Invoke(file, fileDeserialized); + + lock (dictionaryLock) + { + setMethod.Invoke( + result, + isDictionary + ? [_fileUtil.StripExtension(file), fileDeserialized] + : new object[] { fileDeserialized } + ); + } } } catch (Exception ex) @@ -155,50 +158,25 @@ public class ImporterUtil } } - private async Task DeserializeFileAsync(string fileData, Type propertyType) + private async Task DeserializeFileAsync(FileStream fs, string file, Type propertyType) { if (propertyType.IsGenericType && propertyType.GetGenericTypeDefinition() == typeof(LazyLoad<>)) { - return CreateLazyLoadDeserialization(fileData, propertyType); + return CreateLazyLoadDeserialization(file, propertyType); } - return await Task.Run(() => _jsonUtil.Deserialize(fileData, propertyType)); + return await Task.Run(() => _jsonUtil.DeserializeFromFileStream(fs, propertyType)); } - private object CreateLazyLoadDeserialization(string fileData, Type propertyType) + private object CreateLazyLoadDeserialization(string file, Type propertyType) { var genericArgument = propertyType.GetGenericArguments()[0]; - /* - if (!lazyLoadDeserializationCache.TryGetValue(genericArgument, out var cachedDelegate)) - { - // Create the expression for deserialization - var deserializeCall = Expression.Call( - Expression.Constant(_jsonUtil), - "Deserialize", - Type.EmptyTypes, - Expression.Constant(fileData), - Expression.Constant(genericArgument) - ); - - var typeAsExpression = Expression.TypeAs(deserializeCall, genericArgument); - - var expression = Expression.Lambda( - typeof(Func<>).MakeGenericType(genericArgument), - typeAsExpression - ); - - // Compile the expression and store it in the cache - cachedDelegate = expression.Compile(); - lazyLoadDeserializationCache.TryAdd(genericArgument, cachedDelegate); - } - */ - var deserializeCall = Expression.Call( Expression.Constant(_jsonUtil), - "Deserialize", + "DeserializeFromFile", Type.EmptyTypes, - Expression.Constant(fileData), + Expression.Constant(file), Expression.Constant(genericArgument) ); diff --git a/Libraries/Core/Utils/JsonUtil.cs b/Libraries/Core/Utils/JsonUtil.cs index b39c87e5..e1e3a025 100644 --- a/Libraries/Core/Utils/JsonUtil.cs +++ b/Libraries/Core/Utils/JsonUtil.cs @@ -83,6 +83,54 @@ public class JsonUtil return string.IsNullOrEmpty(json) ? null : JsonSerializer.Deserialize(json, type, jsonSerializerOptionsNoIndent); } + /// + /// Convert JSON into an object from a file + /// + /// The JSON File to read + /// + public T? DeserializeFromFile(string file) + { + if (!File.Exists(file)) + { + return default; + } + + using (FileStream fs = new(file, FileMode.Open, FileAccess.Read)) + { + return JsonSerializer.Deserialize(fs, jsonSerializerOptionsNoIndent); + } + } + + /// + /// Convert JSON into an object from a file + /// + /// The JSON File to read + /// The type of the object to deserialize to + /// + public object? DeserializeFromFile(string file, Type type) + { + if (!File.Exists(file)) + { + return default; + } + + using (FileStream fs = new(file, FileMode.Open, FileAccess.Read)) + { + return JsonSerializer.Deserialize(fs, type, jsonSerializerOptionsNoIndent); + } + } + + /// + /// Convert JSON into an object from a FileStream + /// + /// The file stream to deserialize + /// The type of the object to deserialize to + /// + public object? DeserializeFromFileStream(FileStream fs, Type type) + { + return JsonSerializer.Deserialize(fs, type, jsonSerializerOptionsNoIndent); + } + /// /// Convert an object into JSON ///