From 4239de86fcf0a1cb36db07b88cd8ce316017605b Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 8 Jan 2025 21:10:22 +0000 Subject: [PATCH] parallelize load --- Core/Models/Eft/Common/Tables/Trader.cs | 50 ++++++----- Core/Utils/ImporterUtil.cs | 114 +++++++++++++++--------- Server/Server.csproj | 1 + 3 files changed, 98 insertions(+), 67 deletions(-) diff --git a/Core/Models/Eft/Common/Tables/Trader.cs b/Core/Models/Eft/Common/Tables/Trader.cs index 2648d3b2..4fe0aacb 100644 --- a/Core/Models/Eft/Common/Tables/Trader.cs +++ b/Core/Models/Eft/Common/Tables/Trader.cs @@ -1,6 +1,7 @@ using System.Text.Json.Serialization; using Core.Models.Enums; using Core.Models.Spt.Services; +using Core.Utils.Json.Converters; namespace Core.Models.Eft.Common.Tables; @@ -64,7 +65,7 @@ public class TraderBase public decimal? DiscountEnd { get; set; } [JsonPropertyName("gridHeight")] - public int? GridHeight { get; set; } + public double? GridHeight { get; set; } [JsonPropertyName("sell_modifier_for_prohibited_items")] public decimal? SellModifierForProhibitedItems { get; set; } @@ -100,7 +101,7 @@ public class TraderBase public string? Name { get; set; } [JsonPropertyName("nextResupply")] - public int? NextResupply { get; set; } + public double? NextResupply { get; set; } [JsonPropertyName("nickname")] public string? Nickname { get; set; } @@ -136,43 +137,44 @@ public class TraderInsurance public List? ExcludedCategory { get; set; } [JsonPropertyName("max_return_hour")] - public int? MaxReturnHour { get; set; } + public double? MaxReturnHour { get; set; } [JsonPropertyName("max_storage_time")] - public int? MaxStorageTime { get; set; } + public double? MaxStorageTime { get; set; } [JsonPropertyName("min_payment")] - public int? MinPayment { get; set; } + public double? MinPayment { get; set; } [JsonPropertyName("min_return_hour")] - public int? MinReturnHour { get; set; } + public double? MinReturnHour { get; set; } } public class TraderLoyaltyLevel { [JsonPropertyName("buy_price_coef")] - public int? BuyPriceCoefficient { get; set; } + public double? BuyPriceCoefficient { get; set; } [JsonPropertyName("exchange_price_coef")] - public int? ExchangePriceCoefficient { get; set; } + public double? ExchangePriceCoefficient { get; set; } [JsonPropertyName("heal_price_coef")] - public int? HealPriceCoefficient { get; set; } + public double? HealPriceCoefficient { get; set; } [JsonPropertyName("insurance_price_coef")] - public int? InsurancePriceCoefficient { get; set; } + [JsonConverter(typeof(StringToNumberFactoryConverter))] + public double? InsurancePriceCoefficient { get; set; } [JsonPropertyName("minLevel")] - public int? MinLevel { get; set; } + public double? MinLevel { get; set; } [JsonPropertyName("minSalesSum")] - public int? MinSalesSum { get; set; } + public double? MinSalesSum { get; set; } [JsonPropertyName("minStanding")] - public int? MinStanding { get; set; } + public double? MinStanding { get; set; } [JsonPropertyName("repair_price_coef")] - public int? RepairPriceCoefficient { get; set; } + public double? RepairPriceCoefficient { get; set; } } public class TraderRepair @@ -193,13 +195,14 @@ public class TraderRepair public List? ExcludedIdList { get; set; } // Doesn't exist in client object [JsonPropertyName("quality")] - public int? Quality { get; set; } + [JsonConverter(typeof(StringToNumberFactoryConverter))] + public double? Quality { get; set; } } public class TraderAssort { [JsonPropertyName("nextResupply")] - public int? NextResupply { get; set; } + public double? NextResupply { get; set; } [JsonPropertyName("items")] public List? Items { get; set; } @@ -214,7 +217,7 @@ public class TraderAssort public class BarterScheme { [JsonPropertyName("count")] - public int? Count { get; set; } + public double? Count { get; set; } [JsonPropertyName("_tpl")] public string? Template { get; set; } @@ -226,9 +229,10 @@ public class BarterScheme public bool? SptQuestLocked { get; set; } [JsonPropertyName("level")] - public int? Level { get; set; } + public double? Level { get; set; } [JsonPropertyName("side")] + [JsonConverter(typeof(JsonStringEnumConverter))] public DogtagExchangeSide? Side { get; set; } } @@ -265,13 +269,13 @@ public class SuitRequirements public List? AchievementRequirements { get; set; } [JsonPropertyName("loyaltyLevel")] - public int? LoyaltyLevel { get; set; } + public double? LoyaltyLevel { get; set; } [JsonPropertyName("profileLevel")] - public int? ProfileLevel { get; set; } + public double? ProfileLevel { get; set; } [JsonPropertyName("standing")] - public int? Standing { get; set; } + public double? Standing { get; set; } [JsonPropertyName("skillRequirements")] public List? SkillRequirements { get; set; } @@ -289,7 +293,7 @@ public class SuitRequirements public class ItemRequirement { [JsonPropertyName("count")] - public int? Count { get; set; } + public double? Count { get; set; } [JsonPropertyName("_tpl")] public string? Tpl { get; set; } @@ -302,4 +306,4 @@ public class ItemRequirement [JsonPropertyName("type")] public string? Type { get; set; } -} \ No newline at end of file +} diff --git a/Core/Utils/ImporterUtil.cs b/Core/Utils/ImporterUtil.cs index 2d667f8a..0d5f3112 100644 --- a/Core/Utils/ImporterUtil.cs +++ b/Core/Utils/ImporterUtil.cs @@ -21,13 +21,14 @@ public class ImporterUtil * @param filepath Path to folder with files * @returns Promise return T type associated with this class */ - public async Task LoadRecursiveAsync( + public Task LoadRecursiveAsync( string filepath, Type loadedType, Action? onReadCallback = null, Action? onObjectDeserialized = null ) { + var tasks = new List(); var result = Activator.CreateInstance(loadedType); // get all filepaths @@ -40,58 +41,83 @@ public class ImporterUtil if (_fileUtil.GetFileExtension(file) != "json") continue; // to skip ArchivedQuests.json if (file.ToLower().Contains("archived")) continue; - // const filename = this.vfs.stripExtension(file); - // const filePathAndName = `${filepath}${file}`; - var fileData = await File.ReadAllTextAsync(file); - if (onReadCallback != null) - onReadCallback(file, fileData); + tasks.Add( + Task.Factory.StartNew(() => + { + // const filename = this.vfs.stripExtension(file); + // const filePathAndName = `${filepath}${file}`; + var fileData = File.ReadAllText(file); + if (onReadCallback != null) + onReadCallback(file, fileData); - Type propertyType; - MethodInfo setMethod; - var isDictionary = false; - if (loadedType.IsGenericType && loadedType.GetGenericTypeDefinition() == typeof(Dictionary<,>)) - { - propertyType = loadedType.GetGenericArguments()[1]; - setMethod = loadedType.GetMethod("Add"); - isDictionary = true; - } - else - { - var matchedProperty = loadedType.GetProperties() - .FirstOrDefault(prop => prop.Name.ToLower() == Path.GetFileNameWithoutExtension(file).ToLower()); - if (matchedProperty == null) - throw new Exception($"Unable to find property '{Path.GetFileNameWithoutExtension(file)}' for type '{loadedType.Name}'"); - propertyType = matchedProperty.PropertyType; - setMethod = matchedProperty.GetSetMethod(); - } + var setMethod = GetSetMethod( + Path.GetFileNameWithoutExtension(file).ToLower(), + loadedType, + out var propertyType, + out var isDictionary + ); + try + { + var fileDeserialized = JsonSerializer.Deserialize(fileData, propertyType, + new JsonSerializerOptions { UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow }); + if (onObjectDeserialized != null) + onObjectDeserialized(file, fileDeserialized); - try - { - var fileDeserialized = JsonSerializer.Deserialize(fileData, propertyType, - new JsonSerializerOptions { UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow }); - if (onObjectDeserialized != null) - onObjectDeserialized(file, fileDeserialized); - - setMethod.Invoke(result, isDictionary ? [Path.GetFileNameWithoutExtension(file), fileDeserialized] : [fileDeserialized]); - } - catch (Exception e) - { - throw new Exception($"Unable to deserialize or set properties for file '{file}'", e); - } + setMethod.Invoke(result, + isDictionary ? [Path.GetFileNameWithoutExtension(file), fileDeserialized] : [fileDeserialized]); + } + catch (Exception e) + { + throw new Exception($"Unable to deserialize or set properties for file '{file}'", e); + } + }) + ); } // deep tree search foreach (var directory in directories) { - var matchedProperty = loadedType.GetProperties() - .FirstOrDefault(prop => prop.Name.ToLower() == directory.Split("/").Last().Replace("_", "").ToLower()); - if (matchedProperty == null) - throw new Exception($"Unable to find property '{directory}' for type '{loadedType.Name}'"); - matchedProperty.GetSetMethod().Invoke(result, [await LoadRecursiveAsync($"{directory}/", matchedProperty.PropertyType)]); + tasks.Add( + Task.Factory.StartNew(() => + { + var setMethod = GetSetMethod(directory.Split("/").Last().Replace("_", ""), loadedType, out var matchedProperty, out var isDictionary); + var loadTask = LoadRecursiveAsync($"{directory}/", matchedProperty); + loadTask.Wait(); + setMethod.Invoke(result, isDictionary ? [directory, loadTask.Result] : [loadTask.Result]); + }) + ); } // return the result of all async fetch - return result; + return Task.WhenAll(tasks).ContinueWith((t) => + { + if (t.IsCanceled || t.IsFaulted) + tasks.Where(t => t.IsFaulted || t.IsCanceled).ToList().ForEach(t => Console.WriteLine(t.Exception)); + }).ContinueWith(_ => result); + } + + public MethodInfo GetSetMethod(string propertyName, Type type, out Type propertyType, out bool isDictionary) + { + MethodInfo setMethod; + isDictionary = false; + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Dictionary<,>)) + { + propertyType = type.GetGenericArguments()[1]; + setMethod = type.GetMethod("Add"); + isDictionary = true; + } + else + { + var matchedProperty = type.GetProperties() + .FirstOrDefault(prop => + prop.Name.ToLower() == Path.GetFileNameWithoutExtension(propertyName).ToLower()); + if (matchedProperty == null) + throw new Exception( + $"Unable to find property '{Path.GetFileNameWithoutExtension(propertyName)}' for type '{type.Name}'"); + propertyType = matchedProperty.PropertyType; + setMethod = matchedProperty.GetSetMethod(); + } + return setMethod; } /** @@ -204,4 +230,4 @@ public class ImporterUtil } } }*/ -} \ No newline at end of file +} diff --git a/Server/Server.csproj b/Server/Server.csproj index fbd00ca6..f1645c7e 100644 --- a/Server/Server.csproj +++ b/Server/Server.csproj @@ -1,6 +1,7 @@  + true false net9.0 enable