From 47089afdd1b998cf4eaafc24281968bb7d3d926f Mon Sep 17 00:00:00 2001 From: DrakiaXYZ <565558+DrakiaXYZ@users.noreply.github.com> Date: Wed, 22 Oct 2025 22:49:24 -0700 Subject: [PATCH] Further attempt to resolve profile corruption issues (#650) * Further attempt to resolve profile corruption issues - FileUtil now uses File.Replace and does a sync flush - Add restore capabilities to BackupService - If loading a profile fails, attempt to restore from the most recent backup - Trigger a backup creation on raid start, raid end, and game close - Load profiles before starting the backupService to avoid backing up corrupt profiles * - Switch async calls to .GetAwaiter().GetResult() for better exception handling --------- Co-authored-by: DrakiaXYZ <565558+TheDgtl@users.noreply.github.com> --- .../Callbacks/GameCallbacks.cs | 5 ++ .../Callbacks/SaveCallbacks.cs | 4 +- .../Servers/SaveServer.cs | 28 ++++++++++- .../Services/BackupService.cs | 46 ++++++++++++++++++- .../Services/LocationLifecycleService.cs | 8 ++++ .../SPTarkov.Server.Core/Utils/FileUtil.cs | 10 +++- 6 files changed, 97 insertions(+), 4 deletions(-) diff --git a/Libraries/SPTarkov.Server.Core/Callbacks/GameCallbacks.cs b/Libraries/SPTarkov.Server.Core/Callbacks/GameCallbacks.cs index 210686a2..0c970f18 100644 --- a/Libraries/SPTarkov.Server.Core/Callbacks/GameCallbacks.cs +++ b/Libraries/SPTarkov.Server.Core/Callbacks/GameCallbacks.cs @@ -16,6 +16,7 @@ public class GameCallbacks( HttpResponseUtil httpResponseUtil, Watermark watermark, SaveServer saveServer, + BackupService backupService, GameController gameController, ProfileActivityService profileActivityService, TimeUtil timeUtil @@ -66,6 +67,10 @@ public class GameCallbacks( public async ValueTask GameLogout(string url, EmptyRequestData _, MongoId sessionID) { await saveServer.SaveProfileAsync(sessionID); + + // Backup profiles on exit + await backupService.Init(); + return httpResponseUtil.GetBody(new GameLogoutResponseData { Status = "ok" }); } diff --git a/Libraries/SPTarkov.Server.Core/Callbacks/SaveCallbacks.cs b/Libraries/SPTarkov.Server.Core/Callbacks/SaveCallbacks.cs index c897e986..e5d6022c 100644 --- a/Libraries/SPTarkov.Server.Core/Callbacks/SaveCallbacks.cs +++ b/Libraries/SPTarkov.Server.Core/Callbacks/SaveCallbacks.cs @@ -13,8 +13,10 @@ public class SaveCallbacks(SaveServer saveServer, ConfigServer configServer, Bac public async Task OnLoad() { - await backupService.StartBackupSystem(); await saveServer.LoadAsync(); + + // Note: This has to happen after loading the saveServer so we don't backup corrupted profiles + await backupService.StartBackupSystem(); } public async Task OnUpdate(long secondsSinceLastRun) diff --git a/Libraries/SPTarkov.Server.Core/Servers/SaveServer.cs b/Libraries/SPTarkov.Server.Core/Servers/SaveServer.cs index cc92ac56..ff8fb212 100644 --- a/Libraries/SPTarkov.Server.Core/Servers/SaveServer.cs +++ b/Libraries/SPTarkov.Server.Core/Servers/SaveServer.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using System.Diagnostics; +using System.Text.Json; using System.Text.Json.Nodes; using SPTarkov.DI.Annotations; using SPTarkov.Server.Core.DI; @@ -22,6 +23,7 @@ public class SaveServer( HashUtil hashUtil, ServerLocalisationService serverLocalisationService, ProfileValidatorService profileValidatorService, + BackupService backupService, ISptLogger logger, ConfigServer configServer ) @@ -211,7 +213,31 @@ public class SaveServer( if (fileUtil.FileExists(filePath)) // File found, store in profiles[] { - var profile = await jsonUtil.DeserializeFromFileAsync(filePath); + JsonObject? profile = null; + + try + { + profile = await jsonUtil.DeserializeFromFileAsync(filePath); + } + catch (JsonException e) + { + // If the profile fails to deserialize, it may have corrupted, try to restore from a backup + logger.Warning($"Failed loading profile for {sessionID.ToString()}. Attempting to load backup"); + + // We make a copy of the profile before overwriting it, just incase + var corruptBackupPath = $"{profileFilepath}{sessionID.ToString()}-corrupt.json"; + File.Copy(filePath, corruptBackupPath, true); + + if (backupService.RestoreProfile(sessionID)) + { + profile = await jsonUtil.DeserializeFromFileAsync(filePath); + logger.Success("Profile restored from backup!"); + } + else + { + throw new Exception("Failed to restore profile backup", e); + } + } if (profile is not null) { diff --git a/Libraries/SPTarkov.Server.Core/Services/BackupService.cs b/Libraries/SPTarkov.Server.Core/Services/BackupService.cs index 546d43ee..dc44d7eb 100644 --- a/Libraries/SPTarkov.Server.Core/Services/BackupService.cs +++ b/Libraries/SPTarkov.Server.Core/Services/BackupService.cs @@ -13,6 +13,7 @@ namespace SPTarkov.Server.Core.Services; public class BackupService { protected const string ProfileDir = "./user/profiles"; + protected const string activeModsFilename = "activeMods.json"; protected readonly List ActiveServerMods; protected readonly BackupConfig BackupConfig; @@ -131,7 +132,7 @@ public class BackupService } // Write a copy of active mods. - await FileUtil.WriteFileAsync(Path.Combine(targetDir, "activeMods.json"), JsonUtil.Serialize(ActiveServerMods)); + await FileUtil.WriteFileAsync(Path.Combine(targetDir, activeModsFilename), JsonUtil.Serialize(ActiveServerMods)); if (Logger.IsLogEnabled(LogLevel.Debug)) { @@ -223,6 +224,24 @@ public class BackupService return result; } + protected string? GetMostRecentProfileBackup(IEnumerable backupPaths, string profileId) + { + var profileFilename = $"{profileId}.json"; + var backupPathsWithCreationDateTime = GetBackupPathsWithCreationTimestamp(backupPaths); + + foreach (var (backupTimestamp, backupPath) in backupPathsWithCreationDateTime.Reverse()) + { + var profileBackups = FileUtil.GetFiles(backupPath); + var profileBackup = profileBackups.FirstOrDefault(path => path.EndsWith(profileFilename)); + if (profileBackup != null) + { + return profileBackup; + } + } + + return null; + } + /// /// Retrieves and sorts the backup file paths from the specified directory. /// @@ -308,4 +327,29 @@ public class BackupService return result; } + + /// + /// Restores the most recent profile backup for the given profile Id + /// + /// The profile ID of the backup to restore + /// True on success. False on failure + public bool RestoreProfile(string profileId) + { + var backupDir = BackupConfig.Directory; + var backupPaths = GetBackupPaths(backupDir); + var mostRecentBackupForProfile = GetMostRecentProfileBackup(backupPaths, profileId); + + // Verify we have a backup for this profile + if (mostRecentBackupForProfile == null) + { + return false; + } + + // Restore the most recent profile backup + var profileFileName = FileUtil.GetFileNameAndExtension(mostRecentBackupForProfile); + var targetProfilePath = Path.Combine(ProfileDir, profileFileName); + + File.Copy(mostRecentBackupForProfile, targetProfilePath, true); + return true; + } } diff --git a/Libraries/SPTarkov.Server.Core/Services/LocationLifecycleService.cs b/Libraries/SPTarkov.Server.Core/Services/LocationLifecycleService.cs index bc296251..0f8ecda4 100644 --- a/Libraries/SPTarkov.Server.Core/Services/LocationLifecycleService.cs +++ b/Libraries/SPTarkov.Server.Core/Services/LocationLifecycleService.cs @@ -26,6 +26,7 @@ public class LocationLifecycleService( TimeUtil timeUtil, DatabaseService databaseService, ProfileHelper profileHelper, + BackupService backupService, ProfileActivityService profileActivityService, BotNameService botNameService, ICloner cloner, @@ -77,6 +78,9 @@ public class LocationLifecycleService( /// public virtual StartLocalRaidResponseData StartLocalRaid(MongoId sessionId, StartLocalRaidRequestData request) { + // Backup the profile on raid start + backupService.Init().GetAwaiter().GetResult(); + logger.Debug($"Starting: {request.Location}"); var playerProfile = profileHelper.GetFullProfile(sessionId); @@ -409,6 +413,10 @@ public class LocationLifecycleService( HandleCoopExtract(sessionId, pmcProfile, request.Results.ExitName); SendCoopTakenFenceMessage(sessionId); } + + // Save and backup the profile on raid end + saveServer.SaveProfileAsync(sessionId).GetAwaiter().GetResult(); + backupService.Init().GetAwaiter().GetResult(); } /// diff --git a/Libraries/SPTarkov.Server.Core/Utils/FileUtil.cs b/Libraries/SPTarkov.Server.Core/Utils/FileUtil.cs index 8a961dbb..bd317b95 100644 --- a/Libraries/SPTarkov.Server.Core/Utils/FileUtil.cs +++ b/Libraries/SPTarkov.Server.Core/Utils/FileUtil.cs @@ -131,10 +131,18 @@ public class FileUtil // We flush here so we can be sure it's immediately committed to disk await fs.FlushAsync(); + fs.Flush(true); } // Overwrite over the old file - File.Move(tempFilePath, filePath, overwrite: true); + if (File.Exists(filePath)) + { + File.Replace(tempFilePath, filePath, null); + } + else + { + File.Move(tempFilePath, filePath); + } } catch {