Further attempt to resolve profile corruption issues (#650)

* Further attempt to resolve profile corruption issues
- FileUtil now uses File.Replace and does a sync flush
- Add restore capabilities to BackupService
- If loading a profile fails, attempt to restore from the most recent backup
- Trigger a backup creation on raid start, raid end, and game close
- Load profiles before starting the backupService to avoid backing up corrupt profiles

* - Switch async calls to .GetAwaiter().GetResult() for better exception handling

---------

Co-authored-by: DrakiaXYZ <565558+TheDgtl@users.noreply.github.com>

Libraries/SPTarkov.Server.Core/Callbacks/GameCallbacks.cs

@@ -16,6 +16,7 @@ public class GameCallbacks(
     HttpResponseUtil httpResponseUtil,
     Watermark watermark,
     SaveServer saveServer,
+    BackupService backupService,
     GameController gameController,
     ProfileActivityService profileActivityService,
     TimeUtil timeUtil
@@ -66,6 +67,10 @@ public class GameCallbacks(
     public async ValueTask<string> GameLogout(string url, EmptyRequestData _, MongoId sessionID)
     {
         await saveServer.SaveProfileAsync(sessionID);
+
+        // Backup profiles on exit
+        await backupService.Init();
+
         return httpResponseUtil.GetBody(new GameLogoutResponseData { Status = "ok" });
     }
 

Libraries/SPTarkov.Server.Core/Callbacks/SaveCallbacks.cs

@@ -13,8 +13,10 @@ public class SaveCallbacks(SaveServer saveServer, ConfigServer configServer, Bac
 
     public async Task OnLoad()
     {
-        await backupService.StartBackupSystem();
         await saveServer.LoadAsync();
+
+        // Note: This has to happen after loading the saveServer so we don't backup corrupted profiles
+        await backupService.StartBackupSystem();
     }
 
     public async Task<bool> OnUpdate(long secondsSinceLastRun)

Libraries/SPTarkov.Server.Core/Servers/SaveServer.cs

@@ -1,5 +1,6 @@
 using System.Collections.Concurrent;
 using System.Diagnostics;
+using System.Text.Json;
 using System.Text.Json.Nodes;
 using SPTarkov.DI.Annotations;
 using SPTarkov.Server.Core.DI;
@@ -22,6 +23,7 @@ public class SaveServer(
     HashUtil hashUtil,
     ServerLocalisationService serverLocalisationService,
     ProfileValidatorService profileValidatorService,
+    BackupService backupService,
     ISptLogger<SaveServer> logger,
     ConfigServer configServer
 )
@@ -211,7 +213,31 @@ public class SaveServer(
         if (fileUtil.FileExists(filePath))
         // File found, store in profiles[]
         {
-            var profile = await jsonUtil.DeserializeFromFileAsync<JsonObject>(filePath);
+            JsonObject? profile = null;
+
+            try
+            {
+                profile = await jsonUtil.DeserializeFromFileAsync<JsonObject>(filePath);
+            }
+            catch (JsonException e)
+            {
+                // If the profile fails to deserialize, it may have corrupted, try to restore from a backup
+                logger.Warning($"Failed loading profile for {sessionID.ToString()}. Attempting to load backup");
+
+                // We make a copy of the profile before overwriting it, just incase
+                var corruptBackupPath = $"{profileFilepath}{sessionID.ToString()}-corrupt.json";
+                File.Copy(filePath, corruptBackupPath, true);
+
+                if (backupService.RestoreProfile(sessionID))
+                {
+                    profile = await jsonUtil.DeserializeFromFileAsync<JsonObject>(filePath);
+                    logger.Success("Profile restored from backup!");
+                }
+                else
+                {
+                    throw new Exception("Failed to restore profile backup", e);
+                }
+            }
 
             if (profile is not null)
             {

Libraries/SPTarkov.Server.Core/Services/BackupService.cs

@@ -13,6 +13,7 @@ namespace SPTarkov.Server.Core.Services;
 public class BackupService
 {
     protected const string ProfileDir = "./user/profiles";
+    protected const string activeModsFilename = "activeMods.json";
 
     protected readonly List<string> ActiveServerMods;
     protected readonly BackupConfig BackupConfig;
@@ -131,7 +132,7 @@ public class BackupService
             }
 
             // Write a copy of active mods.
-            await FileUtil.WriteFileAsync(Path.Combine(targetDir, "activeMods.json"), JsonUtil.Serialize(ActiveServerMods));
+            await FileUtil.WriteFileAsync(Path.Combine(targetDir, activeModsFilename), JsonUtil.Serialize(ActiveServerMods));
 
             if (Logger.IsLogEnabled(LogLevel.Debug))
             {
@@ -223,6 +224,24 @@ public class BackupService
         return result;
     }
 
+    protected string? GetMostRecentProfileBackup(IEnumerable<string> backupPaths, string profileId)
+    {
+        var profileFilename = $"{profileId}.json";
+        var backupPathsWithCreationDateTime = GetBackupPathsWithCreationTimestamp(backupPaths);
+
+        foreach (var (backupTimestamp, backupPath) in backupPathsWithCreationDateTime.Reverse())
+        {
+            var profileBackups = FileUtil.GetFiles(backupPath);
+            var profileBackup = profileBackups.FirstOrDefault(path => path.EndsWith(profileFilename));
+            if (profileBackup != null)
+            {
+                return profileBackup;
+            }
+        }
+
+        return null;
+    }
+
     /// <summary>
     ///     Retrieves and sorts the backup file paths from the specified directory.
     /// </summary>
@@ -308,4 +327,29 @@ public class BackupService
 
         return result;
     }
+
+    /// <summary>
+    ///     Restores the most recent profile backup for the given profile Id
+    /// </summary>
+    /// <param name="profileId">The profile ID of the backup to restore</param>
+    /// <returns>True on success. False on failure</returns>
+    public bool RestoreProfile(string profileId)
+    {
+        var backupDir = BackupConfig.Directory;
+        var backupPaths = GetBackupPaths(backupDir);
+        var mostRecentBackupForProfile = GetMostRecentProfileBackup(backupPaths, profileId);
+
+        // Verify we have a backup for this profile
+        if (mostRecentBackupForProfile == null)
+        {
+            return false;
+        }
+
+        // Restore the most recent profile backup
+        var profileFileName = FileUtil.GetFileNameAndExtension(mostRecentBackupForProfile);
+        var targetProfilePath = Path.Combine(ProfileDir, profileFileName);
+
+        File.Copy(mostRecentBackupForProfile, targetProfilePath, true);
+        return true;
+    }
 }

Libraries/SPTarkov.Server.Core/Services/LocationLifecycleService.cs

@@ -26,6 +26,7 @@ public class LocationLifecycleService(
     TimeUtil timeUtil,
     DatabaseService databaseService,
     ProfileHelper profileHelper,
+    BackupService backupService,
     ProfileActivityService profileActivityService,
     BotNameService botNameService,
     ICloner cloner,
@@ -77,6 +78,9 @@ public class LocationLifecycleService(
     /// </summary>
     public virtual StartLocalRaidResponseData StartLocalRaid(MongoId sessionId, StartLocalRaidRequestData request)
     {
+        // Backup the profile on raid start
+        backupService.Init().GetAwaiter().GetResult();
+
         logger.Debug($"Starting: {request.Location}");
 
         var playerProfile = profileHelper.GetFullProfile(sessionId);
@@ -409,6 +413,10 @@ public class LocationLifecycleService(
             HandleCoopExtract(sessionId, pmcProfile, request.Results.ExitName);
             SendCoopTakenFenceMessage(sessionId);
         }
+
+        // Save and backup the profile on raid end
+        saveServer.SaveProfileAsync(sessionId).GetAwaiter().GetResult();
+        backupService.Init().GetAwaiter().GetResult();
     }
 
     /// <summary>

Libraries/SPTarkov.Server.Core/Utils/FileUtil.cs

@@ -131,10 +131,18 @@ public class FileUtil
 
                 // We flush here so we can be sure it's immediately committed to disk
                 await fs.FlushAsync();
+                fs.Flush(true);
             }
 
             // Overwrite over the old file
-            File.Move(tempFilePath, filePath, overwrite: true);
+            if (File.Exists(filePath))
+            {
+                File.Replace(tempFilePath, filePath, null);
+            }
+            else
+            {
+                File.Move(tempFilePath, filePath);
+            }
         }
         catch
         {