From 354adf2c0acd0eda339aa301b3f7a17558c4e24d Mon Sep 17 00:00:00 2001 From: Jesse Date: Mon, 2 Jun 2025 21:21:49 +0200 Subject: [PATCH] Add File validation (#344) * Add file validation * Revert "Added checks.dat build script (#343)" This reverts commit 39228f88e705b58858d162256a5b5e10fe99148c. * Update to use pwsh * Wrap code in using --- .gitignore | 1 + .../SPTarkov.Server.Assets/PostBuild.ps1 | 30 +++++++ .../SPTarkov.Server.Assets.csproj | 6 +- .../Utils/DatabaseImporter.cs | 84 ++++++++++++++++++- .../Utils/ImporterUtil.cs | 28 ++++--- .../SPTarkov.Server.Core/Utils/JsonUtil.cs | 10 +++ SPTarkov.Server/SPTarkov.Server.csproj | 5 -- SPTarkov.Server/postBuildScript.ps1 | 57 ------------- 8 files changed, 145 insertions(+), 76 deletions(-) create mode 100644 Libraries/SPTarkov.Server.Assets/PostBuild.ps1 delete mode 100644 SPTarkov.Server/postBuildScript.ps1 diff --git a/.gitignore b/.gitignore index 5b96b1d6..83a4529a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ Server/user/ SPTarkov.Server/user SPTarkov.Server/Assets +Libraries/SPTarkov.Server.Assets/Assets/checks.dat # User-specific files *.rsuser diff --git a/Libraries/SPTarkov.Server.Assets/PostBuild.ps1 b/Libraries/SPTarkov.Server.Assets/PostBuild.ps1 new file mode 100644 index 00000000..6b0e535f --- /dev/null +++ b/Libraries/SPTarkov.Server.Assets/PostBuild.ps1 @@ -0,0 +1,30 @@ +$scriptDir = $PSScriptRoot +$assetsPath = Join-Path $scriptDir 'Assets' +$outputFile = Join-Path $assetsPath 'checks.dat' + +$files = Get-ChildItem -Path $assetsPath -Recurse -File | + Where-Object { $_.FullName -notmatch [regex]::Escape((Join-Path $assetsPath 'images')) } | + Sort-Object FullName + +$hashes = foreach ($file in $files) { + $bytes = [System.IO.File]::ReadAllBytes($file.FullName) + $md5 = [System.Security.Cryptography.MD5]::Create() + $hashBytes = $md5.ComputeHash($bytes) + $md5.Dispose() + + $hashString = [BitConverter]::ToString($hashBytes) -replace '-', '' + + $relativePath = $file.FullName.Substring($assetsPath.Length + 1) -replace '\\', '/' + + [PSCustomObject]@{ + Path = $relativePath + Hash = $hashString + } +} + +$jsonString = $hashes | ConvertTo-Json -Depth 10 + +$bytes = [System.Text.Encoding]::UTF8.GetBytes($jsonString) +$base64String = [Convert]::ToBase64String($bytes) + +Set-Content -Path $outputFile -Value $base64String -Encoding ASCII diff --git a/Libraries/SPTarkov.Server.Assets/SPTarkov.Server.Assets.csproj b/Libraries/SPTarkov.Server.Assets/SPTarkov.Server.Assets.csproj index e6c3d5ec..18d616fe 100644 --- a/Libraries/SPTarkov.Server.Assets/SPTarkov.Server.Assets.csproj +++ b/Libraries/SPTarkov.Server.Assets/SPTarkov.Server.Assets.csproj @@ -1,4 +1,4 @@ - + @@ -25,6 +25,10 @@ + + + + diff --git a/Libraries/SPTarkov.Server.Core/Utils/DatabaseImporter.cs b/Libraries/SPTarkov.Server.Core/Utils/DatabaseImporter.cs index dd7f87c3..99ea1d97 100644 --- a/Libraries/SPTarkov.Server.Core/Utils/DatabaseImporter.cs +++ b/Libraries/SPTarkov.Server.Core/Utils/DatabaseImporter.cs @@ -1,4 +1,6 @@ using System.Diagnostics; +using System.Security.Cryptography; +using System.Text; using SPTarkov.DI.Annotations; using SPTarkov.Server.Core.DI; using SPTarkov.Server.Core.Models.Eft.Common.Tables; @@ -17,14 +19,17 @@ public class DatabaseImporter( LocalisationService _localisationService, DatabaseServer _databaseServer, ImageRouter _imageRouter, - ImporterUtil _importerUtil + ImporterUtil _importerUtil, + JsonUtil _jsonUtil ) : IOnLoad { private const string _sptDataPath = "./Assets/"; protected ISptLogger _logger = logger; + protected Dictionary databaseHashes = []; public async Task OnLoad() { + await LoadHashes(); await HydrateDatabase(_sptDataPath); var imageFilePath = $"{_sptDataPath}images/"; @@ -62,6 +67,41 @@ public class DatabaseImporter( return result; } + protected async Task LoadHashes() + { + var checksFilePath = System.IO.Path.Combine(_sptDataPath, "checks.dat"); + + try + { + if (File.Exists(checksFilePath)) + { + await using FileStream fs = File.OpenRead(checksFilePath); + + using var reader = new StreamReader(fs, Encoding.ASCII); + string base64Content = await reader.ReadToEndAsync(); + + byte[] jsonBytes = Convert.FromBase64String(base64Content); + + await using var ms = new MemoryStream(jsonBytes); + + var FileHashes = await _jsonUtil.DeserializeFromMemoryStreamAsync>(ms) ?? []; + + foreach(var hash in FileHashes) + { + databaseHashes.Add(hash.Path, hash.Hash); + } + } + else + { + _logger.Error(_localisationService.GetText("validation_error_exception", checksFilePath)); + } + } + catch (Exception) + { + _logger.Error(_localisationService.GetText("validation_error_exception", checksFilePath)); + } + } + /** * Read all json files in database folder and map into a json object * @param filepath path to database folder @@ -73,7 +113,8 @@ public class DatabaseImporter( timer.Start(); var dataToImport = await _importerUtil.LoadRecursiveAsync( - $"{filePath}database/" + $"{filePath}database/", + VerifyDatabase ); // TODO: Fix loading of traders, so their full path is not included as the key @@ -92,8 +133,45 @@ public class DatabaseImporter( dataToImport.Traders = tempTraders; - _logger.Info( _localisationService.GetText("importing_database_finish")); + _logger.Info(_localisationService.GetText("importing_database_finish")); _logger.Debug($"Database import took {timer.ElapsedMilliseconds}ms"); _databaseServer.SetTables(dataToImport); } + + protected async Task VerifyDatabase(string fileName) + { + var relativePath = fileName.StartsWith(_sptDataPath, StringComparison.OrdinalIgnoreCase) + ? fileName.Substring(_sptDataPath.Length) + : fileName; + + using (var md5 = MD5.Create()) + { + await using (var stream = File.OpenRead(fileName)) + { + var hashBytes = await md5.ComputeHashAsync(stream); + var hashString = Convert.ToHexString(hashBytes); + + bool hashKeyExists = databaseHashes.ContainsKey(relativePath); + + if (hashKeyExists) + { + if (databaseHashes[relativePath] != hashString) + { + _logger.Warning(_localisationService.GetText("validation_error_file", fileName)); + } + } + else + { + _logger.Warning(_localisationService.GetText("validation_error_file", fileName)); + } + } + } + } } + +public class FileHash +{ + public string Path { get; set; } = string.Empty; + public string Hash { get; set; } = string.Empty; +} + diff --git a/Libraries/SPTarkov.Server.Core/Utils/ImporterUtil.cs b/Libraries/SPTarkov.Server.Core/Utils/ImporterUtil.cs index c9e78ab5..99763d86 100644 --- a/Libraries/SPTarkov.Server.Core/Utils/ImporterUtil.cs +++ b/Libraries/SPTarkov.Server.Core/Utils/ImporterUtil.cs @@ -15,8 +15,8 @@ public class ImporterUtil(ISptLogger _logger, FileUtil _fileUtil, public async Task LoadRecursiveAsync( string filePath, - Action? onReadCallback = null, - Action? onObjectDeserialized = null + Func? onReadCallback = null, + Func? onObjectDeserialized = null ) { var result = await LoadRecursiveAsync(filePath, typeof(T), onReadCallback, onObjectDeserialized); @@ -35,8 +35,8 @@ public class ImporterUtil(ISptLogger _logger, FileUtil _fileUtil, protected async Task LoadRecursiveAsync( string filePath, Type loadedType, - Action? onReadCallback = null, - Action? onObjectDeserialized = null + Func? onReadCallback = null, + Func? onObjectDeserialized = null ) { var tasks = new List(); @@ -66,7 +66,7 @@ public class ImporterUtil(ISptLogger _logger, FileUtil _fileUtil, continue; } - tasks.Add(ProcessDirectoryAsync(directory, loadedType, result, dictionaryLock)); + tasks.Add(ProcessDirectoryAsync(directory, loadedType, result, onReadCallback, onObjectDeserialized, dictionaryLock)); } // Wait for all tasks to finish @@ -78,15 +78,18 @@ public class ImporterUtil(ISptLogger _logger, FileUtil _fileUtil, private async Task ProcessFileAsync( string file, Type loadedType, - Action? onReadCallback, - Action? onObjectDeserialized, + Func? onReadCallback, + Func? onObjectDeserialized, object result, Lock dictionaryLock ) { try { - onReadCallback?.Invoke(file); + if (onReadCallback != null) + { + await onReadCallback(file); + } // Get the set method to update the object var setMethod = GetSetMethod( @@ -98,7 +101,10 @@ public class ImporterUtil(ISptLogger _logger, FileUtil _fileUtil, var fileDeserialized = await DeserializeFileAsync(file, propertyType); - onObjectDeserialized?.Invoke(file, fileDeserialized); + if (onObjectDeserialized != null) + { + await onObjectDeserialized(file, fileDeserialized); + } lock (dictionaryLock) { @@ -120,6 +126,8 @@ public class ImporterUtil(ISptLogger _logger, FileUtil _fileUtil, string directory, Type loadedType, object result, + Func? onReadCallback, + Func? onObjectDeserialized, Lock dictionaryLock ) { @@ -132,7 +140,7 @@ public class ImporterUtil(ISptLogger _logger, FileUtil _fileUtil, out var isDictionary ); - var loadedData = await LoadRecursiveAsync($"{directory}/", matchedProperty); + var loadedData = await LoadRecursiveAsync($"{directory}/", matchedProperty, onReadCallback, onObjectDeserialized); lock (dictionaryLock) { diff --git a/Libraries/SPTarkov.Server.Core/Utils/JsonUtil.cs b/Libraries/SPTarkov.Server.Core/Utils/JsonUtil.cs index e015f6ed..bb821a24 100644 --- a/Libraries/SPTarkov.Server.Core/Utils/JsonUtil.cs +++ b/Libraries/SPTarkov.Server.Core/Utils/JsonUtil.cs @@ -154,6 +154,16 @@ public class JsonUtil return await JsonSerializer.DeserializeAsync(fs, type, jsonSerializerOptionsNoIndent); } + /// + /// Convert JSON into an object from a MemoryStream asynchronously + /// + /// The memory stream to deserialize + /// T + public async Task DeserializeFromMemoryStreamAsync(MemoryStream ms) + { + return await JsonSerializer.DeserializeAsync(ms, jsonSerializerOptionsNoIndent); + } + /// /// Convert an object into JSON /// diff --git a/SPTarkov.Server/SPTarkov.Server.csproj b/SPTarkov.Server/SPTarkov.Server.csproj index b3106fc1..bce8f96e 100644 --- a/SPTarkov.Server/SPTarkov.Server.csproj +++ b/SPTarkov.Server/SPTarkov.Server.csproj @@ -41,15 +41,10 @@ Always - - - - - diff --git a/SPTarkov.Server/postBuildScript.ps1 b/SPTarkov.Server/postBuildScript.ps1 deleted file mode 100644 index 4649f3b5..00000000 --- a/SPTarkov.Server/postBuildScript.ps1 +++ /dev/null @@ -1,57 +0,0 @@ -param ( - [string]$filepath, - [string]$output -) - -function Load-RecursiveAsync { - param ( - [string]$filepath - ) - - $result = @{} - - $filesList = Get-ChildItem -Path $filepath - - foreach ($file in $filesList) { - $curPath = $file.FullName - if ($file.PSIsContainer) { - $result[$file.BaseName] = Load-RecursiveAsync "$filepath\$($file.Name)" - } elseif ($file.Extension -eq ".json") { - $result[$file.BaseName] = Generate-HashForData (Get-Content -Raw -Path "$filepath\$($file.Name)") - } - } - - return $result -} - -function Generate-HashForData { - param ( - [string]$data - ) - - $sha1 = [System.Security.Cryptography.SHA1]::Create() - $hashBytes = $sha1.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($data)) - $hashHex = -join ($hashBytes | ForEach-Object { $_.ToString("x2") }) # Convert bytes to hex - - return $hashHex -} - -function Encode-Base64 { - param ( - [Parameter(ValueFromPipeline=$true)] - [string]$inputString - ) - - process { - if ($inputString -and $inputString -ne "") { - $bytes = [System.Text.Encoding]::UTF8.GetBytes($inputString) - $base64String = [Convert]::ToBase64String($bytes) - return $base64String - } else { - Write-Output "Error: No valid input received!" - } - } -} - -$results = Load-RecursiveAsync $filepath -$results | ConvertTo-Json -Depth 10 -Compress | Encode-Base64 | Out-File $output \ No newline at end of file