chillcog/SPT-Server-Build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add File validation (#344)

Browse Source 
* Add file validation

* Revert "Added checks.dat build script (#343)"

This reverts commit 39228f88e705b58858d162256a5b5e10fe99148c.

* Update to use pwsh

* Wrap code in using
This commit is contained in:
Jesse
2025-06-02 21:21:49 +02:00
committed by GitHub
parent 3fe8072604
commit 354adf2c0a
8 changed files with 145 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -5,6 +5,7 @@
Server/user/
SPTarkov.Server/user
SPTarkov.Server/Assets
Libraries/SPTarkov.Server.Assets/Assets/checks.dat
# User-specific files
*.rsuser
Libraries/SPTarkov.Server.Assets/PostBuild.ps1
+30
View File
@@ -0,0 +1,30 @@
$scriptDir = $PSScriptRoot
$assetsPath = Join-Path $scriptDir 'Assets'
$outputFile = Join-Path $assetsPath 'checks.dat'
$files = Get-ChildItem -Path $assetsPath -Recurse -File |
Where-Object { $_.FullName -notmatch [regex]::Escape((Join-Path $assetsPath 'images')) } |
Sort-Object FullName
$hashes = foreach ($file in $files) {
$bytes = [System.IO.File]::ReadAllBytes($file.FullName)
$md5 = [System.Security.Cryptography.MD5]::Create()
$hashBytes = $md5.ComputeHash($bytes)
$md5.Dispose()
$hashString = [BitConverter]::ToString($hashBytes) -replace '-', ''
$relativePath = $file.FullName.Substring($assetsPath.Length + 1) -replace '\\', '/'
[PSCustomObject]@{
Path = $relativePath
Hash = $hashString
}
}
$jsonString = $hashes | ConvertTo-Json -Depth 10
$bytes = [System.Text.Encoding]::UTF8.GetBytes($jsonString)
$base64String = [Convert]::ToBase64String($bytes)
Set-Content -Path $outputFile -Value $base64String -Encoding ASCII
Libraries/SPTarkov.Server.Assets/SPTarkov.Server.Assets.csproj
+5 -1
View File
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="..\..\Build.props"/>
@@ -25,6 +25,10 @@
</Content>
</ItemGroup>
<Target Name="PostBuildHashFile" AfterTargets="Build">
<Exec Command="pwsh -NoProfile -ExecutionPolicy Bypass -File &quot;$(ProjectDir)PostBuild.ps1&quot;" />
</Target>
<ItemGroup>
<None Include="..\..\LICENSE" Pack="true" Visible="false" PackagePath=""/>
</ItemGroup>
Libraries/SPTarkov.Server.Core/Utils/DatabaseImporter.cs
+81 -3
View File
@@ -1,4 +1,6 @@
using System.Diagnostics;
using System.Security.Cryptography;
using System.Text;
using SPTarkov.DI.Annotations;
using SPTarkov.Server.Core.DI;
using SPTarkov.Server.Core.Models.Eft.Common.Tables;
@@ -17,14 +19,17 @@ public class DatabaseImporter(
LocalisationService _localisationService,
DatabaseServer _databaseServer,
ImageRouter _imageRouter,
ImporterUtil _importerUtil
ImporterUtil _importerUtil,
JsonUtil _jsonUtil
) : IOnLoad
{
private const string _sptDataPath = "./Assets/";
protected ISptLogger<DatabaseImporter> _logger = logger;
protected Dictionary<string, string> databaseHashes = [];
public async Task OnLoad()
{
await LoadHashes();
await HydrateDatabase(_sptDataPath);
var imageFilePath = $"{_sptDataPath}images/";
@@ -62,6 +67,41 @@ public class DatabaseImporter(
return result;
}
protected async Task LoadHashes()
{
var checksFilePath = System.IO.Path.Combine(_sptDataPath, "checks.dat");
try
{
if (File.Exists(checksFilePath))
{
await using FileStream fs = File.OpenRead(checksFilePath);
using var reader = new StreamReader(fs, Encoding.ASCII);
string base64Content = await reader.ReadToEndAsync();
byte[] jsonBytes = Convert.FromBase64String(base64Content);
await using var ms = new MemoryStream(jsonBytes);
var FileHashes = await _jsonUtil.DeserializeFromMemoryStreamAsync<List<FileHash>>(ms) ?? [];
foreach(var hash in FileHashes)
{
databaseHashes.Add(hash.Path, hash.Hash);
}
}
else
{
_logger.Error(_localisationService.GetText("validation_error_exception", checksFilePath));
}
}
catch (Exception)
{
_logger.Error(_localisationService.GetText("validation_error_exception", checksFilePath));
}
}
/**
* Read all json files in database folder and map into a json object
* @param filepath path to database folder
@@ -73,7 +113,8 @@ public class DatabaseImporter(
timer.Start();
var dataToImport = await _importerUtil.LoadRecursiveAsync<DatabaseTables>(
$"{filePath}database/"
$"{filePath}database/",
VerifyDatabase
);
// TODO: Fix loading of traders, so their full path is not included as the key
@@ -92,8 +133,45 @@ public class DatabaseImporter(
dataToImport.Traders = tempTraders;
_logger.Info( _localisationService.GetText("importing_database_finish"));
_logger.Info(_localisationService.GetText("importing_database_finish"));
_logger.Debug($"Database import took {timer.ElapsedMilliseconds}ms");
_databaseServer.SetTables(dataToImport);
}
protected async Task VerifyDatabase(string fileName)
{
var relativePath = fileName.StartsWith(_sptDataPath, StringComparison.OrdinalIgnoreCase)
? fileName.Substring(_sptDataPath.Length)
: fileName;
using (var md5 = MD5.Create())
{
await using (var stream = File.OpenRead(fileName))
{
var hashBytes = await md5.ComputeHashAsync(stream);
var hashString = Convert.ToHexString(hashBytes);
bool hashKeyExists = databaseHashes.ContainsKey(relativePath);
if (hashKeyExists)
{
if (databaseHashes[relativePath] != hashString)
{
_logger.Warning(_localisationService.GetText("validation_error_file", fileName));
}
}
else
{
_logger.Warning(_localisationService.GetText("validation_error_file", fileName));
}
}
}
}
}
public class FileHash
{
public string Path { get; set; } = string.Empty;
public string Hash { get; set; } = string.Empty;
}
Libraries/SPTarkov.Server.Core/Utils/ImporterUtil.cs
+18 -10
View File
@@ -15,8 +15,8 @@ public class ImporterUtil(ISptLogger<ImporterUtil> _logger, FileUtil _fileUtil,
public async Task<T> LoadRecursiveAsync<T>(
string filePath,
Action<string>? onReadCallback = null,
Action<string, object>? onObjectDeserialized = null
Func<string, Task>? onReadCallback = null,
Func<string, object, Task>? onObjectDeserialized = null
)
{
var result = await LoadRecursiveAsync(filePath, typeof(T), onReadCallback, onObjectDeserialized);
@@ -35,8 +35,8 @@ public class ImporterUtil(ISptLogger<ImporterUtil> _logger, FileUtil _fileUtil,
protected async Task<object> LoadRecursiveAsync(
string filePath,
Type loadedType,
Action<string>? onReadCallback = null,
Action<string, object>? onObjectDeserialized = null
Func<string, Task>? onReadCallback = null,
Func<string, object, Task>? onObjectDeserialized = null
)
{
var tasks = new List<Task>();
@@ -66,7 +66,7 @@ public class ImporterUtil(ISptLogger<ImporterUtil> _logger, FileUtil _fileUtil,
continue;
}
tasks.Add(ProcessDirectoryAsync(directory, loadedType, result, dictionaryLock));
tasks.Add(ProcessDirectoryAsync(directory, loadedType, result, onReadCallback, onObjectDeserialized, dictionaryLock));
}
// Wait for all tasks to finish
@@ -78,15 +78,18 @@ public class ImporterUtil(ISptLogger<ImporterUtil> _logger, FileUtil _fileUtil,
private async Task ProcessFileAsync(
string file,
Type loadedType,
Action<string>? onReadCallback,
Action<string, object>? onObjectDeserialized,
Func<string, Task>? onReadCallback,
Func<string, object, Task>? onObjectDeserialized,
object result,
Lock dictionaryLock
)
{
try
{
onReadCallback?.Invoke(file);
if (onReadCallback != null)
{
await onReadCallback(file);
}
// Get the set method to update the object
var setMethod = GetSetMethod(
@@ -98,7 +101,10 @@ public class ImporterUtil(ISptLogger<ImporterUtil> _logger, FileUtil _fileUtil,
var fileDeserialized = await DeserializeFileAsync(file, propertyType);
onObjectDeserialized?.Invoke(file, fileDeserialized);
if (onObjectDeserialized != null)
{
await onObjectDeserialized(file, fileDeserialized);
}
lock (dictionaryLock)
{
@@ -120,6 +126,8 @@ public class ImporterUtil(ISptLogger<ImporterUtil> _logger, FileUtil _fileUtil,
string directory,
Type loadedType,
object result,
Func<string, Task>? onReadCallback,
Func<string, object, Task>? onObjectDeserialized,
Lock dictionaryLock
)
{
@@ -132,7 +140,7 @@ public class ImporterUtil(ISptLogger<ImporterUtil> _logger, FileUtil _fileUtil,
out var isDictionary
);
var loadedData = await LoadRecursiveAsync($"{directory}/", matchedProperty);
var loadedData = await LoadRecursiveAsync($"{directory}/", matchedProperty, onReadCallback, onObjectDeserialized);
lock (dictionaryLock)
{
Libraries/SPTarkov.Server.Core/Utils/JsonUtil.cs
+10
View File
@@ -154,6 +154,16 @@ public class JsonUtil
return await JsonSerializer.DeserializeAsync(fs, type, jsonSerializerOptionsNoIndent);
}
/// <summary>
/// Convert JSON into an object from a MemoryStream asynchronously
/// </summary>
/// <param name="fs">The memory stream to deserialize</param>
/// <returns>T</returns>
public async Task<T?> DeserializeFromMemoryStreamAsync<T>(MemoryStream ms)
{
return await JsonSerializer.DeserializeAsync<T>(ms, jsonSerializerOptionsNoIndent);
}
/// <summary>
/// Convert an object into JSON
/// </summary>
SPTarkov.Server/SPTarkov.Server.csproj
-5
View File
@@ -41,15 +41,10 @@
<None Update="sptLogger.Development.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Remove="postBuildScript.ps1" />
</ItemGroup>
<ItemGroup>
<None Include="..\LICENSE" Pack="true" Visible="false" PackagePath="" />
</ItemGroup>
<Target Name="PostBuildScript" AfterTargets="AfterRebuild" Condition="'$(Configuration)' == 'Release'">
<Exec Command="powershell -ExecutionPolicy Bypass -File $(ProjectDir)\postBuildScript.ps1 $(OutDir)Assets $(OutDir)Assets/checks.dat"/>
</Target>
</Project>
SPTarkov.Server/postBuildScript.ps1
-57
View File
@@ -1,57 +0,0 @@
param (
[string]$filepath,
[string]$output
)
function Load-RecursiveAsync {
param (
[string]$filepath
)
$result = @{}
$filesList = Get-ChildItem -Path $filepath
foreach ($file in $filesList) {
$curPath = $file.FullName
if ($file.PSIsContainer) {
$result[$file.BaseName] = Load-RecursiveAsync "$filepath\$($file.Name)"
} elseif ($file.Extension -eq ".json") {
$result[$file.BaseName] = Generate-HashForData (Get-Content -Raw -Path "$filepath\$($file.Name)")
}
}
return $result
}
function Generate-HashForData {
param (
[string]$data
)
$sha1 = [System.Security.Cryptography.SHA1]::Create()
$hashBytes = $sha1.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($data))
$hashHex = -join ($hashBytes | ForEach-Object { $_.ToString("x2") }) # Convert bytes to hex
return $hashHex
}
function Encode-Base64 {
param (
[Parameter(ValueFromPipeline=$true)]
[string]$inputString
)
process {
if ($inputString -and $inputString -ne "") {
$bytes = [System.Text.Encoding]::UTF8.GetBytes($inputString)
$base64String = [Convert]::ToBase64String($bytes)
return $base64String
} else {
Write-Output "Error: No valid input received!"
}
}
}
$results = Load-RecursiveAsync $filepath
$results | ConvertTo-Json -Depth 10 -Compress | Encode-Base64 | Out-File $output