chillcog/SPT-Server-Build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #317 from sp-tarkov/cleanup-dbimporter

Browse Source 
Cleanup DatabaseImporter & ImporterUtil
This commit is contained in:
Chomp
2025-05-30 14:59:11 +01:00
committed by GitHub
parent 6ee32bc257 ba9c57d699
commit 57c1935db6
2 changed files with 23 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Libraries/SPTarkov.Server.Core/Utils/DatabaseImporter.cs
+13 -181
View File
@@ -2,103 +2,35 @@ using System.Diagnostics;
using SPTarkov.DI.Annotations;
using SPTarkov.Server.Core.DI;
using SPTarkov.Server.Core.Models.Eft.Common.Tables;
using SPTarkov.Server.Core.Models.Spt.Config;
using SPTarkov.Server.Core.Models.Spt.Server;
using SPTarkov.Server.Core.Models.Utils;
using SPTarkov.Server.Core.Routers;
using SPTarkov.Server.Core.Servers;
using SPTarkov.Server.Core.Services;
using LogLevel = SPTarkov.Server.Core.Models.Spt.Logging.LogLevel;
namespace SPTarkov.Server.Core.Utils;
[Injectable(InjectionType.Singleton, TypePriority = OnLoadOrder.Database)]
public class DatabaseImporter : IOnLoad
public class DatabaseImporter(
ISptLogger<DatabaseImporter> logger,
FileUtil _fileUtil,
LocalisationService _localisationService,
DatabaseServer _databaseServer,
ImageRouter _imageRouter,
ImporterUtil _importerUtil
) : IOnLoad
{
private const string _sptDataPath = "./Assets/";
private readonly HttpConfig httpConfig;
private readonly ValidationResult valid = ValidationResult.UNDEFINED;
protected ConfigServer _configServer;
protected DatabaseServer _databaseServer;
protected EncodingUtil _encodingUtil;
protected FileUtil _fileUtil;
protected HashUtil _hashUtil;
protected ImageRouter _imageRouter;
protected ImporterUtil _importerUtil;
protected LocalisationService _localisationService;
protected ISptLogger<DatabaseImporter> _logger;
private string filepath;
private object hashedFile;
public DatabaseImporter(
ISptLogger<DatabaseImporter> logger,
// TODO: are we gonna use this? @inject("JsonUtil") protected jsonUtil: JsonUtil,
FileUtil fileUtil,
LocalisationService localisationService,
DatabaseServer databaseServer,
ImageRouter imageRouter,
EncodingUtil encodingUtil,
HashUtil hashUtil,
ImporterUtil importerUtil,
ConfigServer configServer
)
{
_logger = logger;
_localisationService = localisationService;
_databaseServer = databaseServer;
_encodingUtil = encodingUtil;
_hashUtil = hashUtil;
_importerUtil = importerUtil;
_configServer = configServer;
_fileUtil = fileUtil;
_imageRouter = imageRouter;
httpConfig = _configServer.GetConfig<HttpConfig>();
}
protected ISptLogger<DatabaseImporter> _logger = logger;
public async Task OnLoad()
{
filepath = GetSptDataPath();
await HydrateDatabase(_sptDataPath);
/*
if (ProgramStatics.COMPILED) {
try {
// Reading the dynamic SHA1 file
const file = "checks.dat";
const fileWithPath = `${this.filepath}${file}`;
if (this.vfs.exists(fileWithPath)) {
this.hashedFile = this.jsonUtil.deserialize(
this.encodingUtil.fromBase64(this.vfs.readFile(fileWithPath)),
file,
);
} else {
this.valid = ValidationResult.NOT_FOUND;
this.logger.debug(this.localisationService.getText("validation_not_found"));
}
} catch (e) {
this.valid = ValidationResult.FAILED;
this.logger.warning(this.localisationService.getText("validation_error_decode"));
}
}
*/
await HydrateDatabase(filepath);
var imageFilePath = $"{filepath}images/";
var imageFilePath = $"{_sptDataPath}images/";
CreateRouteMapping(imageFilePath, "files");
}
/**
* Get path to spt data
* @returns path to data
*/
public string GetSptDataPath()
{
return _sptDataPath;
}
private void CreateRouteMapping(string directory, string newBasePath)
{
var directoryContent = GetAllFilesInDirectory(directory);
@@ -141,8 +73,7 @@ public class DatabaseImporter : IOnLoad
timer.Start();
var dataToImport = await _importerUtil.LoadRecursiveAsync<DatabaseTables>(
$"{filePath}database/",
OnReadValidate
$"{filePath}database/"
);
// TODO: Fix loading of traders, so their full path is not included as the key
@@ -161,107 +92,8 @@ public class DatabaseImporter : IOnLoad
dataToImport.Traders = tempTraders;
var validation = valid == ValidationResult.FAILED || valid == ValidationResult.NOT_FOUND ? "." : "";
_logger.Info($"{_localisationService.GetText("importing_database_finish")}{validation}");
_logger.Info( _localisationService.GetText("importing_database_finish"));
_logger.Debug($"Database import took {timer.ElapsedMilliseconds}ms");
_databaseServer.SetTables(dataToImport);
}
protected void OnReadValidate(string fileWithPath)
{
// Validate files
//if (ProgramStatics.COMPILED && hashedFile && !ValidateFile(fileWithPath, data)) {
// this.valid = ValidationResult.FAILED;
//}
}
protected bool ValidateFile(string filePathAndName, object fileData)
{
/*
try {
const finalPath = filePathAndName.replace(this.filepath, "").replace(".json", "");
let tempObject: any;
for (const prop of finalPath.split("/")) {
if (!tempObject) {
tempObject = this.hashedFile[prop];
} else {
tempObject = tempObject[prop];
}
}
if (tempObject !== this.hashUtil.generateSha1ForData(fileData)) {
this.logger.debug(this.localisationService.getText("validation_error_file", filePathAndName));
return false;
}
} catch (e) {
this.logger.warning(this.localisationService.getText("validation_error_exception", filePathAndName));
this.logger.warning(e);
return false;
}
return true;
*/
return true;
}
/**
* absolute dogshit, do not use
* Find and map files with image router inside a designated path
* @param filepath Path to find files in
*/
[Obsolete]
public void LoadImages(string filepath, string[] directories, List<string> routes)
{
for (var i = 0; i < directories.Length; i++)
{
// Get all files in directory
var filesInDirectory = _fileUtil.GetFiles(directories[i]);
foreach (var file in filesInDirectory)
{
var imagePath = file;
// Register each file in image router
var filename = _fileUtil.StripExtension(file);
var routeKey = $"{routes[i]}{filename}";
//var imagePath = $"{filepath}{directories[i]}/{file}";
var pathOverride = GetImagePathOverride(imagePath);
if (!string.IsNullOrEmpty(pathOverride))
{
if (_logger.IsLogEnabled(LogLevel.Debug))
{
_logger.Debug($"overrode route: {routeKey} endpoint: {imagePath} with {pathOverride}");
}
imagePath = pathOverride;
}
_imageRouter.AddRoute(routeKey, imagePath);
}
}
// Map icon file separately
_imageRouter.AddRoute("/favicon.ico", $"{filepath}icon.ico");
}
/**
* Check for a path override in the http json config file
* @param imagePath Key
* @returns override for key
*/
protected string? GetImagePathOverride(string imagePath)
{
if (httpConfig.ServerImagePathOverride.TryGetValue(imagePath, out var value))
{
return value;
}
return null;
}
}
internal enum ValidationResult
{
SUCCESS = 0,
FAILED = 1,
NOT_FOUND = 2,
UNDEFINED = 3
}
Libraries/SPTarkov.Server.Core/Utils/ImporterUtil.cs
+10 -21
View File
@@ -1,4 +1,3 @@
using System.Collections.Concurrent;
using System.Linq.Expressions;
using System.Reflection;
using SPTarkov.DI.Annotations;
@@ -8,43 +7,33 @@ using SPTarkov.Server.Core.Utils.Json;
namespace SPTarkov.Server.Core.Utils;
[Injectable(InjectionType.Singleton)]
public class ImporterUtil
public class ImporterUtil(ISptLogger<ImporterUtil> _logger, FileUtil _fileUtil, JsonUtil _jsonUtil)
{
protected readonly ConcurrentDictionary<Type, Delegate> lazyLoadDeserializationCache = [];
protected FileUtil _fileUtil;
protected JsonUtil _jsonUtil;
protected ISptLogger<ImporterUtil> _logger;
protected HashSet<string> directoriesToIgnore = ["./Assets/database/locales/server"];
protected HashSet<string> filesToIgnore = ["bearsuits.json", "usecsuits.json", "archivedquests.json"];
public ImporterUtil(ISptLogger<ImporterUtil> logger, FileUtil fileUtil, JsonUtil jsonUtil)
{
_logger = logger;
_fileUtil = fileUtil;
_jsonUtil = jsonUtil;
}
public Task<T> LoadRecursiveAsync<T>(
string filepath,
public async Task<T> LoadRecursiveAsync<T>(
string filePath,
Action<string>? onReadCallback = null,
Action<string, object>? onObjectDeserialized = null
)
{
return LoadRecursiveAsync(filepath, typeof(T), onReadCallback, onObjectDeserialized)
.ContinueWith(res => (T) res.Result);
var result = await LoadRecursiveAsync(filePath, typeof(T), onReadCallback, onObjectDeserialized);
return (T) result;
}
/// <summary>
/// Load files into objects recursively (asynchronous)
/// </summary>
/// <param name="filepath">Path to folder with files</param>
/// <param name="filePath">Path to folder with files</param>
/// <param name="loadedType"></param>
/// <param name="onReadCallback"></param>
/// <param name="onObjectDeserialized"></param>
/// <returns>Task</returns>
protected async Task<object> LoadRecursiveAsync(
string filepath,
string filePath,
Type loadedType,
Action<string>? onReadCallback = null,
Action<string, object>? onObjectDeserialized = null
@@ -55,8 +44,8 @@ public class ImporterUtil
var result = Activator.CreateInstance(loadedType);
// get all filepaths
var files = _fileUtil.GetFiles(filepath);
var directories = _fileUtil.GetDirectories(filepath);
var files = _fileUtil.GetFiles(filePath);
var directories = _fileUtil.GetDirectories(filePath);
// Process files
foreach (var file in files)