lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved logging for DatabaseHelper

Browse Source
This commit is contained in:
LD-Reborn
2025-12-13 16:52:26 +01:00
parent 1b2050bb86
commit a1ed65ee41
5 changed files with 33 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/Server/Controllers/EntityController.cs
View File

@@ -12,12 +12,16 @@ public class EntityController : ControllerBase
private readonly ILogger<EntityController> _logger;
private readonly IConfiguration _config;
private SearchdomainManager _domainManager;
private readonly SearchdomainHelper _searchdomainHelper;
private readonly DatabaseHelper _databaseHelper;
public EntityController(ILogger<EntityController> logger, IConfiguration config, SearchdomainManager domainManager)
public EntityController(ILogger<EntityController> logger, IConfiguration config, SearchdomainManager domainManager, SearchdomainHelper searchdomainHelper, DatabaseHelper databaseHelper)
{
_logger = logger;
_config = config;
_domainManager = domainManager;
_searchdomainHelper = searchdomainHelper;
_databaseHelper = databaseHelper;
}
[HttpGet("Query")]
@@ -46,7 +50,7 @@ public class EntityController : ControllerBase
{
try
{
List<Entity>? entities = SearchdomainHelper.EntitiesFromJSON(
List<Entity>? entities = _searchdomainHelper.EntitiesFromJSON(
[],
_domainManager.embeddingCache,
_domainManager.aIProvider,
@@ -140,7 +144,7 @@ public class EntityController : ControllerBase
_logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]);
return Ok(new EntityDeleteResults() {Success = false});
}
DatabaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain);
_databaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain);
return Ok(new EntityDeleteResults() {Success = true});
}
}

17
src/Server/Helper/DatabaseHelper.cs
View File

@@ -3,8 +3,10 @@ using System.Text;
namespace Server.Helper;
public static class DatabaseHelper
public class DatabaseHelper(ILogger<DatabaseHelper> logger)
{
private readonly ILogger<DatabaseHelper> _logger = logger;
public static void DatabaseInsertEmbeddingBulk(SQLHelper helper, int id_datapoint, List<(string model, byte[] embedding)> data)
{
Dictionary<string, object> parameters = [];
@@ -80,7 +82,7 @@ public static class DatabaseHelper
return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO embedding (id_datapoint, model, embedding) VALUES (@id_datapoint, @model, @embedding)", parameters);
}
public static int GetSearchdomainID(SQLHelper helper, string searchdomain)
public int GetSearchdomainID(SQLHelper helper, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{
@@ -98,12 +100,13 @@ public static class DatabaseHelper
}
else
{
throw new Exception($"Unable to retrieve searchdomain ID for {searchdomain}"); // TODO implement logging here; add logger via method injection
_logger.LogError("Unable to retrieve searchdomain ID for {searchdomain}", [searchdomain]);
throw new Exception($"Unable to retrieve searchdomain ID for {searchdomain}");
}
}
}
public static void RemoveEntity(List<Entity> entityCache, SQLHelper helper, string name, string searchdomain)
public void RemoveEntity(List<Entity> entityCache, SQLHelper helper, string name, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{
@@ -118,7 +121,7 @@ public static class DatabaseHelper
entityCache.RemoveAll(entity => entity.name == name);
}
public static int RemoveAllEntities(SQLHelper helper, string searchdomain)
public int RemoveAllEntities(SQLHelper helper, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{
@@ -131,7 +134,7 @@ public static class DatabaseHelper
return helper.ExecuteSQLNonQuery("DELETE FROM entity WHERE entity.id_searchdomain = @searchdomain", parameters);
}
public static bool HasEntity(SQLHelper helper, string name, string searchdomain)
public bool HasEntity(SQLHelper helper, string name, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{
@@ -155,7 +158,7 @@ public static class DatabaseHelper
}
}
public static int? GetEntityID(SQLHelper helper, string name, string searchdomain)
public int? GetEntityID(SQLHelper helper, string name, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{

15
src/Server/Helper/SearchdomainHelper.cs
View File

@@ -7,8 +7,11 @@ using Shared.Models;
namespace Server.Helper;
public static class SearchdomainHelper
public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelper databaseHelper)
{
private readonly ILogger<SearchdomainHelper> _logger = logger;
private readonly DatabaseHelper _databaseHelper = databaseHelper;
public static byte[] BytesFromFloatArray(float[] floats)
{
var byteArray = new byte[floats.Length * 4];
@@ -41,7 +44,7 @@ public static class SearchdomainHelper
return null;
}
public static List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json)
public List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json)
{
List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json);
if (jsonEntities is null)
@@ -79,10 +82,10 @@ public static class SearchdomainHelper
return [.. retVal];
}
public static Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json)
public Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json)
{
Dictionary<string, Dictionary<string, float[]>> embeddingsLUT = []; // embeddingsLUT: hash -> model -> [embeddingValues * n]
int? preexistingEntityID = DatabaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
if (preexistingEntityID is not null)
{
lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after)
@@ -108,9 +111,9 @@ public static class SearchdomainHelper
}
reader.Close();
}
DatabaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create
_databaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create
}
int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, DatabaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
foreach (KeyValuePair<string, string> attribute in jsonEntity.Attributes)
{
DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries

3
src/Server/Program.cs
View File

@@ -3,6 +3,7 @@ using ElmahCore.Mvc;
using Serilog;
using Server;
using Server.HealthChecks;
using Server.Helper;
var builder = WebApplication.CreateBuilder(args);
@@ -16,6 +17,8 @@ Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(builder.Configuration)
.CreateLogger();
builder.Logging.AddSerilog();
builder.Services.AddSingleton<DatabaseHelper>();
builder.Services.AddSingleton<SearchdomainHelper>();
builder.Services.AddSingleton<SearchdomainManager>();
builder.Services.AddSingleton<AIProvider>();
builder.Services.AddHealthChecks()

7
src/Server/SearchdomainManager.cs
View File

@@ -11,16 +11,18 @@ public class SearchdomainManager
private readonly ILogger<SearchdomainManager> _logger;
private readonly IConfiguration _config;
public readonly AIProvider aIProvider;
private readonly DatabaseHelper _databaseHelper;
private readonly string connectionString;
private MySqlConnection connection;
public SQLHelper helper;
public Dictionary<string, Dictionary<string, float[]>> embeddingCache;
public SearchdomainManager(ILogger<SearchdomainManager> logger, IConfiguration config, AIProvider aIProvider)
public SearchdomainManager(ILogger<SearchdomainManager> logger, IConfiguration config, AIProvider aIProvider, DatabaseHelper databaseHelper)
{
_logger = logger;
_config = config;
this.aIProvider = aIProvider;
_databaseHelper = databaseHelper;
embeddingCache = [];
connectionString = _config.GetSection("Embeddingsearch").GetConnectionString("SQL") ?? "";
connection = new MySqlConnection(connectionString);
@@ -96,8 +98,7 @@ public class SearchdomainManager
public int DeleteSearchdomain(string searchdomain)
{
Searchdomain searchdomain_ = GetSearchdomain(searchdomain);
int counter = DatabaseHelper.RemoveAllEntities(helper, searchdomain);
int counter = _databaseHelper.RemoveAllEntities(helper, searchdomain);
_logger.LogDebug($"Number of entities deleted as part of deleting the searchdomain \"{searchdomain}\": {counter}");
helper.ExecuteSQLNonQuery("DELETE FROM searchdomain WHERE name = @name", new() {{"name", searchdomain}});
searchdomains.Remove(searchdomain);