lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved logging for DatabaseHelper

Browse Source
This commit is contained in:
LD-Reborn
2025-12-13 16:52:26 +01:00
parent 1b2050bb86
commit a1ed65ee41
5 changed files with 33 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/Server/Controllers/EntityController.cs
View File

@@ -12,12 +12,16 @@ public class EntityController : ControllerBase
private readonly ILogger<EntityController> _logger; private readonly ILogger<EntityController> _logger;
private readonly IConfiguration _config; private readonly IConfiguration _config;
private SearchdomainManager _domainManager; private SearchdomainManager _domainManager;
private readonly SearchdomainHelper _searchdomainHelper;
private readonly DatabaseHelper _databaseHelper;
public EntityController(ILogger<EntityController> logger, IConfiguration config, SearchdomainManager domainManager) public EntityController(ILogger<EntityController> logger, IConfiguration config, SearchdomainManager domainManager, SearchdomainHelper searchdomainHelper, DatabaseHelper databaseHelper)
{ {
_logger = logger; _logger = logger;
_config = config; _config = config;
_domainManager = domainManager; _domainManager = domainManager;
_searchdomainHelper = searchdomainHelper;
_databaseHelper = databaseHelper;
} }
[HttpGet("Query")] [HttpGet("Query")]
@@ -46,7 +50,7 @@ public class EntityController : ControllerBase
{ {
try try
{ {
List<Entity>? entities = SearchdomainHelper.EntitiesFromJSON( List<Entity>? entities = _searchdomainHelper.EntitiesFromJSON(
[], [],
_domainManager.embeddingCache, _domainManager.embeddingCache,
_domainManager.aIProvider, _domainManager.aIProvider,
@@ -140,7 +144,7 @@ public class EntityController : ControllerBase
_logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]); _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]);
return Ok(new EntityDeleteResults() {Success = false}); return Ok(new EntityDeleteResults() {Success = false});
} }
DatabaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain); _databaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain);
return Ok(new EntityDeleteResults() {Success = true}); return Ok(new EntityDeleteResults() {Success = true});
} }
} }

17
src/Server/Helper/DatabaseHelper.cs
View File

@@ -3,8 +3,10 @@ using System.Text;
namespace Server.Helper; namespace Server.Helper;
public static class DatabaseHelper public class DatabaseHelper(ILogger<DatabaseHelper> logger)
{ {
private readonly ILogger<DatabaseHelper> _logger = logger;
public static void DatabaseInsertEmbeddingBulk(SQLHelper helper, int id_datapoint, List<(string model, byte[] embedding)> data) public static void DatabaseInsertEmbeddingBulk(SQLHelper helper, int id_datapoint, List<(string model, byte[] embedding)> data)
{ {
Dictionary<string, object> parameters = []; Dictionary<string, object> parameters = [];
@@ -80,7 +82,7 @@ public static class DatabaseHelper
return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO embedding (id_datapoint, model, embedding) VALUES (@id_datapoint, @model, @embedding)", parameters); return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO embedding (id_datapoint, model, embedding) VALUES (@id_datapoint, @model, @embedding)", parameters);
} }
public static int GetSearchdomainID(SQLHelper helper, string searchdomain) public int GetSearchdomainID(SQLHelper helper, string searchdomain)
{ {
Dictionary<string, dynamic> parameters = new() Dictionary<string, dynamic> parameters = new()
{ {
@@ -98,12 +100,13 @@ public static class DatabaseHelper
} }
else else
{ {
throw new Exception($"Unable to retrieve searchdomain ID for {searchdomain}"); // TODO implement logging here; add logger via method injection _logger.LogError("Unable to retrieve searchdomain ID for {searchdomain}", [searchdomain]);
throw new Exception($"Unable to retrieve searchdomain ID for {searchdomain}");
} }
} }
} }
public static void RemoveEntity(List<Entity> entityCache, SQLHelper helper, string name, string searchdomain) public void RemoveEntity(List<Entity> entityCache, SQLHelper helper, string name, string searchdomain)
{ {
Dictionary<string, dynamic> parameters = new() Dictionary<string, dynamic> parameters = new()
{ {
@@ -118,7 +121,7 @@ public static class DatabaseHelper
entityCache.RemoveAll(entity => entity.name == name); entityCache.RemoveAll(entity => entity.name == name);
} }
public static int RemoveAllEntities(SQLHelper helper, string searchdomain) public int RemoveAllEntities(SQLHelper helper, string searchdomain)
{ {
Dictionary<string, dynamic> parameters = new() Dictionary<string, dynamic> parameters = new()
{ {
@@ -131,7 +134,7 @@ public static class DatabaseHelper
return helper.ExecuteSQLNonQuery("DELETE FROM entity WHERE entity.id_searchdomain = @searchdomain", parameters); return helper.ExecuteSQLNonQuery("DELETE FROM entity WHERE entity.id_searchdomain = @searchdomain", parameters);
} }
public static bool HasEntity(SQLHelper helper, string name, string searchdomain) public bool HasEntity(SQLHelper helper, string name, string searchdomain)
{ {
Dictionary<string, dynamic> parameters = new() Dictionary<string, dynamic> parameters = new()
{ {
@@ -155,7 +158,7 @@ public static class DatabaseHelper
} }
} }
public static int? GetEntityID(SQLHelper helper, string name, string searchdomain) public int? GetEntityID(SQLHelper helper, string name, string searchdomain)
{ {
Dictionary<string, dynamic> parameters = new() Dictionary<string, dynamic> parameters = new()
{ {

15
src/Server/Helper/SearchdomainHelper.cs
View File

@@ -7,8 +7,11 @@ using Shared.Models;
namespace Server.Helper; namespace Server.Helper;
public static class SearchdomainHelper public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelper databaseHelper)
{ {
private readonly ILogger<SearchdomainHelper> _logger = logger;
private readonly DatabaseHelper _databaseHelper = databaseHelper;
public static byte[] BytesFromFloatArray(float[] floats) public static byte[] BytesFromFloatArray(float[] floats)
{ {
var byteArray = new byte[floats.Length * 4]; var byteArray = new byte[floats.Length * 4];
@@ -41,7 +44,7 @@ public static class SearchdomainHelper
return null; return null;
} }
public static List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json) public List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json)
{ {
List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json); List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json);
if (jsonEntities is null) if (jsonEntities is null)
@@ -79,10 +82,10 @@ public static class SearchdomainHelper
return [.. retVal]; return [.. retVal];
} }
public static Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json) public Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json)
{ {
Dictionary<string, Dictionary<string, float[]>> embeddingsLUT = []; // embeddingsLUT: hash -> model -> [embeddingValues * n] Dictionary<string, Dictionary<string, float[]>> embeddingsLUT = []; // embeddingsLUT: hash -> model -> [embeddingValues * n]
int? preexistingEntityID = DatabaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain); int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
if (preexistingEntityID is not null) if (preexistingEntityID is not null)
{ {
lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after) lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after)
@@ -108,9 +111,9 @@ public static class SearchdomainHelper
} }
reader.Close(); reader.Close();
} }
DatabaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create _databaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create
} }
int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, DatabaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain)); int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
foreach (KeyValuePair<string, string> attribute in jsonEntity.Attributes) foreach (KeyValuePair<string, string> attribute in jsonEntity.Attributes)
{ {
DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries

3
src/Server/Program.cs
View File

@@ -3,6 +3,7 @@ using ElmahCore.Mvc;
using Serilog; using Serilog;
using Server; using Server;
using Server.HealthChecks; using Server.HealthChecks;
using Server.Helper;
var builder = WebApplication.CreateBuilder(args); var builder = WebApplication.CreateBuilder(args);
@@ -16,6 +17,8 @@ Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(builder.Configuration) .ReadFrom.Configuration(builder.Configuration)
.CreateLogger(); .CreateLogger();
builder.Logging.AddSerilog(); builder.Logging.AddSerilog();
builder.Services.AddSingleton<DatabaseHelper>();
builder.Services.AddSingleton<SearchdomainHelper>();
builder.Services.AddSingleton<SearchdomainManager>(); builder.Services.AddSingleton<SearchdomainManager>();
builder.Services.AddSingleton<AIProvider>(); builder.Services.AddSingleton<AIProvider>();
builder.Services.AddHealthChecks() builder.Services.AddHealthChecks()

7
src/Server/SearchdomainManager.cs
View File

@@ -11,16 +11,18 @@ public class SearchdomainManager
private readonly ILogger<SearchdomainManager> _logger; private readonly ILogger<SearchdomainManager> _logger;
private readonly IConfiguration _config; private readonly IConfiguration _config;
public readonly AIProvider aIProvider; public readonly AIProvider aIProvider;
private readonly DatabaseHelper _databaseHelper;
private readonly string connectionString; private readonly string connectionString;
private MySqlConnection connection; private MySqlConnection connection;
public SQLHelper helper; public SQLHelper helper;
public Dictionary<string, Dictionary<string, float[]>> embeddingCache; public Dictionary<string, Dictionary<string, float[]>> embeddingCache;
public SearchdomainManager(ILogger<SearchdomainManager> logger, IConfiguration config, AIProvider aIProvider) public SearchdomainManager(ILogger<SearchdomainManager> logger, IConfiguration config, AIProvider aIProvider, DatabaseHelper databaseHelper)
{ {
_logger = logger; _logger = logger;
_config = config; _config = config;
this.aIProvider = aIProvider; this.aIProvider = aIProvider;
_databaseHelper = databaseHelper;
embeddingCache = []; embeddingCache = [];
connectionString = _config.GetSection("Embeddingsearch").GetConnectionString("SQL") ?? ""; connectionString = _config.GetSection("Embeddingsearch").GetConnectionString("SQL") ?? "";
connection = new MySqlConnection(connectionString); connection = new MySqlConnection(connectionString);
@@ -96,8 +98,7 @@ public class SearchdomainManager
public int DeleteSearchdomain(string searchdomain) public int DeleteSearchdomain(string searchdomain)
{ {
Searchdomain searchdomain_ = GetSearchdomain(searchdomain); int counter = _databaseHelper.RemoveAllEntities(helper, searchdomain);
int counter = DatabaseHelper.RemoveAllEntities(helper, searchdomain);
_logger.LogDebug($"Number of entities deleted as part of deleting the searchdomain \"{searchdomain}\": {counter}"); _logger.LogDebug($"Number of entities deleted as part of deleting the searchdomain \"{searchdomain}\": {counter}");
helper.ExecuteSQLNonQuery("DELETE FROM searchdomain WHERE name = @name", new() {{"name", searchdomain}}); helper.ExecuteSQLNonQuery("DELETE FROM searchdomain WHERE name = @name", new() {{"name", searchdomain}});
searchdomains.Remove(searchdomain); searchdomains.Remove(searchdomain);