lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved logging for DatabaseHelper

Browse Source
This commit is contained in:
LD-Reborn
2025-12-13 16:52:26 +01:00
parent 1b2050bb86
commit a1ed65ee41
5 changed files with 33 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/Server/Helper/DatabaseHelper.cs
View File

@@ -3,8 +3,10 @@ using System.Text;
namespace Server.Helper;
public static class DatabaseHelper
public class DatabaseHelper(ILogger<DatabaseHelper> logger)
{
private readonly ILogger<DatabaseHelper> _logger = logger;
public static void DatabaseInsertEmbeddingBulk(SQLHelper helper, int id_datapoint, List<(string model, byte[] embedding)> data)
{
Dictionary<string, object> parameters = [];
@@ -80,7 +82,7 @@ public static class DatabaseHelper
return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO embedding (id_datapoint, model, embedding) VALUES (@id_datapoint, @model, @embedding)", parameters);
}
public static int GetSearchdomainID(SQLHelper helper, string searchdomain)
public int GetSearchdomainID(SQLHelper helper, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{
@@ -98,12 +100,13 @@ public static class DatabaseHelper
}
else
{
throw new Exception($"Unable to retrieve searchdomain ID for {searchdomain}"); // TODO implement logging here; add logger via method injection
_logger.LogError("Unable to retrieve searchdomain ID for {searchdomain}", [searchdomain]);
throw new Exception($"Unable to retrieve searchdomain ID for {searchdomain}");
}
}
}
public static void RemoveEntity(List<Entity> entityCache, SQLHelper helper, string name, string searchdomain)
public void RemoveEntity(List<Entity> entityCache, SQLHelper helper, string name, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{
@@ -118,7 +121,7 @@ public static class DatabaseHelper
entityCache.RemoveAll(entity => entity.name == name);
}
public static int RemoveAllEntities(SQLHelper helper, string searchdomain)
public int RemoveAllEntities(SQLHelper helper, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{
@@ -131,7 +134,7 @@ public static class DatabaseHelper
return helper.ExecuteSQLNonQuery("DELETE FROM entity WHERE entity.id_searchdomain = @searchdomain", parameters);
}
public static bool HasEntity(SQLHelper helper, string name, string searchdomain)
public bool HasEntity(SQLHelper helper, string name, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{
@@ -155,7 +158,7 @@ public static class DatabaseHelper
}
}
public static int? GetEntityID(SQLHelper helper, string name, string searchdomain)
public int? GetEntityID(SQLHelper helper, string name, string searchdomain)
{
Dictionary<string, dynamic> parameters = new()
{

15
src/Server/Helper/SearchdomainHelper.cs
View File

@@ -7,8 +7,11 @@ using Shared.Models;
namespace Server.Helper;
public static class SearchdomainHelper
public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelper databaseHelper)
{
private readonly ILogger<SearchdomainHelper> _logger = logger;
private readonly DatabaseHelper _databaseHelper = databaseHelper;
public static byte[] BytesFromFloatArray(float[] floats)
{
var byteArray = new byte[floats.Length * 4];
@@ -41,7 +44,7 @@ public static class SearchdomainHelper
return null;
}
public static List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json)
public List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json)
{
List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json);
if (jsonEntities is null)
@@ -79,10 +82,10 @@ public static class SearchdomainHelper
return [.. retVal];
}
public static Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json)
public Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json)
{
Dictionary<string, Dictionary<string, float[]>> embeddingsLUT = []; // embeddingsLUT: hash -> model -> [embeddingValues * n]
int? preexistingEntityID = DatabaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
if (preexistingEntityID is not null)
{
lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after)
@@ -108,9 +111,9 @@ public static class SearchdomainHelper
}
reader.Close();
}
DatabaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create
_databaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create
}
int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, DatabaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
foreach (KeyValuePair<string, string> attribute in jsonEntity.Attributes)
{
DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries