From 7d16f90c718d5f359a24a9e852c39a47ead21546 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sun, 25 Jan 2026 16:32:37 +0100 Subject: [PATCH 1/5] Added bulk attributes insert --- src/Server/Helper/DatabaseHelper.cs | 13 ++++++++++++- src/Server/Helper/SearchdomainHelper.cs | 8 +++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/Server/Helper/DatabaseHelper.cs b/src/Server/Helper/DatabaseHelper.cs index 66a031e..c86808f 100644 --- a/src/Server/Helper/DatabaseHelper.cs +++ b/src/Server/Helper/DatabaseHelper.cs @@ -1,4 +1,3 @@ -using System.Configuration; using System.Data.Common; using System.Text; using System.Text.Json; @@ -72,6 +71,18 @@ public class DatabaseHelper(ILogger logger) return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO attribute (attribute, value, id_entity) VALUES (@attribute, @value, @id_entity)", parameters); } + public static int DatabaseInsertAttributes(SQLHelper helper, List<(string attribute, string value, int id_entity)> values) //string[] attribute, string value, int id_entity) + { + return helper.BulkExecuteNonQuery( + "INSERT INTO attribute (attribute, value, id_entity) VALUES (@attribute, @value, @id_entity)", + values.Select(element => new object[] { + new MySqlParameter("@attribute", element.attribute), + new MySqlParameter("@value", element.value), + new MySqlParameter("@id_entity", element.id_entity) + }) + ); + } + public static int DatabaseInsertDatapoint(SQLHelper helper, string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash, int id_entity) { Dictionary parameters = new() diff --git a/src/Server/Helper/SearchdomainHelper.cs b/src/Server/Helper/SearchdomainHelper.cs index 0ea2ff9..0b307f0 100644 --- a/src/Server/Helper/SearchdomainHelper.cs +++ b/src/Server/Helper/SearchdomainHelper.cs @@ -245,10 +245,16 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp else { int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain)); + List<(string attribute, string value, int id_entity)> values = []; foreach (KeyValuePair attribute in jsonEntity.Attributes) { - DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries + values.Add(new() { + attribute = attribute.Key, + value = attribute.Value, + id_entity = id_entity + }); } + DatabaseHelper.DatabaseInsertAttributes(helper, values); List datapoints = []; foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) From c0189016e8edb5218d1e82cc85ed59f1e2c55953 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sun, 25 Jan 2026 16:32:54 +0100 Subject: [PATCH 2/5] Improved logging in EntityController --- src/Server/Controllers/EntityController.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Server/Controllers/EntityController.cs b/src/Server/Controllers/EntityController.cs index b1a9406..9fbfc53 100644 --- a/src/Server/Controllers/EntityController.cs +++ b/src/Server/Controllers/EntityController.cs @@ -116,12 +116,14 @@ public class EntityController : ControllerBase else { _logger.LogError("Unable to deserialize an entity"); + ElmahCore.ElmahExtensions.RaiseError(new Exception("Unable to deserialize an entity")); return Ok(new EntityIndexResult() { Success = false, Message = "Unable to deserialize an entity"}); } } catch (Exception ex) { if (ex.InnerException is not null) ex = ex.InnerException; _logger.LogError("Unable to index the provided entities. {ex.Message} - {ex.StackTrace}", [ex.Message, ex.StackTrace]); + ElmahCore.ElmahExtensions.RaiseError(ex); return Ok(new EntityIndexResult() { Success = false, Message = ex.Message }); } @@ -142,6 +144,11 @@ public class EntityController : ControllerBase if (entity_ is null) { _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]); + ElmahCore.ElmahExtensions.RaiseError( + new Exception( + $"Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name" + ) + ); return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"}); } searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity_); From 01b0934d6ea290d3e79a3f8f89f180d9ee8ddcf4 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sun, 25 Jan 2026 16:33:49 +0100 Subject: [PATCH 3/5] Removed unused using statements, added early return for GetEmbeddings --- src/Server/Datapoint.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Server/Datapoint.cs b/src/Server/Datapoint.cs index ee17c36..c4c742d 100644 --- a/src/Server/Datapoint.cs +++ b/src/Server/Datapoint.cs @@ -1,6 +1,3 @@ -using AdaptiveExpressions; -using OllamaSharp; -using OllamaSharp.Models; using Shared; namespace Server; @@ -80,6 +77,10 @@ public class Datapoint } } } + if (toBeGenerated.Count == 0) + { + continue; + } IEnumerable generatedEmbeddings = GenerateEmbeddings([.. toBeGenerated], model, aIProvider, embeddingCache); if (generatedEmbeddings.Count() != toBeGenerated.Count) { From a9dada01c008b28f28b60cd5e5de1fa5ed74d023 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sun, 25 Jan 2026 16:34:18 +0100 Subject: [PATCH 4/5] Added missing BulkExecuteNonQuery --- src/Server/Helper/SQLHelper.cs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/Server/Helper/SQLHelper.cs b/src/Server/Helper/SQLHelper.cs index c1dc295..de7e276 100644 --- a/src/Server/Helper/SQLHelper.cs +++ b/src/Server/Helper/SQLHelper.cs @@ -80,6 +80,33 @@ public class SQLHelper:IDisposable } } + public int BulkExecuteNonQuery(string sql, IEnumerable parameterSets) + { + lock (connection) + { + EnsureConnected(); + EnsureDbReaderIsClosed(); + + using var transaction = connection.BeginTransaction(); + using var command = connection.CreateCommand(); + + command.CommandText = sql; + command.Transaction = transaction; + + int affectedRows = 0; + + foreach (var parameters in parameterSets) + { + command.Parameters.Clear(); + command.Parameters.AddRange(parameters); + affectedRows += command.ExecuteNonQuery(); + } + + transaction.Commit(); + return affectedRows; + } + } + public bool EnsureConnected() { if (connection.State != System.Data.ConnectionState.Open) From 7fffd74f26a9c4cc942699f0010662e78acd33c9 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sun, 25 Jan 2026 16:35:02 +0100 Subject: [PATCH 5/5] Changed list conversion for entityCache --- src/Server/Searchdomain.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/Searchdomain.cs b/src/Server/Searchdomain.cs index 70dd895..83e6e44 100644 --- a/src/Server/Searchdomain.cs +++ b/src/Server/Searchdomain.cs @@ -219,7 +219,7 @@ public class Searchdomain public void UpdateModelsInUse() { - modelsInUse = GetModels([.. entityCache]); + modelsInUse = GetModels(entityCache.ToList()); } private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary queryEmbeddings)