Merge pull request #119 from LD-Reborn/116-add-bulk-attributes-insert

116 add bulk attributes insert

src/Server/Controllers/EntityController.cs

@@ -116,12 +116,14 @@ public class EntityController : ControllerBase
             else
             {
                 _logger.LogError("Unable to deserialize an entity");
+                ElmahCore.ElmahExtensions.RaiseError(new Exception("Unable to deserialize an entity"));
                 return Ok(new EntityIndexResult() { Success = false, Message = "Unable to deserialize an entity"});
             }
         } catch (Exception ex)
         {
             if (ex.InnerException is not null) ex = ex.InnerException;
             _logger.LogError("Unable to index the provided entities. {ex.Message} - {ex.StackTrace}", [ex.Message, ex.StackTrace]);
+            ElmahCore.ElmahExtensions.RaiseError(ex);
             return Ok(new EntityIndexResult() { Success = false, Message = ex.Message });
         }
 
@@ -142,6 +144,11 @@ public class EntityController : ControllerBase
         if (entity_ is null)
         {
             _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]);
+            ElmahCore.ElmahExtensions.RaiseError(
+                new Exception(
+                    $"Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name"
+                )
+            );
             return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"});
         }
         searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity_);

src/Server/Datapoint.cs

@@ -1,6 +1,3 @@
-using AdaptiveExpressions;
-using OllamaSharp;
-using OllamaSharp.Models;
 using Shared;
 
 namespace Server;
@@ -80,6 +77,10 @@ public class Datapoint
                     }
                 }
             }
+            if (toBeGenerated.Count == 0)
+            {
+                continue;
+            }
             IEnumerable<float[]> generatedEmbeddings = GenerateEmbeddings([.. toBeGenerated], model, aIProvider, embeddingCache);
             if (generatedEmbeddings.Count() != toBeGenerated.Count)
             {

src/Server/Helper/DatabaseHelper.cs

@@ -1,4 +1,3 @@
-using System.Configuration;
 using System.Data.Common;
 using System.Text;
 using System.Text.Json;
@@ -72,6 +71,18 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
         return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO attribute (attribute, value, id_entity) VALUES (@attribute, @value, @id_entity)", parameters);
     }
 
+    public static int DatabaseInsertAttributes(SQLHelper helper, List<(string attribute, string value, int id_entity)> values) //string[] attribute, string value, int id_entity)
+    {
+        return helper.BulkExecuteNonQuery(
+            "INSERT INTO attribute (attribute, value, id_entity) VALUES (@attribute, @value, @id_entity)",
+            values.Select(element => new object[] {
+                new MySqlParameter("@attribute", element.attribute),
+                new MySqlParameter("@value", element.value),
+                new MySqlParameter("@id_entity", element.id_entity)
+            })
+        );
+    }
+
     public static int DatabaseInsertDatapoint(SQLHelper helper, string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash, int id_entity)
     {
         Dictionary<string, dynamic> parameters = new()

src/Server/Helper/SQLHelper.cs

@@ -80,6 +80,33 @@ public class SQLHelper:IDisposable
         }
     }
 
+    public int BulkExecuteNonQuery(string sql, IEnumerable<object[]> parameterSets)
+    {
+        lock (connection)
+        {
+            EnsureConnected();
+            EnsureDbReaderIsClosed();
+
+            using var transaction = connection.BeginTransaction();
+            using var command = connection.CreateCommand();
+
+            command.CommandText = sql;
+            command.Transaction = transaction;
+
+            int affectedRows = 0;
+
+            foreach (var parameters in parameterSets)
+            {
+                command.Parameters.Clear();
+                command.Parameters.AddRange(parameters);
+                affectedRows += command.ExecuteNonQuery();
+            }
+
+            transaction.Commit();
+            return affectedRows;
+        }
+    }
+
     public bool EnsureConnected()
     {
         if (connection.State != System.Data.ConnectionState.Open)

src/Server/Helper/SearchdomainHelper.cs

@@ -245,10 +245,16 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
         else
         {
             int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
+            List<(string attribute, string value, int id_entity)> values = [];
             foreach (KeyValuePair<string, string> attribute in jsonEntity.Attributes)
             {
-                DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries
+                values.Add(new() {
+                    attribute = attribute.Key,
+                    value = attribute.Value,
+                    id_entity = id_entity
+                });
             }
+            DatabaseHelper.DatabaseInsertAttributes(helper, values);
 
             List<Datapoint> datapoints = [];
             foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)

src/Server/Searchdomain.cs

@@ -219,7 +219,7 @@ public class Searchdomain
 
     public void UpdateModelsInUse()
     {
-        modelsInUse = GetModels([.. entityCache]);
+        modelsInUse = GetModels(entityCache.ToList());
     }
 
     private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary<string, float[]> queryEmbeddings)