From 329af1c103dd4fb8452f5184637c0443fe1e19ef Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Wed, 28 Jan 2026 22:21:51 +0100 Subject: [PATCH] Added datapoint bulk insert --- src/Server/Helper/DatabaseHelper.cs | 29 ++++++++++++++- src/Server/Helper/SearchdomainHelper.cs | 49 ++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/src/Server/Helper/DatabaseHelper.cs b/src/Server/Helper/DatabaseHelper.cs index c86808f..b543347 100644 --- a/src/Server/Helper/DatabaseHelper.cs +++ b/src/Server/Helper/DatabaseHelper.cs @@ -39,6 +39,19 @@ public class DatabaseHelper(ILogger logger) helper.ExecuteSQLNonQuery(query.ToString(), parameters); } + public static int DatabaseInsertEmbeddingBulk(SQLHelper helper, List<(string hash, string model, byte[] embedding)> data) + { + return helper.BulkExecuteNonQuery( + "INSERT INTO embedding (id_datapoint, model, embedding) SELECT d.id, @model, @embedding FROM datapoint d WHERE d.hash = @hash", + data.Select(element => new object[] { + new MySqlParameter("@model", element.model), + new MySqlParameter("@embedding", element.embedding), + new MySqlParameter("@hash", element.hash) + }) + ); + } + + public static int DatabaseInsertSearchdomain(SQLHelper helper, string name, SearchdomainSettings settings = new()) { Dictionary parameters = new() @@ -83,6 +96,20 @@ public class DatabaseHelper(ILogger logger) ); } + public static int DatabaseInsertDatapoints(SQLHelper helper, List<(string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash)> values, int id_entity) + { + return helper.BulkExecuteNonQuery( + "INSERT INTO datapoint (name, probmethod_embedding, similaritymethod, hash, id_entity) VALUES (@name, @probmethod_embedding, @similaritymethod, @hash, @id_entity)", + values.Select(element => new object[] { + new MySqlParameter("@name", element.name), + new MySqlParameter("@probmethod_embedding", element.probmethod_embedding), + new MySqlParameter("@similaritymethod", element.similarityMethod), + new MySqlParameter("@hash", element.hash), + new MySqlParameter("@id_entity", id_entity) + }) + ); + } + public static int DatabaseInsertDatapoint(SQLHelper helper, string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash, int id_entity) { Dictionary parameters = new() @@ -155,7 +182,7 @@ public class DatabaseHelper(ILogger logger) helper.ExecuteSQLNonQuery("DELETE embedding.* FROM embedding JOIN datapoint dp ON id_datapoint = dp.id JOIN entity ON id_entity = entity.id WHERE entity.id_searchdomain = @searchdomain", parameters); helper.ExecuteSQLNonQuery("DELETE datapoint.* FROM datapoint JOIN entity ON id_entity = entity.id WHERE entity.id_searchdomain = @searchdomain", parameters); - helper.ExecuteSQLNonQuery("DELETE attribute.* FROM attribute JOIN entity ON id_entity = entity.id WHERE entity.id_searchdomain = @searchdomain", parameters); + helper.ExecuteSQLNonQuery("DELETE FROM attribute WHERE id_entity IN (SELECT entity.id FROM entity WHERE id_searchdomain = @searchdomain)", parameters); return helper.ExecuteSQLNonQuery("DELETE FROM entity WHERE entity.id_searchdomain = @searchdomain", parameters); } diff --git a/src/Server/Helper/SearchdomainHelper.cs b/src/Server/Helper/SearchdomainHelper.cs index 0b307f0..ae2a553 100644 --- a/src/Server/Helper/SearchdomainHelper.cs +++ b/src/Server/Helper/SearchdomainHelper.cs @@ -245,25 +245,30 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp else { int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain)); - List<(string attribute, string value, int id_entity)> values = []; + List<(string attribute, string value, int id_entity)> toBeInsertedAttributes = []; foreach (KeyValuePair attribute in jsonEntity.Attributes) { - values.Add(new() { + toBeInsertedAttributes.Add(new() { attribute = attribute.Key, value = attribute.Value, id_entity = id_entity }); } - DatabaseHelper.DatabaseInsertAttributes(helper, values); + DatabaseHelper.DatabaseInsertAttributes(helper, toBeInsertedAttributes); List datapoints = []; + List<(JSONDatapoint datapoint, string hash)> toBeInsertedDatapoints = []; foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) { string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); - Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, id_entity, hash); - datapoints.Add(datapoint); + toBeInsertedDatapoints.Add(new() + { + datapoint = jsonDatapoint, + hash = hash + }); } - + List datapoint = DatabaseInsertDatapointsWithEmbeddings(helper, searchdomain, toBeInsertedDatapoints, id_entity); + var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod); Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod.ToString(), datapoints, jsonEntity.Name) { @@ -276,6 +281,38 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp } } + public List DatabaseInsertDatapointsWithEmbeddings(SQLHelper helper, Searchdomain searchdomain, List<(JSONDatapoint datapoint, string hash)> values, int id_entity) + { + List result = []; + List<(string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash)> toBeInsertedDatapoints = []; + List<(string hash, string model, byte[] embedding)> toBeInsertedEmbeddings = []; + foreach ((JSONDatapoint datapoint, string hash) value in values) + { + Datapoint datapoint = BuildDatapointFromJsonDatapoint(value.datapoint, id_entity, searchdomain, value.hash); + toBeInsertedDatapoints.Add(new() + { + name = datapoint.name, + probmethod_embedding = datapoint.probMethod.probMethodEnum, + similarityMethod = datapoint.similarityMethod.similarityMethodEnum, + hash = value.hash + }); + foreach ((string, float[]) embedding in datapoint.embeddings) + { + toBeInsertedEmbeddings.Add(new() + { + hash = value.hash, + model = embedding.Item1, + embedding = BytesFromFloatArray(embedding.Item2) + }); + } + result.Add(datapoint); + } + + DatabaseHelper.DatabaseInsertDatapoints(helper, toBeInsertedDatapoints, id_entity); + DatabaseHelper.DatabaseInsertEmbeddingBulk(helper, toBeInsertedEmbeddings); + return result; + } + public Datapoint DatabaseInsertDatapointWithEmbeddings(SQLHelper helper, Searchdomain searchdomain, JSONDatapoint jsonDatapoint, int id_entity, string? hash = null) { if (jsonDatapoint.Text is null)