diff --git a/src/Client/Client.cs b/src/Client/Client.cs index 10ff796..fc88d56 100644 --- a/src/Client/Client.cs +++ b/src/Client/Client.cs @@ -88,22 +88,12 @@ public class Client public async Task EntityIndexAsync(List jsonEntity) { - return await EntityIndexAsync(searchdomain, jsonEntity); - } - - public async Task EntityIndexAsync(string searchdomain, List jsonEntity) - { - return await EntityIndexAsync(searchdomain, JsonSerializer.Serialize(jsonEntity)); + return await EntityIndexAsync(JsonSerializer.Serialize(jsonEntity)); } public async Task EntityIndexAsync(string jsonEntity) { - return await EntityIndexAsync(searchdomain, jsonEntity); - } - - public async Task EntityIndexAsync(string searchdomain, string jsonEntity) - { - var url = $"{baseUri}/Entity/Index?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}"; + var url = $"{baseUri}/Entity/Index?apiKey={HttpUtility.UrlEncode(apiKey)}"; var content = new StringContent(jsonEntity, Encoding.UTF8, "application/json"); return await PostUrlAndProcessJson(url, content);//new FormUrlEncodedContent(values)); } diff --git a/src/Indexer/Services/IndexerService.cs b/src/Indexer/Services/IndexerService.cs index f8c08a5..0d49f9a 100644 --- a/src/Indexer/Services/IndexerService.cs +++ b/src/Indexer/Services/IndexerService.cs @@ -32,10 +32,6 @@ public class IndexerService : IHostedService foreach (WorkerConfig workerConfig in sectionWorker.Worker) { _logger.LogInformation("Initializing worker: {Name}", workerConfig.Name); - if (client.searchdomain == "" && workerConfig.Searchdomains.Count >= 1) - { - client.searchdomain = workerConfig.Searchdomains.First(); - } ScriptToolSet toolSet = new(workerConfig.Script, client); Worker worker = new(workerConfig.Name, workerConfig, GetScriptable(toolSet)); workerCollection.Workers.Add(worker); diff --git a/src/Server/Controllers/EntityController.cs b/src/Server/Controllers/EntityController.cs index 50939f2..62336d6 100644 --- a/src/Server/Controllers/EntityController.cs +++ b/src/Server/Controllers/EntityController.cs @@ -41,22 +41,28 @@ public class EntityController : ControllerBase } [HttpPost("Index")] - public ActionResult Index(string searchdomain, [FromBody] List? jsonEntity) + public ActionResult Index([FromBody] List? jsonEntities) { - Searchdomain searchdomain_; - try + List? entities = SearchdomainHelper.EntitiesFromJSON( + [], + _domainManager.embeddingCache, + _domainManager.client, + _domainManager.helper, + JsonSerializer.Serialize(jsonEntities)); + if (entities is not null && jsonEntities is not null) { - searchdomain_ = _domainManager.GetSearchdomain(searchdomain); - } - catch (Exception) - { - _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]); - return Ok(new EntityIndexResult() { Success = false }); - } - List? entities = searchdomain_.EntitiesFromJSON(JsonSerializer.Serialize(jsonEntity)); - if (entities is not null) - { - _domainManager.InvalidateSearchdomainCache(searchdomain); + List invalidatedSearchdomains = []; + foreach (var jsonEntity in jsonEntities) + { + string jsonEntityName = jsonEntity.Name; + if (entities.Select(x => x.name == jsonEntityName).Any() + && !invalidatedSearchdomains.Contains(jsonEntityName)) + { + string jsonEntitySearchdomain = jsonEntity.Searchdomain; + invalidatedSearchdomains.Add(jsonEntitySearchdomain); + _domainManager.InvalidateSearchdomainCache(jsonEntitySearchdomain); + } + } return Ok(new EntityIndexResult() { Success = true }); } else @@ -118,22 +124,13 @@ public class EntityController : ControllerBase [HttpGet("Delete")] public ActionResult Delete(string searchdomain, string entityName) { - Searchdomain searchdomain_; - try - { - searchdomain_ = _domainManager.GetSearchdomain(searchdomain); - } catch (Exception) - { - _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - the searchdomain likely does not exist", [entityName, searchdomain]); - return Ok(new EntityDeleteResults() {Success = false}); - } - Entity? entity_ = searchdomain_.GetEntity(entityName); + Entity? entity_ = SearchdomainHelper.CacheGetEntity([], entityName); if (entity_ is null) { _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]); return Ok(new EntityDeleteResults() {Success = false}); } - searchdomain_.RemoveEntity(entityName); + DatabaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain); return Ok(new EntityDeleteResults() {Success = true}); } } diff --git a/src/Server/Controllers/SearchdomainController.cs b/src/Server/Controllers/SearchdomainController.cs index e682b1f..784f01c 100644 --- a/src/Server/Controllers/SearchdomainController.cs +++ b/src/Server/Controllers/SearchdomainController.cs @@ -1,3 +1,4 @@ +using ElmahCore; using Microsoft.AspNetCore.Mvc; using Server.Models; @@ -58,11 +59,13 @@ public class SearchdomainController : ControllerBase { success = true; deletedEntries = _domainManager.DeleteSearchdomain(searchdomain); - } catch (Exception) + } + catch (Exception ex) { _logger.LogError("Unable to delete searchdomain {searchdomain}", [searchdomain]); success = false; deletedEntries = 0; + ElmahExtensions.RaiseError(ex); } return Ok(new SearchdomainDeleteResults(){Success = success, DeletedEntities = deletedEntries}); } diff --git a/src/Server/Helper/DatabaseHelper.cs b/src/Server/Helper/DatabaseHelper.cs new file mode 100644 index 0000000..668af71 --- /dev/null +++ b/src/Server/Helper/DatabaseHelper.cs @@ -0,0 +1,160 @@ +using System.Data.Common; +using System.Text; + +namespace Server; + +public static class DatabaseHelper +{ + public static void DatabaseInsertEmbeddingBulk(SQLHelper helper, int id_datapoint, List<(string model, byte[] embedding)> data) + { + Dictionary parameters = []; + parameters["id_datapoint"] = id_datapoint; + var query = new StringBuilder("INSERT INTO embedding (id_datapoint, model, embedding) VALUES "); + foreach (var (model, embedding) in data) + { + string modelParam = $"model_{Guid.NewGuid()}".Replace("-", ""); + string embeddingParam = $"embedding_{Guid.NewGuid()}".Replace("-", ""); + parameters[modelParam] = model; + parameters[embeddingParam] = embedding; + + query.Append($"(@id_datapoint, @{modelParam}, @{embeddingParam}), "); + } + + query.Length -= 2; // remove trailing comma + helper.ExecuteSQLNonQuery(query.ToString(), parameters); + } + + public static int DatabaseInsertSearchdomain(SQLHelper helper, string name) + { + Dictionary parameters = new() + { + { "name", name }, + { "settings", "{}"} // TODO add settings. It's not used yet, but maybe it's needed someday... + }; + return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO searchdomain (name, settings) VALUES (@name, @settings)", parameters); + } + + public static int DatabaseInsertEntity(SQLHelper helper, string name, string probmethod, int id_searchdomain) + { + Dictionary parameters = new() + { + { "name", name }, + { "probmethod", probmethod }, + { "id_searchdomain", id_searchdomain } + }; + return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO entity (name, probmethod, id_searchdomain) VALUES (@name, @probmethod, @id_searchdomain)", parameters); + } + + public static int DatabaseInsertAttribute(SQLHelper helper, string attribute, string value, int id_entity) + { + Dictionary parameters = new() + { + { "attribute", attribute }, + { "value", value }, + { "id_entity", id_entity } + }; + return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO attribute (attribute, value, id_entity) VALUES (@attribute, @value, @id_entity)", parameters); + } + + public static int DatabaseInsertDatapoint(SQLHelper helper, string name, string probmethod_embedding, string hash, int id_entity) + { + Dictionary parameters = new() + { + { "name", name }, + { "probmethod_embedding", probmethod_embedding }, + { "hash", hash }, + { "id_entity", id_entity } + }; + return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO datapoint (name, probmethod_embedding, hash, id_entity) VALUES (@name, @probmethod_embedding, @hash, @id_entity)", parameters); + } + + public static int DatabaseInsertEmbedding(SQLHelper helper, int id_datapoint, string model, byte[] embedding) + { + Dictionary parameters = new() + { + { "id_datapoint", id_datapoint }, + { "model", model }, + { "embedding", embedding } + }; + return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO embedding (id_datapoint, model, embedding) VALUES (@id_datapoint, @model, @embedding)", parameters); + } + + public static int GetSearchdomainID(SQLHelper helper, string searchdomain) + { + Dictionary parameters = new() + { + { "searchdomain", searchdomain} + }; + lock (helper.connection) + { + DbDataReader reader = helper.ExecuteSQLCommand("SELECT id FROM searchdomain WHERE name = @searchdomain", parameters); + bool success = reader.Read(); + int result = success ? reader.GetInt32(0) : 0; + reader.Close(); + if (success) + { + return result; + } + else + { + throw new Exception($"Unable to retrieve searchdomain ID for {searchdomain}"); // TODO implement logging here; add logger via method injection + } + } + } + + public static void RemoveEntity(List entityCache, SQLHelper helper, string name, string searchdomain) + { + Dictionary parameters = new() + { + { "name", name }, + { "searchdomain", GetSearchdomainID(helper, searchdomain)} + }; + + helper.ExecuteSQLNonQuery("DELETE embedding.* FROM embedding JOIN datapoint dp ON id_datapoint = dp.id JOIN entity ON id_entity = entity.id WHERE entity.name = @name AND entity.id_searchdomain = @searchdomain", parameters); + helper.ExecuteSQLNonQuery("DELETE datapoint.* FROM datapoint JOIN entity ON id_entity = entity.id WHERE entity.name = @name AND entity.id_searchdomain = @searchdomain", parameters); + helper.ExecuteSQLNonQuery("DELETE attribute.* FROM attribute JOIN entity ON id_entity = entity.id WHERE entity.name = @name AND entity.id_searchdomain = @searchdomain", parameters); + helper.ExecuteSQLNonQuery("DELETE FROM entity WHERE name = @name AND entity.id_searchdomain = @searchdomain", parameters); + entityCache.RemoveAll(entity => entity.name == name); + } + + public static bool HasEntity(SQLHelper helper, string name, string searchdomain) + { + Dictionary parameters = new() + { + { "name", name }, + { "searchdomain", GetSearchdomainID(helper, searchdomain)} + }; + lock (helper.connection) + { + DbDataReader reader = helper.ExecuteSQLCommand("SELECT COUNT(*) FROM entity WHERE name = @name AND id_searchdomain = @searchdomain", parameters); + bool success = reader.Read(); + bool result = success && reader.GetInt32(0) > 0; + reader.Close(); + if (success) + { + return result; + } + else + { + throw new Exception($"Unable to determine whether an entity named {name} exists for {searchdomain}"); // TODO implement logging here; add logger via method injection + } + } + } + + public static int? GetEntityID(SQLHelper helper, string name, string searchdomain) + { + Dictionary parameters = new() + { + { "name", name }, + { "searchdomain", GetSearchdomainID(helper, searchdomain)} + }; + lock (helper.connection) + { + DbDataReader reader = helper.ExecuteSQLCommand("SELECT id FROM entity WHERE name = @name AND id_searchdomain = @searchdomain", parameters); + bool success = reader.Read(); + int? result = success ? reader.GetInt32(0) : 0; + reader.Close(); + return result; + } + } +} \ No newline at end of file diff --git a/src/Server/Helper/SQLHelper.cs b/src/Server/Helper/SQLHelper.cs index 9d04d03..3e842cb 100644 --- a/src/Server/Helper/SQLHelper.cs +++ b/src/Server/Helper/SQLHelper.cs @@ -6,10 +6,19 @@ namespace Server; public class SQLHelper { public MySqlConnection connection; - public SQLHelper(MySqlConnection connection) + public string connectionString; + public SQLHelper(MySqlConnection connection, string connectionString) { this.connection = connection; + this.connectionString = connectionString; } + + public SQLHelper DuplicateConnection() + { + MySqlConnection newConnection = new(connectionString); + return new SQLHelper(newConnection, connectionString); + } + public DbDataReader ExecuteSQLCommand(string query, Dictionary parameters) { lock (connection) diff --git a/src/Server/Helper/SearchdomainHelper.cs b/src/Server/Helper/SearchdomainHelper.cs new file mode 100644 index 0000000..1486d78 --- /dev/null +++ b/src/Server/Helper/SearchdomainHelper.cs @@ -0,0 +1,146 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using MySql.Data.MySqlClient; +using OllamaSharp; + +namespace Server; + +public static class SearchdomainHelper +{ + public static byte[] BytesFromFloatArray(float[] floats) + { + var byteArray = new byte[floats.Length * 4]; + var floatArray = floats.ToArray(); + Buffer.BlockCopy(floatArray, 0, byteArray, 0, byteArray.Length); + return byteArray; + } + + public static float[] FloatArrayFromBytes(byte[] bytes) + { + var floatArray = new float[bytes.Length / 4]; + Buffer.BlockCopy(bytes, 0, floatArray, 0, bytes.Length); + return floatArray; + } + + public static bool CacheHasEntity(List entityCache, string name) + { + return CacheGetEntity(entityCache, name) is not null; + } + + public static Entity? CacheGetEntity(List entityCache, string name) + { + foreach (Entity entity in entityCache) + { + if (entity.name == name) + { + return entity; + } + } + return null; + } + + public static List? EntitiesFromJSON(List entityCache, Dictionary> embeddingCache, OllamaApiClient ollama, SQLHelper helper, string json) + { + List? jsonEntities = JsonSerializer.Deserialize>(json); + if (jsonEntities is null) + { + return null; + } + + Dictionary> toBeCached = []; + foreach (JSONEntity jSONEntity in jsonEntities) + { + foreach (JSONDatapoint datapoint in jSONEntity.Datapoints) + { + foreach (string model in datapoint.Model) + { + if (!toBeCached.ContainsKey(model)) + { + toBeCached[model] = []; + } + toBeCached[model].Add(datapoint.Text); + } + } + } + ConcurrentQueue retVal = []; + Parallel.ForEach(jsonEntities, jSONEntity => + { + var tempHelper = helper.DuplicateConnection(); + var entity = EntityFromJSON(entityCache, embeddingCache, ollama, tempHelper, jSONEntity); + if (entity is not null) + { + retVal.Enqueue(entity); + } + }); + return [.. retVal]; + } + + public static Entity? EntityFromJSON(List entityCache, Dictionary> embeddingCache, OllamaApiClient ollama, SQLHelper helper, JSONEntity jsonEntity) //string json) + { + Dictionary> embeddingsLUT = []; + int? preexistingEntityID = DatabaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain); + if (preexistingEntityID is not null) + { + lock (helper.connection) + { + Dictionary parameters = new() + { + { "id", preexistingEntityID } + }; + System.Data.Common.DbDataReader reader = helper.ExecuteSQLCommand("SELECT e.model, e.embedding, d.hash FROM datapoint d JOIN embedding e ON d.id = e.id_datapoint WHERE d.id_entity = @id", parameters); + while (reader.Read()) + { + string model = reader.GetString(0); + long length = reader.GetBytes(1, 0, null, 0, 0); + byte[] embeddingBytes = new byte[length]; + reader.GetBytes(1, 0, embeddingBytes, 0, (int)length); + float[] embeddingValues = FloatArrayFromBytes(embeddingBytes); + string hash = reader.GetString(2); + if (!embeddingsLUT.ContainsKey(hash)) + { + embeddingsLUT[hash] = []; + } + embeddingsLUT[hash].TryAdd(model, embeddingValues); + } + reader.Close(); + } + DatabaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create + } + int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, DatabaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain)); + foreach (KeyValuePair attribute in jsonEntity.Attributes) + { + DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries + } + + List datapoints = []; + foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) + { + string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); + Dictionary embeddings = embeddingsLUT.ContainsKey(hash) ? embeddingsLUT[hash] : []; + if (embeddings.Count == 0) + { + embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], ollama, embeddingCache); + } + var probMethod_embedding = Probmethods.GetMethod(jsonDatapoint.Probmethod_embedding) ?? throw new Exception($"Unknown probmethod name {jsonDatapoint.Probmethod_embedding}"); + Datapoint datapoint = new(jsonDatapoint.Name, probMethod_embedding, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]); + int id_datapoint = DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, hash, id_entity); // TODO make this a bulk add action to reduce number of queries + List<(string model, byte[] embedding)> data = []; + foreach ((string, float[]) embedding in datapoint.embeddings) + { + data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2))); + } + DatabaseHelper.DatabaseInsertEmbeddingBulk(helper, id_datapoint, data); + datapoints.Add(datapoint); + } + + var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new Exception($"Unknown probmethod name {jsonEntity.Probmethod}"); + Entity entity = new(jsonEntity.Attributes, probMethod, datapoints, jsonEntity.Name) + { + id = id_entity + }; + entityCache.Add(entity); + return entity; + } +} \ No newline at end of file diff --git a/src/Server/Searchdomain.cs b/src/Server/Searchdomain.cs index babc300..30260ea 100644 --- a/src/Server/Searchdomain.cs +++ b/src/Server/Searchdomain.cs @@ -41,18 +41,18 @@ public class Searchdomain // TODO Add settings and update cli/program.cs, as well as DatabaseInsertSearchdomain() - public Searchdomain(string searchdomain, string connectionString, OllamaApiClient ollama, string provider = "sqlserver", bool runEmpty = false) + public Searchdomain(string searchdomain, string connectionString, OllamaApiClient ollama, Dictionary> embeddingCache, string provider = "sqlserver", bool runEmpty = false) { _connectionString = connectionString; _provider = provider.ToLower(); this.searchdomain = searchdomain; this.ollama = ollama; + this.embeddingCache = embeddingCache; searchCache = []; entityCache = []; - embeddingCache = []; connection = new MySqlConnection(connectionString); connection.Open(); - helper = new SQLHelper(connection); + helper = new SQLHelper(connection, connectionString); modelsInUse = []; // To make the compiler shut up - it is set in UpdateSearchDomain() don't worry // yeah, about that... if (!runEmpty) { @@ -78,13 +78,13 @@ public class Searchdomain embeddingReader.GetBytes(3, 0, embedding, 0, (int) length); if (embedding_unassigned.TryGetValue(id_datapoint, out Dictionary? embedding_unassigned_id_datapoint)) { - embedding_unassigned[id_datapoint][model] = FloatArrayFromBytes(embedding); + embedding_unassigned[id_datapoint][model] = SearchdomainHelper.FloatArrayFromBytes(embedding); } else { embedding_unassigned[id_datapoint] = new() { - [model] = FloatArrayFromBytes(embedding) + [model] = SearchdomainHelper.FloatArrayFromBytes(embedding) }; } } @@ -219,218 +219,4 @@ public class Searchdomain reader.Close(); return this.id; } - - public static float[] FloatArrayFromBytes(byte[] bytes) - { - var floatArray = new float[bytes.Length / 4]; - Buffer.BlockCopy(bytes, 0, floatArray, 0, bytes.Length); - return floatArray; - } - - public static byte[] BytesFromFloatArray(float[] floats) - { - var byteArray = new byte[floats.Length * 4]; - var floatArray = floats.ToArray(); - Buffer.BlockCopy(floatArray, 0, byteArray, 0, byteArray.Length); - return byteArray; - } - - public Entity? GetEntity(string name) - { - foreach (Entity entity in entityCache) - { - if (entity.name == name) - { - return entity; - } - } - return null; - } - - public bool HasEntity(string name) - { - return GetEntity(name) is not null; - } - - public Entity? EntityFromJSON(string json) - { - JSONEntity? jsonEntity = JsonSerializer.Deserialize(json); - if (jsonEntity is null) - { - return null; - } - bool hasPreexistingEntity = HasEntity(jsonEntity.Name); - Entity? preexistingEntity = null; - if (hasPreexistingEntity) - { - preexistingEntity = GetEntity(jsonEntity.Name); - RemoveEntity(jsonEntity.Name); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create - } - int id_entity = DatabaseInsertEntity(jsonEntity.Name, jsonEntity.Probmethod, id); - foreach (KeyValuePair attribute in jsonEntity.Attributes) - { - DatabaseInsertAttribute(attribute.Key, attribute.Value, id_entity); - } - - List datapoints = []; - foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) - { - Dictionary embeddings = []; - string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); - if (hasPreexistingEntity && preexistingEntity is not null) - { - IEnumerable preexistingDatapoints = preexistingEntity.datapoints.Where(x => x.name == jsonDatapoint.Name && x.hash == hash); - if (preexistingDatapoints.Any()) - { - var preexistingDatapoint = preexistingDatapoints.First(); - embeddings = preexistingDatapoint.embeddings.ToDictionary(item => item.Item1, item => item.Item2); - } - } - if (embeddings.Count == 0) - { - embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], ollama, embeddingCache); - } - var probMethod_embedding = Probmethods.GetMethod(jsonDatapoint.Probmethod_embedding) ?? throw new Exception($"Unknown probmethod name {jsonDatapoint.Probmethod_embedding}"); - Datapoint datapoint = new(jsonDatapoint.Name, probMethod_embedding, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]); - int id_datapoint = DatabaseInsertDatapoint(jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, hash, id_entity); - List<(string model, byte[] embedding)> data = []; - foreach ((string, float[]) embedding in datapoint.embeddings) - { - data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2))); - } - DatabaseInsertEmbeddingBulk(id_datapoint, data); - datapoints.Add(datapoint); - } - - var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new Exception($"Unknown probmethod name {jsonEntity.Probmethod}"); - Entity entity = new(jsonEntity.Attributes, probMethod, datapoints, jsonEntity.Name) - { - id = id_entity - }; - entityCache.Add(entity); - return entity; - } - - public List? EntitiesFromJSON(string json) - { - List? jsonEntities = JsonSerializer.Deserialize>(json); - if (jsonEntities is null) - { - return null; - } - - Dictionary> toBeCached = []; - foreach (JSONEntity jSONEntity in jsonEntities) - { - foreach (JSONDatapoint datapoint in jSONEntity.Datapoints) - { - foreach (string model in datapoint.Model) - { - if (!toBeCached.ContainsKey(model)) - { - toBeCached[model] = []; - } - toBeCached[model].Add(datapoint.Text); - } - } - } - ConcurrentQueue retVal = []; - Parallel.ForEach(jsonEntities, jSONEntity => - { - var entity = EntityFromJSON(JsonSerializer.Serialize(jSONEntity)); - if (entity is not null) - { - retVal.Enqueue(entity); - } - }); - return retVal.ToList(); - } - - public void RemoveEntity(string name) - { - Dictionary parameters = new() - { - { "name", name } - }; - helper.ExecuteSQLNonQuery("DELETE embedding.* FROM embedding JOIN datapoint dp ON id_datapoint = dp.id JOIN entity ON id_entity = entity.id WHERE entity.name = @name", parameters); - helper.ExecuteSQLNonQuery("DELETE datapoint.* FROM datapoint JOIN entity ON id_entity = entity.id WHERE entity.name = @name", parameters); - helper.ExecuteSQLNonQuery("DELETE attribute.* FROM attribute JOIN entity ON id_entity = entity.id WHERE entity.name = @name", parameters); - helper.ExecuteSQLNonQuery("DELETE FROM entity WHERE name = @name", parameters); - entityCache.RemoveAll(entity => entity.name == name); - } - - public int DatabaseInsertSearchdomain(string name) - { - Dictionary parameters = new() - { - { "name", name }, - { "settings", "{}"} // TODO add settings. It's not used yet, but maybe it's needed someday... - }; - return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO searchdomain (name, settings) VALUES (@name, @settings)", parameters); - } - - public int DatabaseInsertEntity(string name, string probmethod, int id_searchdomain) - { - Dictionary parameters = new() - { - { "name", name }, - { "probmethod", probmethod }, - { "id_searchdomain", id_searchdomain } - }; - return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO entity (name, probmethod, id_searchdomain) VALUES (@name, @probmethod, @id_searchdomain)", parameters); - } - - public int DatabaseInsertAttribute(string attribute, string value, int id_entity) - { - Dictionary parameters = new() - { - { "attribute", attribute }, - { "value", value }, - { "id_entity", id_entity } - }; - return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO attribute (attribute, value, id_entity) VALUES (@attribute, @value, @id_entity)", parameters); - } - - - public int DatabaseInsertDatapoint(string name, string probmethod_embedding, string hash, int id_entity) - { - Dictionary parameters = new() - { - { "name", name }, - { "probmethod_embedding", probmethod_embedding }, - { "hash", hash }, - { "id_entity", id_entity } - }; - return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO datapoint (name, probmethod_embedding, hash, id_entity) VALUES (@name, @probmethod_embedding, @hash, @id_entity)", parameters); - } - - public int DatabaseInsertEmbedding(int id_datapoint, string model, byte[] embedding) - { - Dictionary parameters = new() - { - { "id_datapoint", id_datapoint }, - { "model", model }, - { "embedding", embedding } - }; - return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO embedding (id_datapoint, model, embedding) VALUES (@id_datapoint, @model, @embedding)", parameters); - } - - public void DatabaseInsertEmbeddingBulk(int id_datapoint, List<(string model, byte[] embedding)> data) - { - Dictionary parameters = []; - parameters["id_datapoint"] = id_datapoint; - var query = new StringBuilder("INSERT INTO embedding (id_datapoint, model, embedding) VALUES "); - foreach (var (model, embedding) in data) - { - string modelParam = $"model_{Guid.NewGuid()}".Replace("-", ""); - string embeddingParam = $"embedding_{Guid.NewGuid()}".Replace("-", ""); - parameters[modelParam] = model; - parameters[embeddingParam] = embedding; - - query.Append($"(@id_datapoint, @{modelParam}, @{embeddingParam}), "); - } - - query.Length -= 2; // remove trailing comma - helper.ExecuteSQLNonQuery(query.ToString(), parameters); - } } diff --git a/src/Server/SearchdomainManager.cs b/src/Server/SearchdomainManager.cs index a8c8050..a6c2878 100644 --- a/src/Server/SearchdomainManager.cs +++ b/src/Server/SearchdomainManager.cs @@ -14,14 +14,16 @@ public class SearchdomainManager private readonly IConfiguration _config; private readonly string ollamaURL; private readonly string connectionString; - private OllamaApiClient client; + public OllamaApiClient client; private MySqlConnection connection; - private SQLHelper helper; + public SQLHelper helper; + public Dictionary> embeddingCache; public SearchdomainManager(ILogger logger, IConfiguration config) { _logger = logger; _config = config; + embeddingCache = []; ollamaURL = _config.GetSection("Embeddingsearch")["OllamaURL"] ?? ""; connectionString = _config.GetSection("Embeddingsearch").GetConnectionString("SQL") ?? ""; if (ollamaURL.IsNullOrEmpty() || connectionString.IsNullOrEmpty()) @@ -31,7 +33,7 @@ public class SearchdomainManager client = new(new Uri(ollamaURL)); connection = new MySqlConnection(connectionString); connection.Open(); - helper = new SQLHelper(connection); + helper = new SQLHelper(connection, connectionString); try { DatabaseMigrations.Migrate(helper); @@ -51,7 +53,7 @@ public class SearchdomainManager } try { - return SetSearchdomain(searchdomain, new Searchdomain(searchdomain, connectionString, client)); + return SetSearchdomain(searchdomain, new Searchdomain(searchdomain, connectionString, client, embeddingCache)); } catch (MySqlException) { @@ -67,14 +69,17 @@ public class SearchdomainManager public List ListSearchdomains() { - DbDataReader reader = helper.ExecuteSQLCommand("SELECT name FROM searchdomain", []); - List results = []; - while (reader.Read()) + lock (helper.connection) { - results.Add(reader.GetString(0)); + DbDataReader reader = helper.ExecuteSQLCommand("SELECT name FROM searchdomain", []); + List results = []; + while (reader.Read()) + { + results.Add(reader.GetString(0)); + } + reader.Close(); + return results; } - reader.Close(); - return results; } public int CreateSearchdomain(string searchdomain, string settings = "{}") @@ -98,7 +103,7 @@ public class SearchdomainManager int counter = 0; while (searchdomain_.entityCache.Count > 0) { - searchdomain_.RemoveEntity(searchdomain_.entityCache.First().name); + DatabaseHelper.RemoveEntity(searchdomain_.entityCache, helper, searchdomain_.entityCache.First().name, searchdomain); counter += 1; } _logger.LogDebug($"Number of entities deleted as part of deleting the searchdomain \"{searchdomain}\": {counter}");