diff --git a/src/Server/AIProvider.cs b/src/Server/AIProvider.cs index 0a244a2..1e5c485 100644 --- a/src/Server/AIProvider.cs +++ b/src/Server/AIProvider.cs @@ -116,6 +116,79 @@ public class AIProvider throw; } } + + public string[] GetModels() + { + var aIProviders = aIProvidersConfiguration.AiProviders; + List results = []; + foreach (KeyValuePair aIProviderKV in aIProviders) + { + string aIProviderName = aIProviderKV.Key; + AIProviderConfiguration aIProvider = aIProviderKV.Value; + + using var httpClient = new HttpClient(); + + string modelNameJsonPath = ""; + Uri baseUri = new(aIProvider.BaseURL); + Uri requestUri; + string[][] requestHeaders = []; + switch (aIProvider.Handler) + { + case "ollama": + modelNameJsonPath = "$.models[*].name"; + requestUri = new Uri(baseUri, "/api/tags"); + break; + case "openai": + modelNameJsonPath = "$.data[*].id"; + requestUri = new Uri(baseUri, "/v1/models"); + if (aIProvider.ApiKey is not null) + { + requestHeaders = [ + ["Authorization", $"Bearer {aIProvider.ApiKey}"] + ]; + } + break; + default: + _logger.LogError("Unknown handler {aIProvider.Handler} in AiProvider {provider}.", [aIProvider.Handler, aIProvider]); + throw new ServerConfigurationException($"Unknown handler {aIProvider.Handler} in AiProvider {aIProvider}."); + } + + var request = new HttpRequestMessage() + { + RequestUri = requestUri, + Method = HttpMethod.Post + }; + + foreach (var header in requestHeaders) + { + request.Headers.Add(header[0], header[1]); + } + HttpResponseMessage response = httpClient.GetAsync(requestUri).Result; + string responseContent = response.Content.ReadAsStringAsync().Result; + try + { + JObject responseContentJson = JObject.Parse(responseContent); + IEnumerable? responseContentTokens = responseContentJson.SelectTokens(modelNameJsonPath); + if (responseContentTokens is null) + { + _logger.LogError("Unable to select tokens using JSONPath {modelNameJsonPath} for string: {responseContent}.", [modelNameJsonPath, responseContent]); + throw new JSONPathSelectionException(modelNameJsonPath, responseContent); + } + IEnumerable aIProviderResult = responseContentTokens.Values(); + foreach (string? result in aIProviderResult) + { + if (result is null) continue; + results.Add(aIProviderName + ":" + result); + } + } + catch (Exception ex) + { + _logger.LogError("Unable to parse the response to valid models. {ex.Message}", [ex.Message]); + throw; + } + } + return [.. results]; + } } public class AIProvidersConfiguration diff --git a/src/Server/Controllers/EntityController.cs b/src/Server/Controllers/EntityController.cs index 4765311..3326310 100644 --- a/src/Server/Controllers/EntityController.cs +++ b/src/Server/Controllers/EntityController.cs @@ -25,7 +25,7 @@ public class EntityController : ControllerBase } [HttpGet("Query")] - public ActionResult Query(string searchdomain, string query) + public ActionResult Query(string searchdomain, string query, int? topN) { Searchdomain searchdomain_; try @@ -40,7 +40,7 @@ public class EntityController : ControllerBase _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]); return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." }); } - var results = searchdomain_.Search(query); + List<(float, string)> results = searchdomain_.Search(query, topN); List queryResults = [.. results.Select(r => new EntityQueryResult { Name = r.Item2, @@ -55,10 +55,7 @@ public class EntityController : ControllerBase try { List? entities = _searchdomainHelper.EntitiesFromJSON( - [], - _domainManager.embeddingCache, - _domainManager.aIProvider, - _domainManager.helper, + _domainManager, _logger, JsonSerializer.Serialize(jsonEntities)); if (entities is not null && jsonEntities is not null) @@ -85,15 +82,20 @@ public class EntityController : ControllerBase } catch (Exception ex) { if (ex.InnerException is not null) ex = ex.InnerException; - _logger.LogError("Unable to index the provided entities. {ex.Message}", [ex.Message]); + _logger.LogError("Unable to index the provided entities. {ex.Message} - {ex.StackTrace}", [ex.Message, ex.StackTrace]); return Ok(new EntityIndexResult() { Success = false, Message = ex.Message }); } } [HttpGet("List")] - public ActionResult List(string searchdomain, bool returnEmbeddings = false) + public ActionResult List(string searchdomain, bool returnModels = false, bool returnEmbeddings = false) { + if (returnEmbeddings && !returnModels) + { + _logger.LogError("Invalid request for {searchdomain} - embeddings return requested but without models - not possible!", [searchdomain]); + return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Invalid request" }); + } Searchdomain searchdomain_; try { @@ -118,12 +120,12 @@ public class EntityController : ControllerBase List datapointResults = []; foreach (Datapoint datapoint in entity.datapoints) { - if (returnEmbeddings) + if (returnModels) { List embeddingResults = []; foreach ((string, float[]) embedding in datapoint.embeddings) { - embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = embedding.Item2}); + embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = returnEmbeddings ? embedding.Item2 : []}); } datapointResults.Add(new DatapointResult() {Name = datapoint.name, ProbMethod = datapoint.probMethod.name, SimilarityMethod = datapoint.similarityMethod.name, Embeddings = embeddingResults}); } @@ -135,6 +137,7 @@ public class EntityController : ControllerBase EntityListResult entityListResult = new() { Name = entity.name, + ProbMethod = entity.probMethodName, Attributes = attributeResults, Datapoints = datapointResults }; @@ -146,13 +149,28 @@ public class EntityController : ControllerBase [HttpGet("Delete")] public ActionResult Delete(string searchdomain, string entityName) { - Entity? entity_ = SearchdomainHelper.CacheGetEntity([], entityName); + Searchdomain searchdomain_; + try + { + searchdomain_ = _domainManager.GetSearchdomain(searchdomain); + } catch (SearchdomainNotFoundException) + { + _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]); + return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" }); + } catch (Exception ex) + { + _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]); + return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." }); + } + + Entity? entity_ = SearchdomainHelper.CacheGetEntity(searchdomain_.entityCache, entityName); if (entity_ is null) { _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]); return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"}); } _databaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain); + searchdomain_.entityCache.RemoveAll(entity => entity.name == entityName); return Ok(new EntityDeleteResults() {Success = true}); } } diff --git a/src/Server/Controllers/SearchdomainController.cs b/src/Server/Controllers/SearchdomainController.cs index 6d75dbf..286d8b5 100644 --- a/src/Server/Controllers/SearchdomainController.cs +++ b/src/Server/Controllers/SearchdomainController.cs @@ -1,5 +1,6 @@ using System.Text.Json; using ElmahCore; +using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.AspNetCore.Mvc; using Server.Exceptions; using Server.Helper; @@ -156,6 +157,64 @@ public class SearchdomainController : ControllerBase return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true }); } + [HttpDelete("Searches")] + public ActionResult DeleteSearch(string searchdomain, string query) + { + Searchdomain searchdomain_; + try + { + searchdomain_ = _domainManager.GetSearchdomain(searchdomain); + } + catch (SearchdomainNotFoundException) + { + _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]); + return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = "Searchdomain not found" }); + } + catch (Exception ex) + { + _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]); + return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = ex.Message }); + } + Dictionary searchCache = searchdomain_.searchCache; + bool containsKey = searchCache.ContainsKey(query); + if (containsKey) + { + searchCache.Remove(query); + return Ok(new SearchdomainDeleteSearchResult() {Success = true}); + } + return Ok(new SearchdomainDeleteSearchResult() {Success = false, Message = "Query not found in search cache"}); + } + + [HttpPatch("Searches")] + public ActionResult UpdateSearch(string searchdomain, string query, [FromBody]List results) + { + Searchdomain searchdomain_; + try + { + searchdomain_ = _domainManager.GetSearchdomain(searchdomain); + } + catch (SearchdomainNotFoundException) + { + _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]); + return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = "Searchdomain not found" }); + } + catch (Exception ex) + { + _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]); + return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = ex.Message }); + } + Dictionary searchCache = searchdomain_.searchCache; + bool containsKey = searchCache.ContainsKey(query); + if (containsKey) + { + DateTimedSearchResult element = searchCache[query]; + element.Results = results; + searchCache[query] = element; + return Ok(new SearchdomainUpdateSearchResult() {Success = true}); + } + return Ok(new SearchdomainUpdateSearchResult() {Success = false, Message = "Query not found in search cache"}); + } + [HttpGet("GetSettings")] public ActionResult GetSettings(string searchdomain) { diff --git a/src/Server/Controllers/ServerController.cs b/src/Server/Controllers/ServerController.cs new file mode 100644 index 0000000..ca4d9b8 --- /dev/null +++ b/src/Server/Controllers/ServerController.cs @@ -0,0 +1,38 @@ +namespace Server.Controllers; + +using System.Text.Json; +using ElmahCore; +using Microsoft.AspNetCore.Mvc; +using Server.Exceptions; +using Server.Helper; +using Shared.Models; + +[ApiController] +[Route("[controller]")] +public class ServerController : ControllerBase +{ + private readonly ILogger _logger; + private readonly IConfiguration _config; + private AIProvider _aIProvider; + + public ServerController(ILogger logger, IConfiguration config, AIProvider aIProvider) + { + _logger = logger; + _config = config; + _aIProvider = aIProvider; + } + + [HttpGet("GetModels")] + public ActionResult GetModels() + { + try + { + string[] models = _aIProvider.GetModels(); + return new ServerGetModelsResult() { Models = models, Success = true }; + } catch (Exception ex) + { + _logger.LogError("Unable to get models due to exception {ex.Message} - {ex.StackTrace}", [ex.Message, ex.StackTrace]); + return new ServerGetModelsResult() { Success = false, Message = ex.Message}; + } + } +} diff --git a/src/Server/Entity.cs b/src/Server/Entity.cs index edf4104..681b96b 100644 --- a/src/Server/Entity.cs +++ b/src/Server/Entity.cs @@ -1,9 +1,10 @@ namespace Server; -public class Entity(Dictionary attributes, Probmethods.probMethodDelegate probMethod, List datapoints, string name) +public class Entity(Dictionary attributes, Probmethods.probMethodDelegate probMethod, string probMethodName, List datapoints, string name) { public Dictionary attributes = attributes; public Probmethods.probMethodDelegate probMethod = probMethod; + public string probMethodName = probMethodName; public List datapoints = datapoints; public int id; public string name = name; diff --git a/src/Server/Helper/SearchdomainHelper.cs b/src/Server/Helper/SearchdomainHelper.cs index 7b137bd..2dfee5a 100644 --- a/src/Server/Helper/SearchdomainHelper.cs +++ b/src/Server/Helper/SearchdomainHelper.cs @@ -44,8 +44,12 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp return null; } - public List? EntitiesFromJSON(List entityCache, Dictionary> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json) + public List? EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json) { + Dictionary> embeddingCache = searchdomainManager.embeddingCache; + AIProvider aIProvider = searchdomainManager.aIProvider; + SQLHelper helper = searchdomainManager.helper; + List? jsonEntities = JsonSerializer.Deserialize>(json); if (jsonEntities is null) { @@ -72,8 +76,7 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = 16 }; // <-- This is needed! Otherwise if we try to index 100+ entities at once, it spawns 100 threads, exploding the SQL pool Parallel.ForEach(jsonEntities, parallelOptions, jSONEntity => { - using var tempHelper = helper.DuplicateConnection(); - var entity = EntityFromJSON(entityCache, embeddingCache, aIProvider, tempHelper, logger, jSONEntity); + var entity = EntityFromJSON(searchdomainManager, logger, jSONEntity); if (entity is not null) { retVal.Enqueue(entity); @@ -82,87 +85,188 @@ public class SearchdomainHelper(ILogger logger, DatabaseHelp return [.. retVal]; } - public Entity? EntityFromJSON(List entityCache, Dictionary> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json) + public Entity? EntityFromJSON(SearchdomainManager searchdomainManager, ILogger logger, JSONEntity jsonEntity) //string json) { - Dictionary> embeddingsLUT = []; // embeddingsLUT: hash -> model -> [embeddingValues * n] - int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain); - if (preexistingEntityID is not null) + SQLHelper helper = searchdomainManager.helper.DuplicateConnection(); + Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain); + List entityCache = searchdomain.entityCache; + AIProvider aIProvider = searchdomain.aIProvider; + Dictionary> embeddingCache = searchdomain.embeddingCache; + Entity? preexistingEntity = entityCache.FirstOrDefault(entity => entity.name == jsonEntity.Name); + + if (preexistingEntity is not null) { - lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after) + int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain); + if (preexistingEntityID is null) { - Dictionary parameters = new() - { - { "id", preexistingEntityID } - }; - System.Data.Common.DbDataReader reader = helper.ExecuteSQLCommand("SELECT e.model, e.embedding, d.hash FROM datapoint d JOIN embedding e ON d.id = e.id_datapoint WHERE d.id_entity = @id", parameters); - while (reader.Read()) - { - string model = reader.GetString(0); - long length = reader.GetBytes(1, 0, null, 0, 0); - byte[] embeddingBytes = new byte[length]; - reader.GetBytes(1, 0, embeddingBytes, 0, (int)length); - float[] embeddingValues = FloatArrayFromBytes(embeddingBytes); - string hash = reader.GetString(2); - if (!embeddingsLUT.ContainsKey(hash)) - { - embeddingsLUT[hash] = []; - } - embeddingsLUT[hash].TryAdd(model, embeddingValues); - } - reader.Close(); + _logger.LogCritical("Unable to index entity {jsonEntity.Name} because it already exists in the searchdomain but not in the database.", [jsonEntity.Name]); + throw new Exception($"Unable to index entity {jsonEntity.Name} because it already exists in the searchdomain but not in the database."); } - _databaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create - } - int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain)); - foreach (KeyValuePair attribute in jsonEntity.Attributes) - { - DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries - } - - List datapoints = []; - foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) - { - string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); - Dictionary embeddings = []; - if (embeddingsLUT.ContainsKey(hash)) + Dictionary attributes = jsonEntity.Attributes; + + // Attribute + foreach (KeyValuePair attributesKV in preexistingEntity.attributes.ToList()) { - Dictionary hashLUT = embeddingsLUT[hash]; - foreach (string model in jsonDatapoint.Model) + string oldAttributeKey = attributesKV.Key; + string oldAttribute = attributesKV.Value; + bool newHasAttribute = jsonEntity.Attributes.TryGetValue(oldAttributeKey, out string? newAttribute); + if (newHasAttribute && newAttribute is not null && newAttribute != oldAttribute) { - if (hashLUT.ContainsKey(model)) + // Attribute - Updated + Dictionary parameters = new() { - embeddings.Add(model, hashLUT[model]); + { "newValue", newAttribute }, + { "entityId", preexistingEntityID }, + { "attribute", oldAttributeKey} + }; + helper.ExecuteSQLNonQuery("UPDATE attribute SET value=@newValue WHERE id_entity=@entityId AND attribute=@attribute", parameters); + preexistingEntity.attributes[oldAttributeKey] = newAttribute; + } else if (!newHasAttribute) + { + // Attribute - Deleted + Dictionary parameters = new() + { + { "entityId", preexistingEntityID }, + { "attribute", oldAttributeKey} + }; + helper.ExecuteSQLNonQuery("DELETE FROM attribute WHERE id_entity=@entityId AND attribute=@attribute", parameters); + preexistingEntity.attributes.Remove(oldAttributeKey); + } + } + foreach (var attributesKV in jsonEntity.Attributes) + { + string newAttributeKey = attributesKV.Key; + string newAttribute = attributesKV.Value; + bool preexistingHasAttribute = preexistingEntity.attributes.TryGetValue(newAttributeKey, out string? preexistingAttribute); + if (!preexistingHasAttribute) + { + // Attribute - New + DatabaseHelper.DatabaseInsertAttribute(helper, newAttributeKey, newAttribute, (int)preexistingEntityID); + preexistingEntity.attributes.Add(newAttributeKey, newAttribute); + } + } + + // Datapoint + foreach (Datapoint datapoint in preexistingEntity.datapoints.ToList()) + { + bool newEntityHasDatapoint = jsonEntity.Datapoints.Any(x => x.Name == datapoint.name); + if (!newEntityHasDatapoint) + { + // Datapoint - Deleted + Dictionary parameters = new() + { + { "datapointName", datapoint.name }, + { "entityId", preexistingEntityID} + }; + helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters); + helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters); + preexistingEntity.datapoints.Remove(datapoint); + } else + { + JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.name); + if (newEntityDatapoint is not null && newEntityDatapoint.Text is not null) + { + // Datapoint - Updated (text) + Dictionary parameters = new() + { + { "datapointName", datapoint.name }, + { "entityId", preexistingEntityID} + }; + helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters); + helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters); + preexistingEntity.datapoints.Remove(datapoint); + Datapoint newDatapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, newEntityDatapoint, (int)preexistingEntityID); + preexistingEntity.datapoints.Add(newDatapoint); + } - else + if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.name || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.name)) { - var additionalEmbeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [model], aIProvider, embeddingCache); - embeddings.Add(model, additionalEmbeddings.First().Value); + // Datapoint - Updated (probmethod or similaritymethod) + Dictionary parameters = new() + { + { "probmethod", newEntityDatapoint.Probmethod_embedding }, + { "similaritymethod", newEntityDatapoint.SimilarityMethod }, + { "datapointName", datapoint.name }, + { "entityId", preexistingEntityID} + }; + helper.ExecuteSQLNonQuery("UPDATE datapoint SET probmethod_embedding=@probmethod, similaritymethod=@similaritymethod WHERE id_entity=@entityId AND name=@datapointName", parameters); + Datapoint preexistingDatapoint = preexistingEntity.datapoints.First(x => x == datapoint); // The for loop is a copy. This retrieves the original such that it can be updated. + preexistingDatapoint.probMethod = datapoint.probMethod; + preexistingDatapoint.similarityMethod = datapoint.similarityMethod; } } } - else + foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) { - embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], aIProvider, embeddingCache); + bool oldEntityHasDatapoint = preexistingEntity.datapoints.Any(x => x.name == jsonDatapoint.Name); + if (!oldEntityHasDatapoint) + { + // Datapoint - New + Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, (int)preexistingEntityID); + preexistingEntity.datapoints.Add(datapoint); + } } - var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding, logger) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding); - var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod); - Datapoint datapoint = new(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]); - int id_datapoint = DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, id_entity); // TODO make this a bulk add action to reduce number of queries - List<(string model, byte[] embedding)> data = []; - foreach ((string, float[]) embedding in datapoint.embeddings) - { - data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2))); - } - DatabaseHelper.DatabaseInsertEmbeddingBulk(helper, id_datapoint, data); - datapoints.Add(datapoint); - } - var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod); - Entity entity = new(jsonEntity.Attributes, probMethod, datapoints, jsonEntity.Name) + + return preexistingEntity; + } + else { - id = id_entity - }; - entityCache.Add(entity); - return entity; + int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain)); + foreach (KeyValuePair attribute in jsonEntity.Attributes) + { + DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries + } + + List datapoints = []; + foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints) + { + string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); + Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, id_entity, hash); + datapoints.Add(datapoint); + } + + var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod); + Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod, datapoints, jsonEntity.Name) + { + id = id_entity + }; + entityCache.Add(entity); + return entity; + } + } + + public Datapoint DatabaseInsertDatapointWithEmbeddings(SQLHelper helper, Searchdomain searchdomain, JSONDatapoint jsonDatapoint, int id_entity, string? hash = null) + { + if (jsonDatapoint.Text is null) + { + throw new Exception("jsonDatapoint.Text must not be null at this point"); + } + hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); + Datapoint datapoint = BuildDatapointFromJsonDatapoint(jsonDatapoint, id_entity, searchdomain, hash); + int id_datapoint = DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, id_entity); // TODO make this a bulk add action to reduce number of queries + List<(string model, byte[] embedding)> data = []; + foreach ((string, float[]) embedding in datapoint.embeddings) + { + data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2))); + } + DatabaseHelper.DatabaseInsertEmbeddingBulk(helper, id_datapoint, data); + return datapoint; + } + + public Datapoint BuildDatapointFromJsonDatapoint(JSONDatapoint jsonDatapoint, int entityId, Searchdomain searchdomain, string? hash = null) + { + if (jsonDatapoint.Text is null) + { + throw new Exception("jsonDatapoint.Text must not be null at this point"); + } + using SQLHelper helper = searchdomain.helper.DuplicateConnection(); + Dictionary> embeddingCache = searchdomain.embeddingCache; + hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text))); + DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId); + Dictionary embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.aIProvider, embeddingCache); + var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding, logger) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding); + var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod); + return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]); } } \ No newline at end of file diff --git a/src/Server/Resources/SharedResources.de.resx b/src/Server/Resources/SharedResources.de.resx index e18c07d..95fd591 100644 --- a/src/Server/Resources/SharedResources.de.resx +++ b/src/Server/Resources/SharedResources.de.resx @@ -18,4 +18,10 @@ Ungültiger Benutzername oder Passwort. + + Sind Sie sicher, dass Sie <strong id="{0}">diese Entity</strong> löschen wollen? + + + Diese Aktion kann nicht rückgängig gemacht werden. + \ No newline at end of file diff --git a/src/Server/Resources/SharedResources.en.resx b/src/Server/Resources/SharedResources.en.resx index bd6e064..c9635ee 100644 --- a/src/Server/Resources/SharedResources.en.resx +++ b/src/Server/Resources/SharedResources.en.resx @@ -18,4 +18,10 @@ Invalid credentials. + + Are you sure you want to delete <strong id="{0}">this entity</strong>? + + + This action cannot be undone. + \ No newline at end of file diff --git a/src/Server/Searchdomain.cs b/src/Server/Searchdomain.cs index 019eb79..1b0e0c3 100644 --- a/src/Server/Searchdomain.cs +++ b/src/Server/Searchdomain.cs @@ -142,7 +142,7 @@ public class Searchdomain Probmethods.probMethodDelegate? probmethod = Probmethods.GetMethod(probmethodString); if (datapoint_unassigned.TryGetValue(id, out List? datapoints) && probmethod is not null) { - Entity entity = new(attributes, probmethod, datapoints, name) + Entity entity = new(attributes, probmethod, probmethodString, datapoints, name) { id = id }; @@ -154,7 +154,7 @@ public class Searchdomain embeddingCache = []; // TODO remove this and implement proper remediation to improve performance } - public List<(float, string)> Search(string query) + public List<(float, string)> Search(string query, int? topN = null) { if (searchCache.TryGetValue(query, out DateTimedSearchResult cachedResult)) { @@ -190,9 +190,14 @@ public class Searchdomain } result.Add((entity.probMethod(datapointProbs), entity.name)); } - List<(float, string)> results = [.. result.OrderByDescending(s => s.Item1)]; + IEnumerable<(float, string)> sortedResults = result.OrderByDescending(s => s.Item1); + if (topN is not null) + { + sortedResults = sortedResults.Take(topN ?? 0); + } + List<(float, string)> results = [.. sortedResults]; List searchResult = new( - [.. results.Select(r => + [.. sortedResults.Select(r => new ResultItem(r.Item1, r.Item2 ))] ); searchCache[query] = new DateTimedSearchResult(DateTime.Now, searchResult); diff --git a/src/Server/SimilarityMethods.cs b/src/Server/SimilarityMethods.cs index 9521825..9d02c22 100644 --- a/src/Server/SimilarityMethods.cs +++ b/src/Server/SimilarityMethods.cs @@ -21,20 +21,28 @@ public class SimilarityMethod } } +public enum SimilarityMethodEnum +{ + Cosine, + Euclidian, + Manhattan, + Pearson +} + public static class SimilarityMethods { public delegate float similarityMethodProtoDelegate(float[] vector1, float[] vector2); public delegate float similarityMethodDelegate(float[] vector1, float[] vector2); - public static readonly Dictionary probMethods; + public static readonly Dictionary probMethods; static SimilarityMethods() { - probMethods = new Dictionary + probMethods = new Dictionary { - ["Cosine"] = CosineSimilarity, - ["Euclidian"] = EuclidianDistance, - ["Manhattan"] = ManhattanDistance, - ["Pearson"] = PearsonCorrelation + [SimilarityMethodEnum.Cosine] = CosineSimilarity, + [SimilarityMethodEnum.Euclidian] = EuclidianDistance, + [SimilarityMethodEnum.Manhattan] = ManhattanDistance, + [SimilarityMethodEnum.Pearson] = PearsonCorrelation }; } @@ -42,7 +50,12 @@ public static class SimilarityMethods { string methodName = name; - if (!probMethods.TryGetValue(methodName, out similarityMethodProtoDelegate? method)) + SimilarityMethodEnum probMethodEnum = (SimilarityMethodEnum)Enum.Parse( + typeof(SimilarityMethodEnum), + methodName + ); + + if (!probMethods.TryGetValue(probMethodEnum, out similarityMethodProtoDelegate? method)) { return null; } diff --git a/src/Server/Views/Home/Index.cshtml b/src/Server/Views/Home/Index.cshtml index 022967b..6afa4eb 100644 --- a/src/Server/Views/Home/Index.cshtml +++ b/src/Server/Views/Home/Index.cshtml @@ -1,11 +1,17 @@ @using Server.Models @using System.Web @using Server.Services +@using Server + @inject LocalizationService T +@inject AIProvider AIProvider @model HomeIndexViewModel @{ ViewData["Title"] = "Home Page"; int i = 0; + string[] probMethods = [.. Enum.GetValues(typeof(ProbMethodEnum)).Cast().Select(e => e.ToString())]; + string[] similarityMethods = [.. Enum.GetValues(typeof(SimilarityMethodEnum)).Cast().Select(e => e.ToString())]; + string[] models = AIProvider.GetModels(); Dictionary domains = []; Model.Searchdomains.ForEach(domain => { @@ -57,7 +63,7 @@

@T["Settings"]

- +
@@ -130,6 +136,7 @@ + @T["Add new entity"]
@@ -200,26 +207,70 @@
-

Access times

-
    +

    @T["Access times"]

    +
      -

      Results

      +

      @T["Results"]

      - - + + - +
      ScoreName@T["Score"]@T["Name"]
      +
      + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Shared/Models/EntityResults.cs b/src/Shared/Models/EntityResults.cs index 025a2e7..03db7a5 100644 --- a/src/Shared/Models/EntityResults.cs +++ b/src/Shared/Models/EntityResults.cs @@ -41,6 +41,8 @@ public class EntityListResult { [JsonPropertyName("Name")] public required string Name { get; set; } + [JsonPropertyName("ProbMethod")] + public required string ProbMethod { get; set; } [JsonPropertyName("Attributes")] public required List Attributes { get; set; } [JsonPropertyName("Datapoints")] diff --git a/src/Shared/Models/JSONModels.cs b/src/Shared/Models/JSONModels.cs index 86ca7b3..593ae96 100644 --- a/src/Shared/Models/JSONModels.cs +++ b/src/Shared/Models/JSONModels.cs @@ -12,7 +12,7 @@ public class JSONEntity public class JSONDatapoint { public required string Name { get; set; } - public required string Text { get; set; } + public required string? Text { get; set; } public required string Probmethod_embedding { get; set; } public required string SimilarityMethod { get; set; } public required string[] Model { get; set; } diff --git a/src/Shared/Models/SearchdomainModels.cs b/src/Shared/Models/SearchdomainModels.cs index 646d701..f7cf407 100644 --- a/src/Shared/Models/SearchdomainModels.cs +++ b/src/Shared/Models/SearchdomainModels.cs @@ -2,12 +2,19 @@ using System.Text.Json.Serialization; namespace Shared.Models; -public readonly struct ResultItem(float score, string name) +public readonly struct ResultItem { [JsonPropertyName("Score")] - public readonly float Score { get; } = score; + public readonly float Score { get; } [JsonPropertyName("Name")] - public readonly string Name { get; } = name; + public readonly string Name { get; } + + [JsonConstructor] + public ResultItem(float score, string name) + { + Score = score; + Name = name; + } public static long EstimateSize(ResultItem item) { diff --git a/src/Shared/Models/SearchdomainResults.cs b/src/Shared/Models/SearchdomainResults.cs index 92f6990..f6f1de7 100644 --- a/src/Shared/Models/SearchdomainResults.cs +++ b/src/Shared/Models/SearchdomainResults.cs @@ -55,6 +55,24 @@ public class SearchdomainSearchesResults public required Dictionary Searches { get; set; } } +public class SearchdomainDeleteSearchResult +{ + [JsonPropertyName("Success")] + public required bool Success { get; set; } + + [JsonPropertyName("Message")] + public string? Message { get; set; } +} + +public class SearchdomainUpdateSearchResult +{ + [JsonPropertyName("Success")] + public required bool Success { get; set; } + + [JsonPropertyName("Message")] + public string? Message { get; set; } +} + public class SearchdomainSettingsResults { [JsonPropertyName("Success")] diff --git a/src/Shared/Models/ServerModels.cs b/src/Shared/Models/ServerModels.cs new file mode 100644 index 0000000..278582e --- /dev/null +++ b/src/Shared/Models/ServerModels.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace Shared.Models; + +public class ServerGetModelsResult +{ + [JsonPropertyName("Success")] + public required bool Success { get; set; } + + [JsonPropertyName("Message")] + public string? Message { get; set; } + + [JsonPropertyName("Models")] + public string[]? Models { get; set; } +} \ No newline at end of file