Merge pull request #45 from LD-Reborn/36-create-a-front-end---functioning-buttons-2

36 create a front end   functioning buttons 2

src/Server/AIProvider.cs

@@ -116,6 +116,79 @@ public class AIProvider
             throw;
         }
     }
+
+    public string[] GetModels()
+    {
+        var aIProviders = aIProvidersConfiguration.AiProviders;
+        List<string> results = [];
+        foreach (KeyValuePair<string, AIProviderConfiguration> aIProviderKV in aIProviders)
+        {
+            string aIProviderName = aIProviderKV.Key;
+            AIProviderConfiguration aIProvider = aIProviderKV.Value;
+
+            using var httpClient = new HttpClient();
+
+            string modelNameJsonPath = "";
+            Uri baseUri = new(aIProvider.BaseURL);
+            Uri requestUri;
+            string[][] requestHeaders = [];
+            switch (aIProvider.Handler)
+            {
+                case "ollama":
+                    modelNameJsonPath = "$.models[*].name";
+                    requestUri = new Uri(baseUri, "/api/tags");
+                    break;
+                case "openai":
+                    modelNameJsonPath = "$.data[*].id";
+                    requestUri = new Uri(baseUri, "/v1/models");
+                    if (aIProvider.ApiKey is not null)
+                    {
+                        requestHeaders = [
+                            ["Authorization", $"Bearer {aIProvider.ApiKey}"]
+                        ];
+                    }
+                    break;
+                default:
+                    _logger.LogError("Unknown handler {aIProvider.Handler} in AiProvider {provider}.", [aIProvider.Handler, aIProvider]);
+                    throw new ServerConfigurationException($"Unknown handler {aIProvider.Handler} in AiProvider {aIProvider}.");
+            }
+
+            var request = new HttpRequestMessage()
+            {
+                RequestUri = requestUri,
+                Method = HttpMethod.Post
+            };
+            
+            foreach (var header in requestHeaders)
+            {
+                request.Headers.Add(header[0], header[1]);
+            }
+            HttpResponseMessage response = httpClient.GetAsync(requestUri).Result;
+            string responseContent = response.Content.ReadAsStringAsync().Result;
+            try
+            {
+                JObject responseContentJson = JObject.Parse(responseContent);
+                IEnumerable<JToken>? responseContentTokens = responseContentJson.SelectTokens(modelNameJsonPath);
+                if (responseContentTokens is null)
+                {
+                    _logger.LogError("Unable to select tokens using JSONPath {modelNameJsonPath} for string: {responseContent}.", [modelNameJsonPath, responseContent]);
+                    throw new JSONPathSelectionException(modelNameJsonPath, responseContent);
+                }
+                IEnumerable<string?> aIProviderResult = responseContentTokens.Values<string>();
+                foreach (string? result in aIProviderResult)
+                {
+                    if (result is null) continue;
+                    results.Add(aIProviderName + ":" + result);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError("Unable to parse the response to valid models. {ex.Message}", [ex.Message]);
+                throw;
+            }
+        }
+        return [.. results];
+    }
 }
 
 public class AIProvidersConfiguration

src/Server/Controllers/EntityController.cs

@@ -25,7 +25,7 @@ public class EntityController : ControllerBase
     }
 
     [HttpGet("Query")]
-    public ActionResult<EntityQueryResults> Query(string searchdomain, string query)
+    public ActionResult<EntityQueryResults> Query(string searchdomain, string query, int? topN)
     {
         Searchdomain searchdomain_;
         try
@@ -40,7 +40,7 @@ public class EntityController : ControllerBase
             _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
             return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
         }
-        var results = searchdomain_.Search(query);
+        List<(float, string)> results = searchdomain_.Search(query, topN);
         List<EntityQueryResult> queryResults = [.. results.Select(r => new EntityQueryResult
         {
             Name = r.Item2,
@@ -55,10 +55,7 @@ public class EntityController : ControllerBase
         try
         {
             List<Entity>? entities = _searchdomainHelper.EntitiesFromJSON(
-                [],
+                _domainManager,
-                _domainManager.embeddingCache,
-                _domainManager.aIProvider,
-                _domainManager.helper,
                 _logger,
                 JsonSerializer.Serialize(jsonEntities));
             if (entities is not null && jsonEntities is not null)
@@ -85,15 +82,20 @@ public class EntityController : ControllerBase
         } catch (Exception ex)
         {
             if (ex.InnerException is not null) ex = ex.InnerException;
-            _logger.LogError("Unable to index the provided entities. {ex.Message}", [ex.Message]);
+            _logger.LogError("Unable to index the provided entities. {ex.Message} - {ex.StackTrace}", [ex.Message, ex.StackTrace]);
             return Ok(new EntityIndexResult() { Success = false, Message = ex.Message });
         }
 
     }
 
     [HttpGet("List")]
-    public ActionResult<EntityListResults> List(string searchdomain, bool returnEmbeddings = false)
+    public ActionResult<EntityListResults> List(string searchdomain, bool returnModels = false, bool returnEmbeddings = false)
     {
+        if (returnEmbeddings && !returnModels)
+        {
+            _logger.LogError("Invalid request for {searchdomain} - embeddings return requested but without models - not possible!", [searchdomain]);
+            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Invalid request" });            
+        }
         Searchdomain searchdomain_;
         try
         {
@@ -118,12 +120,12 @@ public class EntityController : ControllerBase
             List<DatapointResult> datapointResults = [];
             foreach (Datapoint datapoint in entity.datapoints)
             {
-                if (returnEmbeddings)
+                if (returnModels)
                 {
                     List<EmbeddingResult> embeddingResults = [];
                     foreach ((string, float[]) embedding in datapoint.embeddings)
                     {
-                        embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = embedding.Item2});
+                        embeddingResults.Add(new EmbeddingResult() {Model = embedding.Item1, Embeddings = returnEmbeddings ? embedding.Item2 : []});
                     }
                     datapointResults.Add(new DatapointResult() {Name = datapoint.name, ProbMethod = datapoint.probMethod.name, SimilarityMethod = datapoint.similarityMethod.name, Embeddings = embeddingResults});
                 }
@@ -135,6 +137,7 @@ public class EntityController : ControllerBase
             EntityListResult entityListResult = new()
             {
                 Name = entity.name,
+                ProbMethod = entity.probMethodName,
                 Attributes = attributeResults,
                 Datapoints = datapointResults
             };
@@ -146,13 +149,28 @@ public class EntityController : ControllerBase
     [HttpGet("Delete")]
     public ActionResult<EntityDeleteResults> Delete(string searchdomain, string entityName)
     {
-        Entity? entity_ = SearchdomainHelper.CacheGetEntity([], entityName);
+        Searchdomain searchdomain_;
+        try
+        {
+            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
+        } catch (SearchdomainNotFoundException)
+        {
+            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
+            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
+        } catch (Exception ex)
+        {
+            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
+            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
+        }
+        
+        Entity? entity_ = SearchdomainHelper.CacheGetEntity(searchdomain_.entityCache, entityName);
         if (entity_ is null)
         {
             _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]);
             return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"});
         }
         _databaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain);
+        searchdomain_.entityCache.RemoveAll(entity => entity.name == entityName);
         return Ok(new EntityDeleteResults() {Success = true});
     }
 }

src/Server/Controllers/SearchdomainController.cs

@@ -1,5 +1,6 @@
 using System.Text.Json;
 using ElmahCore;
+using Microsoft.AspNetCore.Http.HttpResults;
 using Microsoft.AspNetCore.Mvc;
 using Server.Exceptions;
 using Server.Helper;
@@ -156,6 +157,64 @@ public class SearchdomainController : ControllerBase
         return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true });
     }
 
+    [HttpDelete("Searches")]
+    public ActionResult<SearchdomainDeleteSearchResult> DeleteSearch(string searchdomain, string query)
+    {
+        Searchdomain searchdomain_;
+        try
+        {
+            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
+        }
+        catch (SearchdomainNotFoundException)
+        {
+            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
+            return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = "Searchdomain not found" });
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
+            return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = ex.Message });
+        }
+        Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
+        bool containsKey = searchCache.ContainsKey(query);
+        if (containsKey)
+        {
+            searchCache.Remove(query);
+            return Ok(new SearchdomainDeleteSearchResult() {Success = true});
+        }
+        return Ok(new SearchdomainDeleteSearchResult() {Success = false, Message = "Query not found in search cache"});
+    }
+
+    [HttpPatch("Searches")]
+    public ActionResult<SearchdomainUpdateSearchResult> UpdateSearch(string searchdomain, string query, [FromBody]List<ResultItem> results)
+    {
+        Searchdomain searchdomain_;
+        try
+        {
+            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
+        }
+        catch (SearchdomainNotFoundException)
+        {
+            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
+            return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = "Searchdomain not found" });
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
+            return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = ex.Message });
+        }
+        Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
+        bool containsKey = searchCache.ContainsKey(query);
+        if (containsKey)
+        {
+            DateTimedSearchResult element = searchCache[query];
+            element.Results = results;
+            searchCache[query] = element;
+            return Ok(new SearchdomainUpdateSearchResult() {Success = true});
+        }
+        return Ok(new SearchdomainUpdateSearchResult() {Success = false, Message = "Query not found in search cache"});
+    }
+
     [HttpGet("GetSettings")]
     public ActionResult<SearchdomainSettingsResults> GetSettings(string searchdomain)
     {

src/Server/Controllers/ServerController.cs

@@ -0,0 +1,38 @@
+namespace Server.Controllers;
+
+using System.Text.Json;
+using ElmahCore;
+using Microsoft.AspNetCore.Mvc;
+using Server.Exceptions;
+using Server.Helper;
+using Shared.Models;
+
+[ApiController]
+[Route("[controller]")]
+public class ServerController : ControllerBase
+{
+    private readonly ILogger<ServerController> _logger;
+    private readonly IConfiguration _config;
+    private AIProvider _aIProvider;
+
+    public ServerController(ILogger<ServerController> logger, IConfiguration config, AIProvider aIProvider)
+    {
+        _logger = logger;
+        _config = config;
+        _aIProvider = aIProvider;
+    }
+
+    [HttpGet("GetModels")]
+    public ActionResult<ServerGetModelsResult> GetModels()
+    {
+        try
+        {
+            string[] models = _aIProvider.GetModels();
+            return new ServerGetModelsResult() { Models = models, Success = true };
+        } catch (Exception ex)
+        {
+            _logger.LogError("Unable to get models due to exception {ex.Message} - {ex.StackTrace}", [ex.Message, ex.StackTrace]);
+            return new ServerGetModelsResult() { Success = false, Message = ex.Message};
+        }
+    }
+}

src/Server/Entity.cs

@@ -1,9 +1,10 @@
 namespace Server;
 
-public class Entity(Dictionary<string, string> attributes, Probmethods.probMethodDelegate probMethod, List<Datapoint> datapoints, string name)
+public class Entity(Dictionary<string, string> attributes, Probmethods.probMethodDelegate probMethod, string probMethodName, List<Datapoint> datapoints, string name)
 {
     public Dictionary<string, string> attributes = attributes;
     public Probmethods.probMethodDelegate probMethod = probMethod;
+    public string probMethodName = probMethodName;
     public List<Datapoint> datapoints = datapoints;
     public int id;
     public string name = name;

src/Server/Helper/SearchdomainHelper.cs

@@ -44,8 +44,12 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
         return null;
     }
 
-    public List<Entity>? EntitiesFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, string json)
+    public List<Entity>? EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json)
     {
+        Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
+        AIProvider aIProvider = searchdomainManager.aIProvider;
+        SQLHelper helper = searchdomainManager.helper;
+
         List<JSONEntity>? jsonEntities = JsonSerializer.Deserialize<List<JSONEntity>>(json);
         if (jsonEntities is null)
         {
@@ -72,8 +76,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
         ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = 16 }; // <-- This is needed! Otherwise if we try to index 100+ entities at once, it spawns 100 threads, exploding the SQL pool
         Parallel.ForEach(jsonEntities, parallelOptions, jSONEntity =>
         {
-            using var tempHelper = helper.DuplicateConnection();
+            var entity = EntityFromJSON(searchdomainManager, logger, jSONEntity);
-            var entity = EntityFromJSON(entityCache, embeddingCache, aIProvider, tempHelper, logger, jSONEntity);
             if (entity is not null)
             {
                 retVal.Enqueue(entity);
@@ -82,87 +85,188 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
         return [.. retVal];
     }
 
-    public Entity? EntityFromJSON(List<Entity> entityCache, Dictionary<string, Dictionary<string, float[]>> embeddingCache, AIProvider aIProvider, SQLHelper helper, ILogger logger, JSONEntity jsonEntity) //string json)
+    public Entity? EntityFromJSON(SearchdomainManager searchdomainManager, ILogger logger, JSONEntity jsonEntity) //string json)
     {
-        Dictionary<string, Dictionary<string, float[]>> embeddingsLUT = []; // embeddingsLUT: hash -> model -> [embeddingValues * n]
+        SQLHelper helper = searchdomainManager.helper.DuplicateConnection();
-        int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
+        Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain);
-        if (preexistingEntityID is not null)
+        List<Entity> entityCache = searchdomain.entityCache;
-        {
+        AIProvider aIProvider = searchdomain.aIProvider;
-            lock (helper.connection) // TODO change this to helper and do A/B tests (i.e. before/after)
+        Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
-            {
+        Entity? preexistingEntity = entityCache.FirstOrDefault(entity => entity.name == jsonEntity.Name);
-                Dictionary<string, dynamic> parameters = new()
-                {
-                    { "id", preexistingEntityID }
-                };
-                System.Data.Common.DbDataReader reader = helper.ExecuteSQLCommand("SELECT e.model, e.embedding, d.hash FROM datapoint d JOIN embedding e ON d.id = e.id_datapoint WHERE d.id_entity = @id", parameters);
-                while (reader.Read())
-                {
-                    string model = reader.GetString(0);
-                    long length = reader.GetBytes(1, 0, null, 0, 0);
-                    byte[] embeddingBytes = new byte[length];
-                    reader.GetBytes(1, 0, embeddingBytes, 0, (int)length);
-                    float[] embeddingValues = FloatArrayFromBytes(embeddingBytes);
-                    string hash = reader.GetString(2);
-                    if (!embeddingsLUT.ContainsKey(hash))
-                    {
-                        embeddingsLUT[hash] = [];
-                    }
-                    embeddingsLUT[hash].TryAdd(model, embeddingValues);
-                }
-                reader.Close();
-            }
-            _databaseHelper.RemoveEntity(entityCache, helper, jsonEntity.Name, jsonEntity.Searchdomain); // TODO only remove entity if there is actually a change somewhere. Perhaps create 3 datapoint lists to operate with: 1. delete, 2. update, 3. create
-        }
-        int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
-        foreach (KeyValuePair<string, string> attribute in jsonEntity.Attributes)
-        {
-            DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries
-        }
         
-        List<Datapoint> datapoints = [];
+        if (preexistingEntity is not null)
-        foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)
         {
-            string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
+            int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
-            Dictionary<string, float[]> embeddings = [];
+            if (preexistingEntityID is null)
-            if (embeddingsLUT.ContainsKey(hash))
             {
-                Dictionary<string, float[]> hashLUT = embeddingsLUT[hash];
+                _logger.LogCritical("Unable to index entity {jsonEntity.Name} because it already exists in the searchdomain but not in the database.", [jsonEntity.Name]);
-                foreach (string model in jsonDatapoint.Model)
+                throw new Exception($"Unable to index entity {jsonEntity.Name} because it already exists in the searchdomain but not in the database.");
+            }
+            Dictionary<string, string> attributes = jsonEntity.Attributes;
+            
+            // Attribute
+            foreach (KeyValuePair<string, string> attributesKV in preexistingEntity.attributes.ToList())
+            {
+                string oldAttributeKey = attributesKV.Key;
+                string oldAttribute = attributesKV.Value;
+                bool newHasAttribute = jsonEntity.Attributes.TryGetValue(oldAttributeKey, out string? newAttribute);
+                if (newHasAttribute && newAttribute is not null && newAttribute != oldAttribute)
                 {
-                    if (hashLUT.ContainsKey(model))
+                    // Attribute - Updated
+                    Dictionary<string, dynamic> parameters = new()
                     {
-                        embeddings.Add(model, hashLUT[model]);
+                        { "newValue", newAttribute },
+                        { "entityId", preexistingEntityID },
+                        { "attribute", oldAttributeKey}
+                    };
+                    helper.ExecuteSQLNonQuery("UPDATE attribute SET value=@newValue WHERE id_entity=@entityId AND attribute=@attribute", parameters);
+                    preexistingEntity.attributes[oldAttributeKey] = newAttribute;
+                } else if (!newHasAttribute)
+                {
+                    // Attribute - Deleted
+                    Dictionary<string, dynamic> parameters = new()
+                    {
+                        { "entityId", preexistingEntityID },
+                        { "attribute", oldAttributeKey}
+                    };
+                    helper.ExecuteSQLNonQuery("DELETE FROM attribute WHERE id_entity=@entityId AND attribute=@attribute", parameters);
+                    preexistingEntity.attributes.Remove(oldAttributeKey);
+                }
+            }
+            foreach (var attributesKV in jsonEntity.Attributes)
+            {
+                string newAttributeKey = attributesKV.Key;
+                string newAttribute = attributesKV.Value;
+                bool preexistingHasAttribute = preexistingEntity.attributes.TryGetValue(newAttributeKey, out string? preexistingAttribute);
+                if (!preexistingHasAttribute)
+                {
+                    // Attribute - New
+                    DatabaseHelper.DatabaseInsertAttribute(helper, newAttributeKey, newAttribute, (int)preexistingEntityID);
+                    preexistingEntity.attributes.Add(newAttributeKey, newAttribute);
+                }
+            }
+
+            // Datapoint
+            foreach (Datapoint datapoint in preexistingEntity.datapoints.ToList())
+            {
+                bool newEntityHasDatapoint = jsonEntity.Datapoints.Any(x => x.Name == datapoint.name);
+                if (!newEntityHasDatapoint)
+                {
+                    // Datapoint - Deleted
+                    Dictionary<string, dynamic> parameters = new()
+                    {
+                        { "datapointName", datapoint.name },
+                        { "entityId", preexistingEntityID}
+                    };
+                    helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters);
+                    helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters);
+                    preexistingEntity.datapoints.Remove(datapoint);
+                } else
+                {
+                    JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.name);
+                    if (newEntityDatapoint is not null && newEntityDatapoint.Text is not null)
+                    {
+                        // Datapoint - Updated (text)
+                        Dictionary<string, dynamic> parameters = new()
+                        {
+                            { "datapointName", datapoint.name },
+                            { "entityId", preexistingEntityID}
+                        };
+                        helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters);
+                        helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters);
+                        preexistingEntity.datapoints.Remove(datapoint);
+                        Datapoint newDatapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, newEntityDatapoint, (int)preexistingEntityID);
+                        preexistingEntity.datapoints.Add(newDatapoint);
+
                     }
-                    else
+                    if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.name || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.name))
                     {
-                        var additionalEmbeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [model], aIProvider, embeddingCache);
+                        // Datapoint - Updated (probmethod or similaritymethod)
-                        embeddings.Add(model, additionalEmbeddings.First().Value);
+                        Dictionary<string, dynamic> parameters = new()
+                        {
+                            { "probmethod", newEntityDatapoint.Probmethod_embedding },
+                            { "similaritymethod", newEntityDatapoint.SimilarityMethod },
+                            { "datapointName", datapoint.name },
+                            { "entityId", preexistingEntityID}
+                        };
+                        helper.ExecuteSQLNonQuery("UPDATE datapoint SET probmethod_embedding=@probmethod, similaritymethod=@similaritymethod WHERE id_entity=@entityId AND name=@datapointName", parameters);
+                        Datapoint preexistingDatapoint = preexistingEntity.datapoints.First(x => x == datapoint); // The for loop is a copy. This retrieves the original such that it can be updated.
+                        preexistingDatapoint.probMethod = datapoint.probMethod;
+                        preexistingDatapoint.similarityMethod = datapoint.similarityMethod;
                     }
                 }
             }
-            else
+            foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)
             {
-                embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], aIProvider, embeddingCache);
+                bool oldEntityHasDatapoint = preexistingEntity.datapoints.Any(x => x.name == jsonDatapoint.Name);
+                if (!oldEntityHasDatapoint)
+                {
+                    // Datapoint - New
+                    Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, (int)preexistingEntityID);
+                    preexistingEntity.datapoints.Add(datapoint);
+                }
             }
-            var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding, logger) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding);
-            var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
-            Datapoint datapoint = new(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]);
-            int id_datapoint = DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, id_entity); // TODO make this a bulk add action to reduce number of queries
-            List<(string model, byte[] embedding)> data = [];
-            foreach ((string, float[]) embedding in datapoint.embeddings)
-            {
-                data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2)));
-            }
-            DatabaseHelper.DatabaseInsertEmbeddingBulk(helper, id_datapoint, data);
-            datapoints.Add(datapoint);
-        }
 
-        var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
+
-        Entity entity = new(jsonEntity.Attributes, probMethod, datapoints, jsonEntity.Name)
+            return preexistingEntity;
+        }
+        else
         {
-            id = id_entity
+            int id_entity = DatabaseHelper.DatabaseInsertEntity(helper, jsonEntity.Name, jsonEntity.Probmethod, _databaseHelper.GetSearchdomainID(helper, jsonEntity.Searchdomain));
-        };
+            foreach (KeyValuePair<string, string> attribute in jsonEntity.Attributes)
-        entityCache.Add(entity);
+            {
-        return entity;
+                DatabaseHelper.DatabaseInsertAttribute(helper, attribute.Key, attribute.Value, id_entity); // TODO implement bulk insert to reduce number of queries
+            }
+
+            List<Datapoint> datapoints = [];
+            foreach (JSONDatapoint jsonDatapoint in jsonEntity.Datapoints)
+            {
+                string hash = Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
+                Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, id_entity, hash);
+                datapoints.Add(datapoint);
+            }
+
+            var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
+            Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod, datapoints, jsonEntity.Name)
+            {
+                id = id_entity
+            };
+            entityCache.Add(entity);
+            return entity;
+        }
+    }
+
+    public Datapoint DatabaseInsertDatapointWithEmbeddings(SQLHelper helper, Searchdomain searchdomain, JSONDatapoint jsonDatapoint, int id_entity, string? hash = null)
+    {
+        if (jsonDatapoint.Text is null)
+        {
+            throw new Exception("jsonDatapoint.Text must not be null at this point");
+        }
+        hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
+        Datapoint datapoint = BuildDatapointFromJsonDatapoint(jsonDatapoint, id_entity, searchdomain, hash);
+        int id_datapoint = DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, id_entity); // TODO make this a bulk add action to reduce number of queries
+        List<(string model, byte[] embedding)> data = [];
+        foreach ((string, float[]) embedding in datapoint.embeddings)
+        {
+            data.Add((embedding.Item1, BytesFromFloatArray(embedding.Item2)));
+        }
+        DatabaseHelper.DatabaseInsertEmbeddingBulk(helper, id_datapoint, data);
+        return datapoint;
+    }
+
+    public Datapoint BuildDatapointFromJsonDatapoint(JSONDatapoint jsonDatapoint, int entityId, Searchdomain searchdomain, string? hash = null)
+    {
+        if (jsonDatapoint.Text is null)
+        {
+            throw new Exception("jsonDatapoint.Text must not be null at this point");
+        }
+        using SQLHelper helper = searchdomain.helper.DuplicateConnection();
+        Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
+        hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
+        DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId);
+        Dictionary<string, float[]> embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.aIProvider, embeddingCache);
+        var probMethod_embedding = new ProbMethod(jsonDatapoint.Probmethod_embedding, logger) ?? throw new ProbMethodNotFoundException(jsonDatapoint.Probmethod_embedding);
+        var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
+        return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]);
     }
 }

src/Server/Resources/SharedResources.de.resx

@@ -18,4 +18,10 @@
   <data name="InvalidCredentials" xml:space="preserve">
     <value>Ungültiger Benutzername oder Passwort.</value>
   </data>
+  <data name="GenericDeleteConfirmation" xml:space="preserve">
+    <value>Sind Sie sicher, dass Sie &lt;strong id="{0}"&gt;diese Entity&lt;/strong&gt; löschen wollen?</value>
+  </data>
+  <data name="IrreversibleActionWarning" xml:space="preserve">
+    <value>Diese Aktion kann nicht rückgängig gemacht werden.</value>
+  </data>
 </root>

src/Server/Resources/SharedResources.en.resx

@@ -18,4 +18,10 @@
   <data name="InvalidCredentials" xml:space="preserve">
     <value>Invalid credentials.</value>
   </data>
+  <data name="GenericDeleteConfirmation" xml:space="preserve">
+    <value>Are you sure you want to delete &lt;strong id="{0}"&gt;this entity&lt;/strong&gt;?</value>
+  </data>
+  <data name="IrreversibleActionWarning" xml:space="preserve">
+    <value>This action cannot be undone.</value>
+  </data>
 </root>

src/Server/Searchdomain.cs

@@ -142,7 +142,7 @@ public class Searchdomain
             Probmethods.probMethodDelegate? probmethod = Probmethods.GetMethod(probmethodString);
             if (datapoint_unassigned.TryGetValue(id, out List<Datapoint>? datapoints) && probmethod is not null)
             {
-                Entity entity = new(attributes, probmethod, datapoints, name)
+                Entity entity = new(attributes, probmethod, probmethodString, datapoints, name)
                 {
                     id = id
                 };
@@ -154,7 +154,7 @@ public class Searchdomain
         embeddingCache = []; // TODO remove this and implement proper remediation to improve performance
     }
 
-    public List<(float, string)> Search(string query)
+    public List<(float, string)> Search(string query, int? topN = null)
     {
         if (searchCache.TryGetValue(query, out DateTimedSearchResult cachedResult))
         {
@@ -190,9 +190,14 @@ public class Searchdomain
             }
             result.Add((entity.probMethod(datapointProbs), entity.name));
         }
-        List<(float, string)> results = [.. result.OrderByDescending(s => s.Item1)];
+        IEnumerable<(float, string)> sortedResults = result.OrderByDescending(s => s.Item1);
+        if (topN is not null)
+        {
+            sortedResults = sortedResults.Take(topN ?? 0);
+        }
+        List<(float, string)> results = [.. sortedResults];
         List<ResultItem> searchResult = new(
-            [.. results.Select(r =>
+            [.. sortedResults.Select(r =>
                 new ResultItem(r.Item1, r.Item2 ))]
         );
         searchCache[query] = new DateTimedSearchResult(DateTime.Now, searchResult);

src/Server/SimilarityMethods.cs

@@ -21,20 +21,28 @@ public class SimilarityMethod
     }
 }
 
+public enum SimilarityMethodEnum
+{
+    Cosine,
+    Euclidian,
+    Manhattan,
+    Pearson
+}
+
 public static class SimilarityMethods
 {
     public delegate float similarityMethodProtoDelegate(float[] vector1, float[] vector2);
     public delegate float similarityMethodDelegate(float[] vector1, float[] vector2);
-    public static readonly Dictionary<string, similarityMethodProtoDelegate> probMethods;
+    public static readonly Dictionary<SimilarityMethodEnum, similarityMethodProtoDelegate> probMethods;
 
     static SimilarityMethods()
     {
-        probMethods = new Dictionary<string, similarityMethodProtoDelegate>
+        probMethods = new Dictionary<SimilarityMethodEnum, similarityMethodProtoDelegate>
         {
-            ["Cosine"] = CosineSimilarity,
+            [SimilarityMethodEnum.Cosine] = CosineSimilarity,
-            ["Euclidian"] = EuclidianDistance,
+            [SimilarityMethodEnum.Euclidian] = EuclidianDistance,
-            ["Manhattan"] = ManhattanDistance,
+            [SimilarityMethodEnum.Manhattan] = ManhattanDistance,
-            ["Pearson"] = PearsonCorrelation
+            [SimilarityMethodEnum.Pearson] = PearsonCorrelation
         };
     }
 
@@ -42,7 +50,12 @@ public static class SimilarityMethods
     {
         string methodName = name;
 
-        if (!probMethods.TryGetValue(methodName, out similarityMethodProtoDelegate? method))
+        SimilarityMethodEnum probMethodEnum = (SimilarityMethodEnum)Enum.Parse(
+            typeof(SimilarityMethodEnum),
+            methodName
+        );
+
+        if (!probMethods.TryGetValue(probMethodEnum, out similarityMethodProtoDelegate? method))
         {
             return null;
         }

src/Shared/Models/EntityResults.cs

@@ -41,6 +41,8 @@ public class EntityListResult
 {
     [JsonPropertyName("Name")]
     public required string Name { get; set; }
+    [JsonPropertyName("ProbMethod")]
+    public required string ProbMethod { get; set; }
     [JsonPropertyName("Attributes")]
     public required List<AttributeResult> Attributes { get; set; }
     [JsonPropertyName("Datapoints")]

src/Shared/Models/JSONModels.cs

@@ -12,7 +12,7 @@ public class JSONEntity
 public class JSONDatapoint
 {
     public required string Name { get; set; }
-    public required string Text { get; set; }
+    public required string? Text { get; set; }
     public required string Probmethod_embedding { get; set; }
     public required string SimilarityMethod { get; set; }
     public required string[] Model { get; set; }

src/Shared/Models/SearchdomainModels.cs

@@ -2,12 +2,19 @@
 using System.Text.Json.Serialization;
 
 namespace Shared.Models;
-public readonly struct ResultItem(float score, string name)
+public readonly struct ResultItem
 {
     [JsonPropertyName("Score")]
-    public readonly float Score { get; } = score;
+    public readonly float Score { get; }
     [JsonPropertyName("Name")]
-    public readonly string Name { get; } = name;
+    public readonly string Name { get; }
+
+    [JsonConstructor]
+    public ResultItem(float score, string name)
+    {
+        Score = score;
+        Name = name;
+    }
 
     public static long EstimateSize(ResultItem item)
     {

src/Shared/Models/SearchdomainResults.cs

@@ -55,6 +55,24 @@ public class SearchdomainSearchesResults
     public required Dictionary<string, DateTimedSearchResult> Searches { get; set; }
 }
 
+public class SearchdomainDeleteSearchResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+
+    [JsonPropertyName("Message")]
+    public string? Message { get; set; }    
+}
+
+public class SearchdomainUpdateSearchResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+
+    [JsonPropertyName("Message")]
+    public string? Message { get; set; }    
+}
+
 public class SearchdomainSettingsResults
 {
     [JsonPropertyName("Success")]

src/Shared/Models/ServerModels.cs

@@ -0,0 +1,15 @@
+using System.Text.Json.Serialization;
+
+namespace Shared.Models;
+
+public class ServerGetModelsResult
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+
+    [JsonPropertyName("Message")]
+    public string? Message { get; set; }
+
+    [JsonPropertyName("Models")]
+    public string[]? Models { get; set; }    
+}