Merge pull request #58 from LD-Reborn/56-bug-exception-when-update-indexing-entity

Fixed datapoint stale reference causing issues when updating datapoin…

src/Client/Client.cs

@@ -89,7 +89,7 @@ public class Client
 
         public async Task<EntityQueryResults> EntityQueryAsync(string searchdomain, string query)
         {
-            return await GetUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Entity", "Query", apiKey, new Dictionary<string, string>()
+            return await GetUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, new Dictionary<string, string>()
             {
                 {"searchdomain", searchdomain},
                 {"query", query}

src/Server/Controllers/AccountController.cs

@@ -6,6 +6,7 @@ using Server.Models;
 
 namespace Server.Controllers;
 
+[ApiExplorerSettings(IgnoreApi = true)]
 [Route("[Controller]")]
 public class AccountController : Controller
 {

src/Server/Controllers/EntityController.cs

@@ -24,31 +24,6 @@ public class EntityController : ControllerBase
         _databaseHelper = databaseHelper;
     }
 
-    [HttpGet("Query")]
-    public ActionResult<EntityQueryResults> Query(string searchdomain, string query, int? topN)
-    {
-        Searchdomain searchdomain_;
-        try
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        } catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
-        } catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
-        }
-        List<(float, string)> results = searchdomain_.Search(query, topN);
-        List<EntityQueryResult> queryResults = [.. results.Select(r => new EntityQueryResult
-        {
-            Name = r.Item2,
-            Value = r.Item1
-        })];
-        return Ok(new EntityQueryResults(){Results = queryResults, Success = true });
-    }
-
     [HttpPost("Index")]
     public ActionResult<EntityIndexResult> Index([FromBody] List<JSONEntity>? jsonEntities)
     {
@@ -69,7 +44,6 @@ public class EntityController : ControllerBase
                         && !invalidatedSearchdomains.Contains(jsonEntitySearchdomainName))
                     {
                         invalidatedSearchdomains.Add(jsonEntitySearchdomainName);
-                        _domainManager.InvalidateSearchdomainCache(jsonEntitySearchdomainName);
                     }
                 }
                 return Ok(new EntityIndexResult() { Success = true });
@@ -94,21 +68,10 @@ public class EntityController : ControllerBase
         if (returnEmbeddings && !returnModels)
         {
             _logger.LogError("Invalid request for {searchdomain} - embeddings return requested but without models - not possible!", [searchdomain]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Invalid request" });            
-        }
-        Searchdomain searchdomain_;
-        try
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        } catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
-        } catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
+            return BadRequest(new EntityListResults() {Results = [], Success = false, Message = "Invalid request" });            
         }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
         EntityListResults entityListResults = new() {Results = [], Success = true};
         foreach (Entity entity in searchdomain_.entityCache)
         {
@@ -149,19 +112,8 @@ public class EntityController : ControllerBase
     [HttpGet("Delete")]
     public ActionResult<EntityDeleteResults> Delete(string searchdomain, string entityName)
     {
-        Searchdomain searchdomain_;
-        try
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        } catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
-        } catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
-        }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
         
         Entity? entity_ = SearchdomainHelper.CacheGetEntity(searchdomain_.entityCache, entityName);
         if (entity_ is null)
@@ -169,6 +121,7 @@ public class EntityController : ControllerBase
             _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]);
             return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"});
         }
+        searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity_);
         _databaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain);
         searchdomain_.entityCache.RemoveAll(entity => entity.name == entityName);
         return Ok(new EntityDeleteResults() {Success = true});

src/Server/Controllers/Frontend/HomeController.cs

@@ -7,7 +7,7 @@ using Server.Exceptions;
 using Server.Models;
 namespace Server.Controllers;
 
-[ApiController]
+[ApiExplorerSettings(IgnoreApi = true)]
 [Route("/")]
 public class HomeController : Controller
 {

src/Server/Controllers/SearchdomainController.cs

@@ -87,71 +87,53 @@ public class SearchdomainController : ControllerBase
     [HttpGet("Update")]
     public ActionResult<SearchdomainUpdateResults> Update(string searchdomain, string newName, string settings = "{}")
     {
-        try
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
+        Dictionary<string, dynamic> parameters = new()
         {
-            Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-            Dictionary<string, dynamic> parameters = new()
-            {
-                {"name", newName},
-                {"settings", settings},
-                {"id", searchdomain_.id}
-            };
-            searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters);
-        } catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to update searchdomain {searchdomain} - not found", [searchdomain]);
-            return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update searchdomain {searchdomain} - not found" });
-        } catch (Exception ex)
-        {
-            _logger.LogError("Unable to update searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update searchdomain {searchdomain}" });
-        }
+            {"name", newName},
+            {"settings", settings},
+            {"id", searchdomain_.id}
+        };
+        searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters);
         return Ok(new SearchdomainUpdateResults(){Success = true});
     }
 
+    [HttpGet("Query")]
+    public ActionResult<EntityQueryResults> Query(string searchdomain, string query, int? topN, bool returnAttributes = false)
+    {
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
+        List<(float, string)> results = searchdomain_.Search(query, topN);
+        List<EntityQueryResult> queryResults = [.. results.Select(r => new EntityQueryResult
+        {
+            Name = r.Item2,
+            Value = r.Item1,
+            Attributes = returnAttributes ? (searchdomain_.entityCache.FirstOrDefault(x => x.name == r.Item2)?.attributes ?? null) : null
+        })];
+        return Ok(new EntityQueryResults(){Results = queryResults, Success = true });
+    }
+
     [HttpPost("UpdateSettings")]
     public ActionResult<SearchdomainUpdateResults> UpdateSettings(string searchdomain, [FromBody] SearchdomainSettings request)
     {
-        try
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
+        Dictionary<string, dynamic> parameters = new()
         {
-            Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-            Dictionary<string, dynamic> parameters = new()
-            {
-                {"settings", JsonSerializer.Serialize(request)},
-                {"id", searchdomain_.id}
-            };
-            searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters);
-            searchdomain_.settings = request;
-        } catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to update settings for searchdomain {searchdomain} - not found", [searchdomain]);
-            return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update settings for searchdomain {searchdomain} - not found" });
-        } catch (Exception ex)
-        {
-            _logger.LogError("Unable to update settings for searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update settings for searchdomain {searchdomain}" });
-        }
+            {"settings", JsonSerializer.Serialize(request)},
+            {"id", searchdomain_.id}
+        };
+        searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters);
+        searchdomain_.settings = request;
         return Ok(new SearchdomainUpdateResults(){Success = true});
     }
 
     [HttpGet("GetSearches")]
     public ActionResult<SearchdomainSearchesResults> GetSearches(string searchdomain)
     {
-        Searchdomain searchdomain_;
-        try
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        }
-        catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new SearchdomainSearchesResults() { Searches = [], Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainSearchesResults() { Searches = [], Success = false, Message = ex.Message });
-        }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
         Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
         
         return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true });
@@ -160,21 +142,8 @@ public class SearchdomainController : ControllerBase
     [HttpDelete("Searches")]
     public ActionResult<SearchdomainDeleteSearchResult> DeleteSearch(string searchdomain, string query)
     {
-        Searchdomain searchdomain_;
-        try
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        }
-        catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = ex.Message });
-        }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
         Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
         bool containsKey = searchCache.ContainsKey(query);
         if (containsKey)
@@ -188,21 +157,8 @@ public class SearchdomainController : ControllerBase
     [HttpPatch("Searches")]
     public ActionResult<SearchdomainUpdateSearchResult> UpdateSearch(string searchdomain, string query, [FromBody]List<ResultItem> results)
     {
-        Searchdomain searchdomain_;
-        try
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        }
-        catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = ex.Message });
-        }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
         Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
         bool containsKey = searchCache.ContainsKey(query);
         if (containsKey)
@@ -218,21 +174,8 @@ public class SearchdomainController : ControllerBase
     [HttpGet("GetSettings")]
     public ActionResult<SearchdomainSettingsResults> GetSettings(string searchdomain)
     {
-        Searchdomain searchdomain_;
-        try
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        }
-        catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new SearchdomainSettingsResults() { Settings = null, Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainSettingsResults() { Settings = null, Success = false, Message = ex.Message });
-        }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
         SearchdomainSettings settings = searchdomain_.settings;
         return Ok(new SearchdomainSettingsResults() { Settings = settings, Success = true });
     }
@@ -240,21 +183,8 @@ public class SearchdomainController : ControllerBase
     [HttpGet("GetSearchCacheSize")]
     public ActionResult<SearchdomainSearchCacheSizeResults> GetSearchCacheSize(string searchdomain)
     {
-        Searchdomain searchdomain_;
-        try
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        }
-        catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = null, Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = null, Success = false, Message = ex.Message });
-        }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
         Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
         long sizeInBytes = 0;
         foreach (var entry in searchCache)
@@ -269,40 +199,17 @@ public class SearchdomainController : ControllerBase
     [HttpGet("ClearSearchCache")]
     public ActionResult<SearchdomainInvalidateCacheResults> InvalidateSearchCache(string searchdomain)
     {
-        try
-        {
-            Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-            searchdomain_.InvalidateSearchCache();
-        } catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to invalidate search cache for searchdomain {searchdomain} - not found", [searchdomain]);
-            return Ok(new SearchdomainInvalidateCacheResults() { Success = false, Message = $"Unable to invalidate search cache for searchdomain {searchdomain} - not found" });
-        } catch (Exception ex)
-        {
-            _logger.LogError("Unable to invalidate search cache for searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainInvalidateCacheResults() { Success = false, Message = $"Unable to invalidate search cache for searchdomain {searchdomain}" });
-        }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
+        searchdomain_.InvalidateSearchCache();
         return Ok(new SearchdomainInvalidateCacheResults(){Success = true});
     }
 
     [HttpGet("GetDatabaseSize")]
     public ActionResult<SearchdomainGetDatabaseSizeResult> GetDatabaseSize(string searchdomain)
     {
-        Searchdomain searchdomain_;
-        try
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        }
-        catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = null, Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = null, Success = false, Message = ex.Message });
-        }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
         long sizeInBytes = DatabaseHelper.GetSearchdomainDatabaseSize(searchdomain_.helper, searchdomain);
         return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = sizeInBytes, Success = true });        
     } 

src/Server/Datapoint.cs

@@ -1,3 +1,4 @@
+using AdaptiveExpressions;
 using OllamaSharp;
 using OllamaSharp.Models;
 
@@ -27,74 +28,31 @@ public class Datapoint
 
     public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider)
     {
-        return GenerateEmbeddings(content, models, aIProvider, []);
+        return GenerateEmbeddings(content, models, aIProvider, new());
     }
 
-    public static Dictionary<string, float[]> GenerateEmbeddings(List<string> contents, string model, OllamaApiClient ollama, Dictionary<string, Dictionary<string, float[]>> embeddingCache)
-    {
-        Dictionary<string, float[]> retVal = [];
-
-        List<string> remainingContents = new List<string>(contents);
-        for (int i = contents.Count - 1; i >= 0; i--) // Compare against cache and remove accordingly
-        {
-            string content = contents[i];
-            if (embeddingCache.ContainsKey(model) && embeddingCache[model].ContainsKey(content))
-            {
-                retVal[content] = embeddingCache[model][content];
-                remainingContents.RemoveAt(i);
-            }
-        }
-        if (remainingContents.Count == 0)
-        {
-            return retVal;
-        } 
-
-        EmbedRequest request = new()
-        {
-            Model = model,
-            Input = remainingContents
-        };
-
-        EmbedResponse response = ollama.EmbedAsync(request).Result;
-        for (int i = 0; i < response.Embeddings.Count; i++)
-        {
-            string content = remainingContents.ElementAt(i);
-            float[] embeddings = response.Embeddings.ElementAt(i);
-            retVal[content] = embeddings;
-            if (!embeddingCache.ContainsKey(model))
-            {
-                embeddingCache[model] = [];
-            }
-            if (!embeddingCache[model].ContainsKey(content))
-            {
-                embeddingCache[model][content] = embeddings;
-            }
-        }
-
-        return retVal;
-    }
-
-    public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider, Dictionary<string, Dictionary<string, float[]>> embeddingCache)
+    public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider, LRUCache<string, Dictionary<string, float[]>> embeddingCache)
     {
         Dictionary<string, float[]> retVal = [];
         foreach (string model in models)
         {
-            if (embeddingCache.ContainsKey(model) && embeddingCache[model].ContainsKey(content))
+            bool embeddingCacheHasModel = embeddingCache.TryGet(model, out var embeddingCacheForModel);
+            if (embeddingCacheHasModel && embeddingCacheForModel.ContainsKey(content))
             {
-                retVal[model] = embeddingCache[model][content];
+                retVal[model] = embeddingCacheForModel[content];
                 continue;
             }
             var response = aIProvider.GenerateEmbeddings(model, [content]);
             if (response is not null)
             {
                 retVal[model] = response;
-                if (!embeddingCache.ContainsKey(model))
+                if (!embeddingCacheHasModel)
                 {
-                    embeddingCache[model] = [];
+                    embeddingCacheForModel = [];
                 }
-                if (!embeddingCache[model].ContainsKey(content))
+                if (!embeddingCacheForModel.ContainsKey(content))
                 {
-                    embeddingCache[model][content] = response;
+                    embeddingCacheForModel[content] = response;
                 }
             }
         }

src/Server/Exceptions/GeneralExceptions.cs

@@ -1,7 +1,9 @@
+using Shared.Models;
+
 namespace Server.Exceptions;
 
-public class ProbMethodNotFoundException(string probMethod) : Exception($"Unknown probMethod name {probMethod}") { }
+public class ProbMethodNotFoundException(ProbMethodEnum probMethod) : Exception($"Unknown probMethod name {probMethod}") { }
 
-public class SimilarityMethodNotFoundException(string similarityMethod) : Exception($"Unknown similarityMethod name \"{similarityMethod}\"") { }
+public class SimilarityMethodNotFoundException(SimilarityMethodEnum similarityMethod) : Exception($"Unknown similarityMethod name \"{similarityMethod}\"") { }
 
 public class JSONPathSelectionException(string path, string testedContent) : Exception($"Unable to select tokens using JSONPath {path} for string: {testedContent}.") { }

src/Server/Helper/DatabaseHelper.cs

@@ -38,12 +38,12 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
         return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO searchdomain (name, settings) VALUES (@name, @settings)", parameters);
     }
 
-    public static int DatabaseInsertEntity(SQLHelper helper, string name, string probmethod, int id_searchdomain)
+    public static int DatabaseInsertEntity(SQLHelper helper, string name, ProbMethodEnum probmethod, int id_searchdomain)
     {
         Dictionary<string, dynamic> parameters = new()
         {
             { "name", name },
-            { "probmethod", probmethod },
+            { "probmethod", probmethod.ToString() },
             { "id_searchdomain", id_searchdomain }
         };
         return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO entity (name, probmethod, id_searchdomain) VALUES (@name, @probmethod, @id_searchdomain)", parameters);
@@ -60,13 +60,13 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
         return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO attribute (attribute, value, id_entity) VALUES (@attribute, @value, @id_entity)", parameters);
     }
 
-    public static int DatabaseInsertDatapoint(SQLHelper helper, string name, string probmethod_embedding, string similarityMethod, string hash, int id_entity)
+    public static int DatabaseInsertDatapoint(SQLHelper helper, string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash, int id_entity)
     {
         Dictionary<string, dynamic> parameters = new()
         {
             { "name", name },
-            { "probmethod_embedding", probmethod_embedding },
-            { "similaritymethod", similarityMethod },
+            { "probmethod_embedding", probmethod_embedding.ToString() },
+            { "similaritymethod", similarityMethod.ToString() },
             { "hash", hash },
             { "id_entity", id_entity }
         };

src/Server/Helper/SearchdomainHelper.cs

@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
 using System.Security.Cryptography;
 using System.Text;
 using System.Text.Json;
+using AdaptiveExpressions;
 using Server.Exceptions;
 using Shared.Models;
 
@@ -46,7 +47,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
 
     public List<Entity>? EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json)
     {
-        Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
+        LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
         AIProvider aIProvider = searchdomainManager.aIProvider;
         SQLHelper helper = searchdomainManager.helper;
 
@@ -91,9 +92,10 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
         Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain);
         List<Entity> entityCache = searchdomain.entityCache;
         AIProvider aIProvider = searchdomain.aIProvider;
-        Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
+        LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
         Entity? preexistingEntity = entityCache.FirstOrDefault(entity => entity.name == jsonEntity.Name);
-        
+        bool invalidateSearchCache = false;
+
         if (preexistingEntity is not null)
         {
             int? preexistingEntityID = _databaseHelper.GetEntityID(helper, jsonEntity.Name, jsonEntity.Searchdomain);
@@ -147,8 +149,9 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             }
 
             // Datapoint
-            foreach (Datapoint datapoint in preexistingEntity.datapoints.ToList())
+            foreach (Datapoint datapoint_ in preexistingEntity.datapoints.ToList())
             {
+                Datapoint datapoint = datapoint_; // To enable replacing the datapoint reference as foreach iterators cannot be overwritten
                 bool newEntityHasDatapoint = jsonEntity.Datapoints.Any(x => x.Name == datapoint.name);
                 if (!newEntityHasDatapoint)
                 {
@@ -161,6 +164,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
                     helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters);
                     helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters);
                     preexistingEntity.datapoints.Remove(datapoint);
+                    invalidateSearchCache = true;
                 } else
                 {
                     JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.name);
@@ -177,22 +181,24 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
                         preexistingEntity.datapoints.Remove(datapoint);
                         Datapoint newDatapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, newEntityDatapoint, (int)preexistingEntityID);
                         preexistingEntity.datapoints.Add(newDatapoint);
-
+                        datapoint = newDatapoint;
+                        invalidateSearchCache = true;
                     }
-                    if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.name || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.name))
+                    if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.probMethodEnum || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.similarityMethodEnum))
                     {
                         // Datapoint - Updated (probmethod or similaritymethod)
                         Dictionary<string, dynamic> parameters = new()
                         {
-                            { "probmethod", newEntityDatapoint.Probmethod_embedding },
-                            { "similaritymethod", newEntityDatapoint.SimilarityMethod },
+                            { "probmethod", newEntityDatapoint.Probmethod_embedding.ToString() },
+                            { "similaritymethod", newEntityDatapoint.SimilarityMethod.ToString() },
                             { "datapointName", datapoint.name },
                             { "entityId", preexistingEntityID}
                         };
                         helper.ExecuteSQLNonQuery("UPDATE datapoint SET probmethod_embedding=@probmethod, similaritymethod=@similaritymethod WHERE id_entity=@entityId AND name=@datapointName", parameters);
                         Datapoint preexistingDatapoint = preexistingEntity.datapoints.First(x => x == datapoint); // The for loop is a copy. This retrieves the original such that it can be updated.
-                        preexistingDatapoint.probMethod = datapoint.probMethod;
-                        preexistingDatapoint.similarityMethod = datapoint.similarityMethod;
+                        preexistingDatapoint.probMethod = new(newEntityDatapoint.Probmethod_embedding, _logger);
+                        preexistingDatapoint.similarityMethod = new(newEntityDatapoint.SimilarityMethod, _logger);
+                        invalidateSearchCache = true;
                     }
                 }
             }
@@ -204,10 +210,14 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
                     // Datapoint - New
                     Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, (int)preexistingEntityID);
                     preexistingEntity.datapoints.Add(datapoint);
+                    invalidateSearchCache = true;
                 }
             }
 
-
+            if (invalidateSearchCache)
+            {
+                searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(preexistingEntity);
+            }
             return preexistingEntity;
         }
         else
@@ -227,11 +237,12 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             }
 
             var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
-            Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod, datapoints, jsonEntity.Name)
+            Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod.ToString(), datapoints, jsonEntity.Name)
             {
                 id = id_entity
             };
             entityCache.Add(entity);
+            searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(entity);
             return entity;
         }
     }
@@ -261,7 +272,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             throw new Exception("jsonDatapoint.Text must not be null at this point");
         }
         using SQLHelper helper = searchdomain.helper.DuplicateConnection();
-        Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
+        LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
         hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
         DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId);
         Dictionary<string, float[]> embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.aIProvider, embeddingCache);
@@ -269,4 +280,21 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
         var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
         return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]);
     }
+
+    public static (Searchdomain?, int?, string?) TryGetSearchdomain(SearchdomainManager searchdomainManager, string searchdomain, ILogger logger)
+    {
+        try
+        {
+            Searchdomain searchdomain_ = searchdomainManager.GetSearchdomain(searchdomain);
+            return (searchdomain_, null, null);
+        } catch (SearchdomainNotFoundException)
+        {
+            logger.LogError("Unable to update searchdomain {searchdomain} - not found", [searchdomain]);
+            return (null, 500, $"Unable to update searchdomain {searchdomain} - not found");
+        } catch (Exception ex)
+        {
+            logger.LogError("Unable to update searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
+            return (null, 404, $"Unable to update searchdomain {searchdomain}");
+        }
+    }
 }

src/Server/Probmethods.cs

@@ -1,37 +1,29 @@
 using System.Text.Json;
 using Server.Exceptions;
+using Shared.Models;
 
 namespace Server;
 
 public class ProbMethod
 {
     public Probmethods.probMethodDelegate method;
+    public ProbMethodEnum probMethodEnum;
     public string name;
 
-    public ProbMethod(string name, ILogger logger)
+    public ProbMethod(ProbMethodEnum probMethodEnum, ILogger logger)
     {
-        this.name = name;
+        this.probMethodEnum = probMethodEnum;
+        this.name = probMethodEnum.ToString();
         Probmethods.probMethodDelegate? probMethod = Probmethods.GetMethod(name);
         if (probMethod is null)
         {
             logger.LogError("Unable to retrieve probMethod {name}", [name]);
-            throw new ProbMethodNotFoundException(name);
+            throw new ProbMethodNotFoundException(probMethodEnum);
         }
         method = probMethod;
     }
 }
 
-public enum ProbMethodEnum
-{
-    Mean,
-    HarmonicMean,
-    QuadraticMean,
-    GeometricMean,
-    EVEWAvg,
-    HVEWAvg,
-    LVEWAvg,
-    DictionaryWeightedAverage
-}
 
 public static class Probmethods
 {
@@ -54,6 +46,11 @@ public static class Probmethods
         };
     }
 
+    public static probMethodDelegate? GetMethod(ProbMethodEnum probMethodEnum)
+    {
+        return GetMethod(probMethodEnum.ToString());
+    }
+
     public static probMethodDelegate? GetMethod(string name)
     {
         string methodName = name;

src/Server/Program.cs

@@ -8,12 +8,19 @@ using Server.HealthChecks;
 using Server.Helper;
 using Server.Models;
 using Server.Services;
+using System.Text.Json.Serialization;
 
 var builder = WebApplication.CreateBuilder(args);
 
 // Add services to the container.
 
-builder.Services.AddControllersWithViews();
+builder.Services.AddControllersWithViews()
+    .AddJsonOptions(options =>
+    {
+        options.JsonSerializerOptions.Converters.Add(
+            new JsonStringEnumConverter()
+        );
+    });
 
 // Add Localization
 builder.Services.AddLocalization(options => options.ResourcesPath = "Resources");

src/Server/Searchdomain.cs

@@ -5,6 +5,7 @@ using ElmahCore.Mvc.Logger;
 using MySql.Data.MySqlClient;
 using Server.Helper;
 using Shared.Models;
+using AdaptiveExpressions;
 
 namespace Server;
 
@@ -19,13 +20,12 @@ public class Searchdomain
     public Dictionary<string, DateTimedSearchResult> searchCache; // Key: query, Value: Search results for that query (with timestamp)
     public List<Entity> entityCache;
     public List<string> modelsInUse;
-    public Dictionary<string, Dictionary<string, float[]>> embeddingCache;
-    public int embeddingCacheMaxSize = 10000000;
+    public LRUCache<string, Dictionary<string, float[]>> embeddingCache;
     private readonly MySqlConnection connection;
     public SQLHelper helper;
     private readonly ILogger _logger;
 
-    public Searchdomain(string searchdomain, string connectionString, AIProvider aIProvider, Dictionary<string, Dictionary<string, float[]>> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false)
+    public Searchdomain(string searchdomain, string connectionString, AIProvider aIProvider, LRUCache<string, Dictionary<string, float[]>> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false)
     {
         _connectionString = connectionString;
         _provider = provider.ToLower();
@@ -96,8 +96,16 @@ public class Searchdomain
             string probmethodString = datapointReader.GetString(3);
             string similarityMethodString = datapointReader.GetString(4);
             string hash = datapointReader.GetString(5);
-            ProbMethod probmethod = new(probmethodString, _logger);
-            SimilarityMethod similarityMethod = new(similarityMethodString, _logger);
+            ProbMethodEnum probmethodEnum = (ProbMethodEnum)Enum.Parse(
+                typeof(ProbMethodEnum),
+                probmethodString
+            );
+            SimilarityMethodEnum similairtyMethodEnum = (SimilarityMethodEnum)Enum.Parse(
+                typeof(SimilarityMethodEnum),
+                similarityMethodString
+            );
+            ProbMethod probmethod = new(probmethodEnum, _logger);
+            SimilarityMethod similarityMethod = new(similairtyMethodEnum, _logger);
             if (embedding_unassigned.TryGetValue(id, out Dictionary<string, float[]>? embeddings) && probmethod is not null)
             {
                 embedding_unassigned.Remove(id);
@@ -151,7 +159,6 @@ public class Searchdomain
         }
         entityReader.Close();
         modelsInUse = GetModels(entityCache);
-        embeddingCache = []; // TODO remove this and implement proper remediation to improve performance
     }
 
     public List<(float, string)> Search(string query, int? topN = null)
@@ -162,33 +169,13 @@ public class Searchdomain
             return [.. cachedResult.Results.Select(r => (r.Score, r.Name))];
         }
 
-        if (!embeddingCache.TryGetValue(query, out Dictionary<string, float[]>? queryEmbeddings))
-        {
-            queryEmbeddings = Datapoint.GenerateEmbeddings(query, modelsInUse, aIProvider);
-            if (embeddingCache.Count < embeddingCacheMaxSize) // TODO add better way of managing cache limit hits
-            { // Idea: Add access count to each entry. On limit hit, sort the entries by access count and remove the bottom 10% of entries
-                embeddingCache.Add(query, queryEmbeddings);
-            }
-        } // TODO implement proper cache remediation for embeddingCache here
+        Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query);
 
         List<(float, string)> result = [];
 
         foreach (Entity entity in entityCache)
         {
-            List<(string, float)> datapointProbs = [];
-            foreach (Datapoint datapoint in entity.datapoints)
-            {
-                SimilarityMethod similarityMethod = datapoint.similarityMethod;
-                List<(string, float)> list = [];
-                foreach ((string, float[]) embedding in datapoint.embeddings)
-                {
-                    string key = embedding.Item1;
-                    float value = similarityMethod.method(queryEmbeddings[embedding.Item1], embedding.Item2);
-                    list.Add((key, value));
-                }
-                datapointProbs.Add((datapoint.name, datapoint.probMethod.method(list)));
-            }
-            result.Add((entity.probMethod(datapointProbs), entity.name));
+            result.Add((EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings), entity.name));
         }
         IEnumerable<(float, string)> sortedResults = result.OrderByDescending(s => s.Item1);
         if (topN is not null)
@@ -204,6 +191,49 @@ public class Searchdomain
         return results;
     }
 
+    public Dictionary<string, float[]> GetQueryEmbeddings(string query)
+    {
+        bool hasQuery = embeddingCache.TryGet(query, out Dictionary<string, float[]> queryEmbeddings);
+        bool allModelsInQuery = queryEmbeddings is not null && modelsInUse.All(model => queryEmbeddings.ContainsKey(model));
+        if (!(hasQuery && allModelsInQuery) || queryEmbeddings is null)
+        {
+            queryEmbeddings = Datapoint.GenerateEmbeddings(query, modelsInUse, aIProvider, embeddingCache);
+            if (!embeddingCache.TryGet(query, out var embeddingCacheForCurrentQuery))
+            {
+                embeddingCache.Set(query, queryEmbeddings);
+            }
+            else // embeddingCache already has an entry for this query, so the missing model-embedding pairs have to be filled in
+            {
+                foreach (KeyValuePair<string, float[]> kvp in queryEmbeddings) // kvp.Key = model, kvp.Value = embedding
+                {
+                    if (!embeddingCache.TryGet(kvp.Key, out var _))
+                    {
+                        embeddingCacheForCurrentQuery[kvp.Key] = kvp.Value;
+                    }
+                }
+            }
+        }
+        return queryEmbeddings;
+    }
+
+    private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary<string, float[]> queryEmbeddings)
+    {
+        List<(string, float)> datapointProbs = [];
+        foreach (Datapoint datapoint in entity.datapoints)
+        {
+            SimilarityMethod similarityMethod = datapoint.similarityMethod;
+            List<(string, float)> list = [];
+            foreach ((string, float[]) embedding in datapoint.embeddings)
+            {
+                string key = embedding.Item1;
+                float value = similarityMethod.method(queryEmbeddings[embedding.Item1], embedding.Item2);
+                list.Add((key, value));
+            }
+            datapointProbs.Add((datapoint.name, datapoint.probMethod.method(list)));
+        }
+        return entity.probMethod(datapointProbs);
+    }
+
     public static List<string> GetModels(List<Entity> entities)
     {
         List<string> result = [];
@@ -250,6 +280,53 @@ public class Searchdomain
         return JsonSerializer.Deserialize<SearchdomainSettings>(settingsString);
     }
 
+    public void ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(Entity entity)
+    {
+        if (settings.CacheReconciliation)
+        {
+            foreach (KeyValuePair<string, DateTimedSearchResult> element in searchCache)
+            {
+                string query = element.Key;
+                DateTimedSearchResult searchResult = element.Value;
+
+                Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query);
+                float evaluationResult = EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings);
+
+                searchResult.Results.RemoveAll(x => x.Name == entity.name); // If entity already exists in that results list: remove it.
+
+                ResultItem newItem = new(evaluationResult, entity.name);
+                int index = searchResult.Results.BinarySearch(
+                    newItem,
+                    Comparer<ResultItem>.Create((a, b) => b.Score.CompareTo(a.Score)) // Invert searching order
+                );
+                if (index < 0) // If not found, BinarySearch gives the bitwise complement
+                    index = ~index;
+                searchResult.Results.Insert(index, newItem);
+            }
+        }
+        else
+        {
+            InvalidateSearchCache();
+        }
+    }
+
+    public void ReconciliateOrInvalidateCacheForDeletedEntity(Entity entity)
+    {
+        if (settings.CacheReconciliation)
+        {
+            foreach (KeyValuePair<string, DateTimedSearchResult> element in searchCache)
+            {
+                string query = element.Key;
+                DateTimedSearchResult searchResult = element.Value;
+                searchResult.Results.RemoveAll(x => x.Name == entity.name);
+            }
+        }
+        else
+        {
+            InvalidateSearchCache();
+        }
+    }
+
     public void InvalidateSearchCache()
     {
         searchCache = [];

src/Server/SearchdomainManager.cs

@@ -3,6 +3,7 @@ using System.Data.Common;
 using Server.Migrations;
 using Server.Helper;
 using Server.Exceptions;
+using AdaptiveExpressions;
 
 namespace Server;
 
@@ -16,7 +17,8 @@ public class SearchdomainManager
     private readonly string connectionString;
     private MySqlConnection connection;
     public SQLHelper helper;
-    public Dictionary<string, Dictionary<string, float[]>> embeddingCache;
+    public LRUCache<string, Dictionary<string, float[]>> embeddingCache;
+    public int EmbeddingCacheMaxCount;
 
     public SearchdomainManager(ILogger<SearchdomainManager> logger, IConfiguration config, AIProvider aIProvider, DatabaseHelper databaseHelper)
     {
@@ -24,7 +26,8 @@ public class SearchdomainManager
         _config = config;
         this.aIProvider = aIProvider;
         _databaseHelper = databaseHelper;
-        embeddingCache = [];
+        EmbeddingCacheMaxCount = config.GetValue<int?>("Embeddingsearch:EmbeddingCacheMaxCount") ?? 1000000;
+        embeddingCache = new(EmbeddingCacheMaxCount);
         connectionString = _config.GetSection("Embeddingsearch").GetConnectionString("SQL") ?? "";
         connection = new MySqlConnection(connectionString);
         connection.Open();
@@ -66,7 +69,7 @@ public class SearchdomainManager
     {
         var searchdomain = GetSearchdomain(searchdomainName);
         searchdomain.UpdateEntityCache();
-        searchdomain.InvalidateSearchCache(); // TODO implement cache remediation (Suggestion: searchdomain-wide setting for cache remediation / invalidation - )
+        searchdomain.InvalidateSearchCache();
     }
 
     public List<string> ListSearchdomains()

src/Server/Server.csproj

@@ -7,6 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AdaptiveExpressions" Version="4.23.0" />
     <PackageReference Include="ElmahCore" Version="2.1.2" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />

src/Server/SimilarityMethods.cs

@@ -1,16 +1,18 @@
 using System.Numerics.Tensors;
-using System.Text.Json;
+using Shared.Models;
 
 namespace Server;
 
 public class SimilarityMethod
 {
     public SimilarityMethods.similarityMethodDelegate method;
+    public SimilarityMethodEnum similarityMethodEnum;
     public string name;
 
-    public SimilarityMethod(string name, ILogger logger)
+    public SimilarityMethod(SimilarityMethodEnum similarityMethodEnum, ILogger logger)
     {
-        this.name = name;
+        this.similarityMethodEnum = similarityMethodEnum;
+        this.name = similarityMethodEnum.ToString();
         SimilarityMethods.similarityMethodDelegate? probMethod = SimilarityMethods.GetMethod(name);
         if (probMethod is null)
         {
@@ -21,14 +23,6 @@ public class SimilarityMethod
     }
 }
 
-public enum SimilarityMethodEnum
-{
-    Cosine,
-    Euclidian,
-    Manhattan,
-    Pearson
-}
-
 public static class SimilarityMethods
 {
     public delegate float similarityMethodProtoDelegate(float[] vector1, float[] vector2);

src/Server/Views/Home/Index.cshtml

@@ -1,5 +1,6 @@
 @using Server.Models
 @using System.Web
+@using Shared.Models
 @using Server.Services
 @using Server
 

src/Server/appsettings.Development.json

@@ -24,6 +24,7 @@
         "172.17.0.1"
       ]
     },
+    "EmbeddingCacheMaxCount": 5,
     "AiProviders": {
       "ollama": {
         "handler": "ollama",
@@ -35,6 +36,15 @@
         "ApiKey": "Some API key here"
       }
     },
+    "SimpleAuth": {
+      "Users": [
+        {
+          "Username": "admin",
+          "Password": "UnsafePractice.67",
+          "Roles": ["Admin"]
+        }
+      ]
+    },
     "ApiKeys": ["Some UUID here", "Another UUID here"],
     "UseHttpsRedirection": true
   }

src/Shared/Models/BaseModels.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace Shared.Models;
+
+public class SuccesMessageBaseModel
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+    [JsonPropertyName("Message")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Message { get; set; }
+}

src/Shared/Models/EntityResults.cs

@@ -3,14 +3,10 @@ using System.Text.Json.Serialization;
 namespace Shared.Models;
 
 
-public class EntityQueryResults
+public class EntityQueryResults : SuccesMessageBaseModel
 {
     [JsonPropertyName("Results")]
     public required List<EntityQueryResult> Results { get; set; }
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
 }
 
 public class EntityQueryResult
@@ -19,20 +15,19 @@ public class EntityQueryResult
     public required string Name { get; set; }
     [JsonPropertyName("Value")]
     public float Value { get; set; }
+    [JsonPropertyName("Attributes")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public Dictionary<string, string>? Attributes { get; set; }
 }
 
-public class EntityIndexResult
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-}
+public class EntityIndexResult : SuccesMessageBaseModel {}
 
 public class EntityListResults
 {
     [JsonPropertyName("Results")]
     public required List<EntityListResult> Results { get; set; }
+    [JsonPropertyName("Message")]
+    public string? Message { get; set; }
     [JsonPropertyName("Success")]
     public required bool Success { get; set; }
 }
@@ -77,11 +72,5 @@ public class EmbeddingResult
     public required float[] Embeddings { get; set; }
 }
 
-public class EntityDeleteResults
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-}
+public class EntityDeleteResults : SuccesMessageBaseModel {}
 

src/Shared/Models/JSONModels.cs

@@ -3,7 +3,7 @@ namespace Shared.Models;
 public class JSONEntity
 {
     public required string Name { get; set; }
-    public required string Probmethod { get; set; }
+    public required ProbMethodEnum Probmethod { get; set; }
     public required string Searchdomain { get; set; }
     public required Dictionary<string, string> Attributes { get; set; }
     public required JSONDatapoint[] Datapoints { get; set; }
@@ -13,7 +13,27 @@ public class JSONDatapoint
 {
     public required string Name { get; set; }
     public required string? Text { get; set; }
-    public required string Probmethod_embedding { get; set; }
-    public required string SimilarityMethod { get; set; }
+    public required ProbMethodEnum Probmethod_embedding { get; set; }
+    public required SimilarityMethodEnum SimilarityMethod { get; set; }
     public required string[] Model { get; set; }
+}
+
+public enum ProbMethodEnum
+{
+    Mean,
+    HarmonicMean,
+    QuadraticMean,
+    GeometricMean,
+    EVEWAvg,
+    HVEWAvg,
+    LVEWAvg,
+    DictionaryWeightedAverage
+}
+
+public enum SimilarityMethodEnum
+{
+    Cosine,
+    Euclidian,
+    Manhattan,
+    Pearson
 }

src/Shared/Models/SearchdomainResults.cs

@@ -11,109 +11,46 @@ public class SearchdomainListResults
     public string? Message { get; set; }
 }
 
-public class SearchdomainCreateResults
+public class SearchdomainCreateResults : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("Id")]
     public int? Id { get; set; }
 }
 
-public class SearchdomainUpdateResults
+public class SearchdomainUpdateResults : SuccesMessageBaseModel {}
+
+public class SearchdomainDeleteResults : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-}
-
-public class SearchdomainDeleteResults
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("DeletedEntities")]
     public required int DeletedEntities { get; set; }
 }
 
-public class SearchdomainSearchesResults
+public class SearchdomainSearchesResults : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
     [JsonPropertyName("Searches")]
     public required Dictionary<string, DateTimedSearchResult> Searches { get; set; }
 }
 
-public class SearchdomainDeleteSearchResult
+public class SearchdomainDeleteSearchResult : SuccesMessageBaseModel {}
+
+public class SearchdomainUpdateSearchResult : SuccesMessageBaseModel {}
+
+public class SearchdomainSettingsResults : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }    
-}
-
-public class SearchdomainUpdateSearchResult
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }    
-}
-
-public class SearchdomainSettingsResults
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("Settings")]
     public required SearchdomainSettings? Settings { get; set; }
 }
 
-public class SearchdomainSearchCacheSizeResults
+public class SearchdomainSearchCacheSizeResults : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("SearchCacheSizeBytes")]
     public required long? SearchCacheSizeBytes { get; set; }
 }
 
-public class SearchdomainInvalidateCacheResults
+public class SearchdomainInvalidateCacheResults : SuccesMessageBaseModel {}
+
+public class SearchdomainGetDatabaseSizeResult : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-}
-
-public class SearchdomainGetDatabaseSizeResult
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("SearchdomainDatabaseSizeBytes")]
     public required long? SearchdomainDatabaseSizeBytes { get; set; }    
 }

src/Shared/Models/ServerModels.cs

@@ -2,14 +2,8 @@ using System.Text.Json.Serialization;
 
 namespace Shared.Models;
 
-public class ServerGetModelsResult
+public class ServerGetModelsResult : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("Models")]
     public string[]? Models { get; set; }    
 }