Merge pull request #89 from LD-Reborn/65-add-number-of-cached-queries-to-front-end

65 add number of cached queries to front end

.gitignore

@@ -18,3 +18,5 @@ src/Server/logs
 src/Shared/bin
 src/Shared/obj
 src/Server/wwwroot/logs/*
+src/Server/Tools/CriticalCSS/node_modules
+src/Server/Tools/CriticalCSS/package*.json

src/Client/Client.cs

@@ -8,6 +8,8 @@ using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Configuration;
 using System.Reflection.Metadata.Ecma335;
 using Shared.Models;
+using System.Net;
+using Microsoft.Extensions.Options;
 
 namespace Client;
 
@@ -24,19 +26,65 @@ public class Client
             this.searchdomain = searchdomain;
         }
 
-        public Client(IConfiguration configuration)
+        public Client(IOptions<ServerOptions> configuration)
         {
-            string? baseUri = configuration.GetSection("Embeddingsearch").GetValue<string>("BaseUri");
+            string baseUri = configuration.Value.BaseUri;
-            string? apiKey = configuration.GetSection("Embeddingsearch").GetValue<string>("ApiKey");
+            string? apiKey = configuration.Value.ApiKey;
-            string? searchdomain = configuration.GetSection("Embeddingsearch").GetValue<string>("Searchdomain");
+            string? searchdomain = configuration.Value.Searchdomain;
-            this.baseUri = baseUri ?? "";
+            this.baseUri = baseUri;
             this.apiKey = apiKey ?? "";
             this.searchdomain = searchdomain ?? "";
         }
 
+        public async Task<EntityListResults> EntityListAsync(bool returnEmbeddings = false)
+        {
+            return await EntityListAsync(searchdomain, returnEmbeddings);
+        }
+
+        public async Task<EntityListResults> EntityListAsync(string searchdomain, bool returnEmbeddings = false)
+        {
+            var url = $"{baseUri}/Entities?searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
+            return await FetchUrlAndProcessJson<EntityListResults>(HttpMethod.Get, url);
+        }
+
+        public async Task<EntityIndexResult> EntityIndexAsync(List<JSONEntity> jsonEntity)
+        {
+            return await EntityIndexAsync(JsonSerializer.Serialize(jsonEntity));
+        }
+
+        public async Task<EntityIndexResult> EntityIndexAsync(string jsonEntity)
+        {
+            var content = new StringContent(jsonEntity, Encoding.UTF8, "application/json");
+            return await FetchUrlAndProcessJson<EntityIndexResult>(HttpMethod.Put, GetUrl($"{baseUri}", "Entities", []), content);
+        }
+
+        public async Task<EntityDeleteResults> EntityDeleteAsync(string entityName)
+        {
+            return await EntityDeleteAsync(searchdomain, entityName);
+        }
+
+        public async Task<EntityDeleteResults> EntityDeleteAsync(string searchdomain, string entityName)
+        {
+            var url = $"{baseUri}/Entity?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&entity={HttpUtility.UrlEncode(entityName)}";
+            return await FetchUrlAndProcessJson<EntityDeleteResults>(HttpMethod.Delete, url);
+        }
+
         public async Task<SearchdomainListResults> SearchdomainListAsync()
         {
-            return await GetUrlAndProcessJson<SearchdomainListResults>(GetUrl($"{baseUri}/Searchdomain", "List", apiKey, []));
+            return await FetchUrlAndProcessJson<SearchdomainListResults>(HttpMethod.Get, GetUrl($"{baseUri}", "Searchdomains", []));
+        }
+
+        public async Task<SearchdomainCreateResults> SearchdomainCreateAsync()
+        {
+            return await SearchdomainCreateAsync(searchdomain);
+        }
+
+        public async Task<SearchdomainCreateResults> SearchdomainCreateAsync(string searchdomain, SearchdomainSettings searchdomainSettings = new())
+        {
+            return await FetchUrlAndProcessJson<SearchdomainCreateResults>(HttpMethod.Post, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
+            {
+                {"searchdomain", searchdomain}
+            }), new StringContent(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json"));
         }
 
         public async Task<SearchdomainDeleteResults> SearchdomainDeleteAsync()
@@ -46,20 +94,7 @@ public class Client
 
         public async Task<SearchdomainDeleteResults> SearchdomainDeleteAsync(string searchdomain)
         {
-            return await GetUrlAndProcessJson<SearchdomainDeleteResults>(GetUrl($"{baseUri}/Searchdomain", "Delete", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainDeleteResults>(HttpMethod.Delete, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
-            {
-                {"searchdomain", searchdomain}
-            }));
-        }
-
-        public async Task<SearchdomainCreateResults> SearchdomainCreateAsync()
-        {
-            return await SearchdomainCreateAsync(searchdomain);
-        }
-
-        public async Task<SearchdomainCreateResults> SearchdomainCreateAsync(string searchdomain)
-        {
-            return await GetUrlAndProcessJson<SearchdomainCreateResults>(GetUrl($"{baseUri}/Searchdomain", "Create", apiKey, new Dictionary<string, string>()
             {
                 {"searchdomain", searchdomain}
             }));
@@ -72,87 +107,147 @@ public class Client
             return updateResults;
         }
 
+        public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string searchdomain, string newName, SearchdomainSettings settings = new())
+        {
+             return await SearchdomainUpdateAsync(searchdomain, newName, JsonSerializer.Serialize(settings));
+        }
+
         public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string searchdomain, string newName, string settings = "{}")
         {
-            return await GetUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}/Searchdomain", "Update", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
             {
                 {"searchdomain", searchdomain},
-                {"newName", newName},
+                {"newName", newName}
-                {"settings", settings}
+            }), new StringContent(settings, Encoding.UTF8, "application/json"));
-            }));
         }
 
-        public async Task<EntityQueryResults> EntityQueryAsync(string query)
+        public async Task<SearchdomainQueriesResults> SearchdomainGetQueriesAsync(string searchdomain)
         {
-            return await EntityQueryAsync(searchdomain, query);
+            Dictionary<string, string> parameters = new()
+            {
+                {"searchdomain", searchdomain}
+            };
+            return await FetchUrlAndProcessJson<SearchdomainQueriesResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Queries", parameters));
         }
 
-        public async Task<EntityQueryResults> EntityQueryAsync(string searchdomain, string query)
+        public async Task<EntityQueryResults> SearchdomainQueryAsync(string query)
         {
-            return await GetUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Entity", "Query", apiKey, new Dictionary<string, string>()
+            return await SearchdomainQueryAsync(searchdomain, query);
+        }
+
+        public async Task<EntityQueryResults> SearchdomainQueryAsync(string searchdomain, string query, int? topN = null, bool returnAttributes = false)
+        {
+            Dictionary<string, string> parameters = new()
             {
                 {"searchdomain", searchdomain},
                 {"query", query}
-            }));
+            };
+            if (topN is not null) parameters.Add("topN", ((int)topN).ToString());
+            if (returnAttributes) parameters.Add("returnAttributes", returnAttributes.ToString());
+
+            return await FetchUrlAndProcessJson<EntityQueryResults>(HttpMethod.Post, GetUrl($"{baseUri}/Searchdomain", "Query", parameters), null);
         }
 
-        public async Task<EntityIndexResult> EntityIndexAsync(List<JSONEntity> jsonEntity)
+        public async Task<SearchdomainDeleteSearchResult> SearchdomainDeleteQueryAsync(string searchdomain, string query)
         {
-            return await EntityIndexAsync(JsonSerializer.Serialize(jsonEntity));
+            Dictionary<string, string> parameters = new()
+            {
+                {"searchdomain", searchdomain},
+                {"query", query}
+            };
+            return await FetchUrlAndProcessJson<SearchdomainDeleteSearchResult>(HttpMethod.Delete, GetUrl($"{baseUri}/Searchdomain", "Query", parameters));
         }
 
-        public async Task<EntityIndexResult> EntityIndexAsync(string jsonEntity)
+        public async Task<SearchdomainUpdateSearchResult> SearchdomainUpdateQueryAsync(string searchdomain, string query, List<ResultItem> results)
         {
-            var content = new StringContent(jsonEntity, Encoding.UTF8, "application/json");
+            Dictionary<string, string> parameters = new()
-            return await PostUrlAndProcessJson<EntityIndexResult>(GetUrl($"{baseUri}/Entity", "Index", apiKey, []), content);//new FormUrlEncodedContent(values));
+            {
+                {"searchdomain", searchdomain},
+                {"query", query}
+            };
+            return await FetchUrlAndProcessJson<SearchdomainUpdateSearchResult>(
+                HttpMethod.Patch,
+                GetUrl($"{baseUri}/Searchdomain", "Query", parameters),
+                new StringContent(JsonSerializer.Serialize(results), Encoding.UTF8, "application/json"));
         }
 
-        public async Task<EntityListResults> EntityListAsync(bool returnEmbeddings = false)
+        public async Task<SearchdomainSettingsResults> SearchdomainGetSettingsAsync(string searchdomain)
         {
-            return await EntityListAsync(searchdomain, returnEmbeddings);
+            Dictionary<string, string> parameters = new()
+            {
+                {"searchdomain", searchdomain}
+            };
+            return await FetchUrlAndProcessJson<SearchdomainSettingsResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters));
         }
 
-        public async Task<EntityListResults> EntityListAsync(string searchdomain, bool returnEmbeddings = false)
+        public async Task<SearchdomainUpdateResults> SearchdomainUpdateSettingsAsync(string searchdomain, SearchdomainSettings searchdomainSettings)
         {
-            var url = $"{baseUri}/Entity/List?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
+            Dictionary<string, string> parameters = new()
-            return await GetUrlAndProcessJson<EntityListResults>(url);
+            {
+                {"searchdomain", searchdomain}
+            };
+            StringContent content = new(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json");
+            return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters), content);
         }
 
-        public async Task<EntityDeleteResults> EntityDeleteAsync(string entityName)
+        public async Task<SearchdomainQueryCacheSizeResults> SearchdomainGetQueryCacheSizeAsync(string searchdomain)
         {
-            return await EntityDeleteAsync(searchdomain, entityName);
+            Dictionary<string, string> parameters = new()
+            {
+                {"searchdomain", searchdomain}
+            };
+            return await FetchUrlAndProcessJson<SearchdomainQueryCacheSizeResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", parameters));
         }
 
-        public async Task<EntityDeleteResults> EntityDeleteAsync(string searchdomain, string entityName)
+        public async Task<SearchdomainInvalidateCacheResults> SearchdomainClearQueryCache(string searchdomain)
         {
-            var url = $"{baseUri}/Entity/Delete?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&entity={HttpUtility.UrlEncode(entityName)}";
+            Dictionary<string, string> parameters = new()
-            return await GetUrlAndProcessJson<EntityDeleteResults>(url);
+            {
+                {"searchdomain", searchdomain}
+            };
+            return await FetchUrlAndProcessJson<SearchdomainInvalidateCacheResults>(HttpMethod.Post, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Clear", parameters), null);
         }
 
-        private static async Task<T> GetUrlAndProcessJson<T>(string url)
+        public async Task<SearchdomainGetDatabaseSizeResult> SearchdomainGetDatabaseSizeAsync(string searchdomain)
         {
+            Dictionary<string, string> parameters = new()
+            {
+                {"searchdomain", searchdomain}
+            };
+            return await FetchUrlAndProcessJson<SearchdomainGetDatabaseSizeResult>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/Database", "Size", parameters));
+        }
+
+        public async Task<ServerGetModelsResult> ServerGetModelsAsync()
+        {
+            return await FetchUrlAndProcessJson<ServerGetModelsResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server", "Models", []));
+        }
+
+        public async Task<ServerGetStatsResult> ServerGetStatsAsync()
+        {
+            return await FetchUrlAndProcessJson<ServerGetStatsResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server/Stats", "Size", []));
+        }
+
+        private async Task<T> FetchUrlAndProcessJson<T>(HttpMethod httpMethod, string url, HttpContent? content = null)
+        {
+            HttpRequestMessage requestMessage = new(httpMethod, url)
+            {
+                Content = content,
+            };
+            requestMessage.Headers.Add("X-API-KEY", apiKey);
             using var client = new HttpClient();
-            var response = await client.GetAsync(url);
+            var response = await client.SendAsync(requestMessage);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-        private static async Task<T> PostUrlAndProcessJson<T>(string url, HttpContent content)
-        {
-            using var client = new HttpClient();
-            var response = await client.PostAsync(url, content);
             string responseContent = await response.Content.ReadAsStringAsync();
+            if (response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Unauthorized) throw new UnauthorizedAccessException(responseContent); // TODO implement distinct exceptions
+            if (response.StatusCode == HttpStatusCode.InternalServerError) throw new Exception($"Request was unsuccessful due to an internal server error: {responseContent}"); // TODO implement proper InternalServerErrorException
             var result = JsonSerializer.Deserialize<T>(responseContent)
                 ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
             return result;
         }
 
-        public static string GetUrl(string baseUri, string endpoint, string apiKey, Dictionary<string, string> parameters)
+        public static string GetUrl(string baseUri, string endpoint, Dictionary<string, string> parameters)
         {
             var uriBuilder = new UriBuilder($"{baseUri}/{endpoint}");
             var query = HttpUtility.ParseQueryString(uriBuilder.Query);
-            if (apiKey.Length > 0) query["apiKey"] = apiKey;
             foreach (var param in parameters)
             {
                 query[param.Key] = param.Value;

src/Indexer/Indexer.csproj

@@ -15,6 +15,7 @@
     <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
     <PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
+    <PackageReference Include="System.Configuration.ConfigurationManager" Version="9.0.3" />
     <PackageReference Include="Python" Version="3.13.3" />
     <PackageReference Include="Pythonnet" Version="3.0.5" />
   </ItemGroup>

src/Indexer/Models/OptionModels.cs

@@ -0,0 +1,9 @@
+using Shared.Models;
+namespace Indexer.Models;
+
+public class IndexerOptions : ApiKeyOptions
+{
+    public required WorkerConfig[] Workers { get; set; }
+    public required ServerOptions Server { get; set;}
+    public required string PythonRuntime { get; set; } = "libpython3.13.so";
+}

src/Indexer/Models/ScriptModels.cs

@@ -15,11 +15,11 @@ public class ScriptToolSet
     public Client.Client Client;
     public LoggerWrapper Logger;
     public ICallbackInfos? CallbackInfos;
-    public IConfiguration Configuration;
+    public IndexerOptions Configuration;
     public CancellationToken CancellationToken;
     public string Name;
 
-    public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IConfiguration configuration, CancellationToken cancellationToken, string name)
+    public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IndexerOptions configuration, CancellationToken cancellationToken, string name)
     {
         Configuration = configuration;
         Name = name;

src/Indexer/Program.cs

@@ -6,6 +6,8 @@ using ElmahCore.Mvc;
 using ElmahCore.Mvc.Logger;
 using Serilog;
 using Quartz;
+using System.Configuration;
+using Shared.Models;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -21,6 +23,12 @@ Log.Logger = new LoggerConfiguration()
 builder.Logging.AddSerilog();
 builder.Services.AddHttpContextAccessor();
 builder.Services.AddSingleton<IConfigurationRoot>(builder.Configuration);
+
+IConfigurationSection configurationSection = builder.Configuration.GetSection("Indexer");
+IndexerOptions configuration = configurationSection.Get<IndexerOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
+builder.Services.Configure<IndexerOptions>(configurationSection);
+builder.Services.Configure<ServerOptions>(configurationSection.GetSection("Server"));
+builder.Services.Configure<ApiKeyOptions>(configurationSection);
 builder.Services.AddSingleton<Client.Client>();
 builder.Services.AddSingleton<WorkerManager>();
 builder.Services.AddHostedService<IndexerService>();

src/Indexer/ScriptContainers/PythonScriptContainer.cs

@@ -15,11 +15,8 @@ public class PythonScriptable : IScriptContainer
     public ILogger _logger { get; set; }
     public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
     {
-        string? runtime = toolSet.Configuration.GetValue<string>("EmbeddingsearchIndexer:PythonRuntime");
+        string runtime = toolSet.Configuration.PythonRuntime;
-        if (runtime is not null)
+        Runtime.PythonDLL ??= runtime;
-        {
-            Runtime.PythonDLL ??= runtime;
-        }
         _logger = logger;
         SourceLoaded = false;
         if (!PythonEngine.IsInitialized)

src/Indexer/WorkerManager.cs

@@ -1,21 +1,22 @@
 using Indexer.Exceptions;
 using Indexer.Models;
 using Indexer.ScriptContainers;
+using Microsoft.Extensions.Options;
 
 public class WorkerManager
 {
     public Dictionary<string, Worker> Workers;
     public List<Type> types;
     private readonly ILogger<WorkerManager> _logger;
-    private readonly IConfiguration _configuration;
+    private readonly IndexerOptions _configuration;
     private readonly Client.Client client;
 
-    public WorkerManager(ILogger<WorkerManager> logger, IConfiguration configuration, Client.Client client)
+    public WorkerManager(ILogger<WorkerManager> logger, IOptions<IndexerOptions> configuration, Client.Client client)
     {
         Workers = [];
         types = [typeof(PythonScriptable), typeof(CSharpScriptable)];
         _logger = logger;
-        _configuration = configuration;
+        _configuration = configuration.Value;
         this.client = client;
     }
 
@@ -23,27 +24,12 @@ public class WorkerManager
     {
         _logger.LogInformation("Initializing workers");
         // Load and configure all workers
-        var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer");
-        if (!sectionMain.Exists())
-        {
-            _logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
-            throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
-        }
 
-        WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
+        foreach (WorkerConfig workerConfig in _configuration.Workers)
-        if (sectionWorker is not null)
         {
-            foreach (WorkerConfig workerConfig in sectionWorker.Worker)
+            CancellationTokenSource cancellationTokenSource = new();
-            {
+            ScriptToolSet toolSet = new(workerConfig.Script, client, _logger, _configuration, cancellationTokenSource.Token, workerConfig.Name);
-                CancellationTokenSource cancellationTokenSource = new();
+            InitializeWorker(toolSet, workerConfig, cancellationTokenSource);
-                ScriptToolSet toolSet = new(workerConfig.Script, client, _logger, _configuration, cancellationTokenSource.Token, workerConfig.Name);
-                InitializeWorker(toolSet, workerConfig, cancellationTokenSource);
-            }
-        }
-        else
-        {
-            _logger.LogCritical("Unable to load section \"Worker\"");
-            throw new IndexerConfigurationException("Unable to load section \"Worker\"");
         }
         _logger.LogInformation("Initialized workers");
     }

src/Indexer/appsettings.Development.json

@@ -5,46 +5,23 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
-  "Embeddingsearch": {
+  "Indexer": {
-    "BaseUri": "http://localhost:5146"
+    "Workers": [
-  },
-  "EmbeddingsearchIndexer": {
-    "Worker":
-    [
       {
         "Name": "pythonExample",
         "Script": "Scripts/example.py",
-        "Calls": [
-          {
-            "Name": "intervalCall",
-            "Type": "interval",
-            "Interval": 30000
-          }
-        ]
-      },
-      {
-        "Name": "csharpExample",
-        "Script": "Scripts/example.csx",
         "Calls": [
           {
             "Name": "runonceCall",
             "Type": "runonce"
-          },
-          {
-            "Name": "scheduleCall",
-            "Type": "schedule",
-            "Schedule": "0 0/5 * * * ?"
-          },
-          {
-            "Name": "fileupdateCall",
-            "Type": "fileupdate",
-            "Path": "./Scripts/example_content",
-            "Events": ["Created", "Changed", "Deleted", "Renamed"],
-            "Filters": ["*.md", "*.txt"],
-            "IncludeSubdirectories": true
           }
         ]
       }
-    ]
+    ],
+    "ApiKeys": ["xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"],
+    "Server": {
+      "BaseUri": "http://localhost:5146",
+      "ApiKey": "yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy"
+    }
   }
 }

src/Indexer/appsettings.json

@@ -24,7 +24,7 @@
       ],
       "LogFolder": "./logs"
     },
-    "PythonRuntime": "libpython3.12.so"
+    "PythonRuntime": "libpython3.13.so"
   },
   "AllowedHosts": "*"
 }

src/Server/AIProvider.cs

@@ -1,24 +1,25 @@
 
 using System.Text;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Server.Exceptions;
+using Server.Models;
 
 namespace Server;
 
 public class AIProvider
 {
     private readonly ILogger<AIProvider> _logger;
-    private readonly IConfiguration _configuration;
+    private readonly EmbeddingSearchOptions _configuration;
-    public AIProvidersConfiguration aIProvidersConfiguration;
+    public Dictionary<string, AiProvider> aIProvidersConfiguration;
 
-    public AIProvider(ILogger<AIProvider> logger, IConfiguration configuration)
+    public AIProvider(ILogger<AIProvider> logger, IOptions<EmbeddingSearchOptions> configuration)
     {
         _logger = logger;
-        _configuration = configuration;
+        _configuration = configuration.Value;
-        AIProvidersConfiguration? retrievedAiProvidersConfiguration = _configuration
+        Dictionary<string, AiProvider>? retrievedAiProvidersConfiguration = _configuration.AiProviders;
-            .GetSection("Embeddingsearch")
-            .Get<AIProvidersConfiguration>();
         if (retrievedAiProvidersConfiguration is null)
         {
             _logger.LogCritical("Unable to build AIProvidersConfiguration. Please check your configuration.");
@@ -35,8 +36,8 @@ public class AIProvider
         Uri uri = new(modelUri);
         string provider = uri.Scheme;
         string model = uri.AbsolutePath;
-        AIProviderConfiguration? aIProvider = aIProvidersConfiguration.AiProviders
+        AiProvider? aIProvider = aIProvidersConfiguration
-            .FirstOrDefault(x => String.Equals(x.Key.ToLower(), provider.ToLower()))
+            .FirstOrDefault(x => string.Equals(x.Key.ToLower(), provider.ToLower()))
             .Value;
         if (aIProvider is null)
         {
@@ -119,12 +120,12 @@ public class AIProvider
 
     public string[] GetModels()
     {
-        var aIProviders = aIProvidersConfiguration.AiProviders;
+        var aIProviders = aIProvidersConfiguration;
         List<string> results = [];
-        foreach (KeyValuePair<string, AIProviderConfiguration> aIProviderKV in aIProviders)
+        foreach (KeyValuePair<string, AiProvider> aIProviderKV in aIProviders)
         {
             string aIProviderName = aIProviderKV.Key;
-            AIProviderConfiguration aIProvider = aIProviderKV.Value;
+            AiProvider aIProvider = aIProviderKV.Value;
 
             using var httpClient = new HttpClient();
 
@@ -178,7 +179,12 @@ public class AIProvider
                 foreach (string? result in aIProviderResult)
                 {
                     if (result is null) continue;
-                    results.Add(aIProviderName + ":" + result);
+                    bool isInAllowList = ElementMatchesAnyRegexInList(result, aIProvider.Allowlist);
+                    bool isInDenyList =  ElementMatchesAnyRegexInList(result, aIProvider.Denylist);
+                    if (isInAllowList && !isInDenyList)
+                    {
+                        results.Add(aIProviderName + ":" + result);
+                    }
                 }
             }
             catch (Exception ex)
@@ -189,6 +195,11 @@ public class AIProvider
         }
         return [.. results];
     }
+
+    private static bool ElementMatchesAnyRegexInList(string element, string[] list)
+    {
+        return list?.Any(pattern => pattern != null && Regex.IsMatch(element, pattern)) ?? false;
+    }
 }
 
 public class AIProvidersConfiguration

src/Server/Controllers/AccountController.cs

@@ -6,14 +6,15 @@ using Server.Models;
 
 namespace Server.Controllers;
 
+[ApiExplorerSettings(IgnoreApi = true)]
 [Route("[Controller]")]
 public class AccountController : Controller
 {
     private readonly SimpleAuthOptions _options;
 
-    public AccountController(IOptions<SimpleAuthOptions> options)
+    public AccountController(IOptions<EmbeddingSearchOptions> options)
     {
-        _options = options.Value;
+        _options = options.Value.SimpleAuth;
     }
 
     [HttpGet("Login")]

src/Server/Controllers/EntityController.cs

@@ -24,91 +24,27 @@ public class EntityController : ControllerBase
         _databaseHelper = databaseHelper;
     }
 
-    [HttpGet("Query")]
+    /// <summary>
-    public ActionResult<EntityQueryResults> Query(string searchdomain, string query, int? topN)
+    /// List the entities in a searchdomain
-    {
+    /// </summary>
-        Searchdomain searchdomain_;
+    /// <remarks>
-        try
+    /// With returnModels = false expect: "Datapoints": [..., "Embeddings": null]<br/>
-        {
+    /// With returnModels = true expect: "Datapoints": [..., "Embeddings": [{"Model": "...", "Embeddings": []}, ...]]<br/>
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
+    /// With returnEmbeddings = true expect: "Datapoints": [..., "Embeddings": [{"Model": "...", "Embeddings": [0.007384672,0.01309805,0.0012528514,...]}, ...]]
-        } catch (SearchdomainNotFoundException)
+    /// </remarks>
-        {
+    /// <param name="searchdomain">Name of the searchdomain</param>
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
+    /// <param name="returnModels">Include the models in the response</param>
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
+    /// <param name="returnEmbeddings">Include the embeddings in the response (requires returnModels)</param>
-        } catch (Exception ex)
+    [HttpGet("/Entities")]
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
-        }
-        List<(float, string)> results = searchdomain_.Search(query, topN);
-        List<EntityQueryResult> queryResults = [.. results.Select(r => new EntityQueryResult
-        {
-            Name = r.Item2,
-            Value = r.Item1
-        })];
-        return Ok(new EntityQueryResults(){Results = queryResults, Success = true });
-    }
-
-    [HttpPost("Index")]
-    public ActionResult<EntityIndexResult> Index([FromBody] List<JSONEntity>? jsonEntities)
-    {
-        try
-        {
-            List<Entity>? entities = _searchdomainHelper.EntitiesFromJSON(
-                _domainManager,
-                _logger,
-                JsonSerializer.Serialize(jsonEntities));
-            if (entities is not null && jsonEntities is not null)
-            {
-                List<string> invalidatedSearchdomains = [];
-                foreach (var jsonEntity in jsonEntities)
-                {
-                    string jsonEntityName = jsonEntity.Name;
-                    string jsonEntitySearchdomainName = jsonEntity.Searchdomain;
-                    if (entities.Select(x => x.name == jsonEntityName).Any()
-                        && !invalidatedSearchdomains.Contains(jsonEntitySearchdomainName))
-                    {
-                        invalidatedSearchdomains.Add(jsonEntitySearchdomainName);
-                        _domainManager.InvalidateSearchdomainCache(jsonEntitySearchdomainName);
-                    }
-                }
-                return Ok(new EntityIndexResult() { Success = true });
-            }
-            else
-            {
-                _logger.LogError("Unable to deserialize an entity");
-                return Ok(new EntityIndexResult() { Success = false, Message = "Unable to deserialize an entity"});
-            }
-        } catch (Exception ex)
-        {
-            if (ex.InnerException is not null) ex = ex.InnerException;
-            _logger.LogError("Unable to index the provided entities. {ex.Message} - {ex.StackTrace}", [ex.Message, ex.StackTrace]);
-            return Ok(new EntityIndexResult() { Success = false, Message = ex.Message });
-        }
-
-    }
-
-    [HttpGet("List")]
     public ActionResult<EntityListResults> List(string searchdomain, bool returnModels = false, bool returnEmbeddings = false)
     {
         if (returnEmbeddings && !returnModels)
         {
             _logger.LogError("Invalid request for {searchdomain} - embeddings return requested but without models - not possible!", [searchdomain]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Invalid request" });            
+            return BadRequest(new EntityListResults() {Results = [], Success = false, Message = "Invalid request" });            
-        }
-        Searchdomain searchdomain_;
-        try
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        } catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
-        } catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
         }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
         EntityListResults entityListResults = new() {Results = [], Success = true};
         foreach (Entity entity in searchdomain_.entityCache)
         {
@@ -146,29 +82,69 @@ public class EntityController : ControllerBase
         return Ok(entityListResults);
     }
 
-    [HttpGet("Delete")]
+    /// <summary>
-    public ActionResult<EntityDeleteResults> Delete(string searchdomain, string entityName)
+    /// Index entities
+    /// </summary>
+    /// <remarks>
+    /// Behavior: Creates new entities, but overwrites existing entities that have the same name
+    /// </remarks>
+    /// <param name="jsonEntities">Entities to index</param>
+    [HttpPut("/Entities")]
+    public ActionResult<EntityIndexResult> Index([FromBody] List<JSONEntity>? jsonEntities)
     {
-        Searchdomain searchdomain_;
         try
         {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
+            List<Entity>? entities = _searchdomainHelper.EntitiesFromJSON(
-        } catch (SearchdomainNotFoundException)
+                _domainManager,
-        {
+                _logger,
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
+                JsonSerializer.Serialize(jsonEntities));
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Searchdomain not found" });
+            if (entities is not null && jsonEntities is not null)
+            {
+                List<string> invalidatedSearchdomains = [];
+                foreach (var jsonEntity in jsonEntities)
+                {
+                    string jsonEntityName = jsonEntity.Name;
+                    string jsonEntitySearchdomainName = jsonEntity.Searchdomain;
+                    if (entities.Select(x => x.name == jsonEntityName).Any()
+                        && !invalidatedSearchdomains.Contains(jsonEntitySearchdomainName))
+                    {
+                        invalidatedSearchdomains.Add(jsonEntitySearchdomainName);
+                    }
+                }
+                return Ok(new EntityIndexResult() { Success = true });
+            }
+            else
+            {
+                _logger.LogError("Unable to deserialize an entity");
+                return Ok(new EntityIndexResult() { Success = false, Message = "Unable to deserialize an entity"});
+            }
         } catch (Exception ex)
         {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
+            if (ex.InnerException is not null) ex = ex.InnerException;
-            return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." });
+            _logger.LogError("Unable to index the provided entities. {ex.Message} - {ex.StackTrace}", [ex.Message, ex.StackTrace]);
+            return Ok(new EntityIndexResult() { Success = false, Message = ex.Message });
         }
 
+    }
+
+    /// <summary>
+    /// Deletes an entity
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    /// <param name="entityName">Name of the entity</param>
+    [HttpDelete]
+    public ActionResult<EntityDeleteResults> Delete(string searchdomain, string entityName)
+    {
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
+        
         Entity? entity_ = SearchdomainHelper.CacheGetEntity(searchdomain_.entityCache, entityName);
         if (entity_ is null)
         {
             _logger.LogError("Unable to delete the entity {entityName} in {searchdomain} - it was not found under the specified name", [entityName, searchdomain]);
             return Ok(new EntityDeleteResults() {Success = false, Message = "Entity not found"});
         }
+        searchdomain_.ReconciliateOrInvalidateCacheForDeletedEntity(entity_);
         _databaseHelper.RemoveEntity([], _domainManager.helper, entityName, searchdomain);
         searchdomain_.entityCache.RemoveAll(entity => entity.name == entityName);
         return Ok(new EntityDeleteResults() {Success = true});

src/Server/Controllers/Frontend/HomeController.cs

@@ -7,8 +7,8 @@ using Server.Exceptions;
 using Server.Models;
 namespace Server.Controllers;
 
-[ApiController]
+[ApiExplorerSettings(IgnoreApi = true)]
-[Route("/")]
+[Route("[Controller]")]
 public class HomeController : Controller
 {
     private readonly ILogger<EntityController> _logger;
@@ -20,9 +20,22 @@ public class HomeController : Controller
         _domainManager = domainManager;
     }
 
-    [Authorize]
     [HttpGet("/")]
+    public IActionResult Root()
+    {
+        return Redirect("/Home/Index");
+    }
+
+    [Authorize]
+    [HttpGet("Index")]
     public IActionResult Index()
+    {
+        return View();
+    }
+
+    [Authorize]
+    [HttpGet("Searchdomains")]
+    public IActionResult Searchdomains()
     {
         HomeIndexViewModel viewModel = new()
         {

src/Server/Controllers/SearchdomainController.cs

@@ -1,9 +1,11 @@
+using System.ComponentModel.DataAnnotations;
 using System.Text.Json;
 using ElmahCore;
 using Microsoft.AspNetCore.Http.HttpResults;
 using Microsoft.AspNetCore.Mvc;
 using Server.Exceptions;
 using Server.Helper;
+using Shared;
 using Shared.Models;
 
 namespace Server.Controllers;
@@ -23,7 +25,10 @@ public class SearchdomainController : ControllerBase
         _domainManager = domainManager;
     }
 
-    [HttpGet("List")]
+    /// <summary>
+    /// Lists all searchdomains
+    /// </summary>
+    [HttpGet("/Searchdomains")]
     public ActionResult<SearchdomainListResults> List()
     {
         List<string> results;
@@ -40,11 +45,20 @@ public class SearchdomainController : ControllerBase
         return Ok(searchdomainListResults);
     }
 
-    [HttpGet("Create")]
+    /// <summary>
-    public ActionResult<SearchdomainCreateResults> Create(string searchdomain, string settings = "{}")
+    /// Creates a new searchdomain
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    /// <param name="settings">Optional initial settings</param>
+    [HttpPost]
+    public ActionResult<SearchdomainCreateResults> Create([Required]string searchdomain, [FromBody]SearchdomainSettings settings = new())
     {
         try
         {
+            if (settings.QueryCacheSize <= 0)
+            {
+                settings.QueryCacheSize = 1_000_000; // TODO get rid of this magic number
+            }
             int id = _domainManager.CreateSearchdomain(searchdomain, settings);
             return Ok(new SearchdomainCreateResults(){Id = id, Success = true});
         } catch (Exception)
@@ -54,8 +68,12 @@ public class SearchdomainController : ControllerBase
         }
     }
 
-    [HttpGet("Delete")]
+    /// <summary>
-    public ActionResult<SearchdomainDeleteResults> Delete(string searchdomain)
+    /// Deletes a searchdomain
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    [HttpDelete]
+    public ActionResult<SearchdomainDeleteResults> Delete([Required]string searchdomain)
     {
         bool success;
         int deletedEntries;
@@ -84,12 +102,27 @@ public class SearchdomainController : ControllerBase
         return Ok(new SearchdomainDeleteResults(){Success = success, DeletedEntities = deletedEntries, Message = message});
     }
 
-    [HttpGet("Update")]
+    /// <summary>
-    public ActionResult<SearchdomainUpdateResults> Update(string searchdomain, string newName, string settings = "{}")
+    /// Updates name and settings of a searchdomain
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    /// <param name="newName">Updated name of the searchdomain</param>
+    /// <param name="settings">Updated settings of searchdomain</param>
+    [HttpPut]
+    public ActionResult<SearchdomainUpdateResults> Update([Required]string searchdomain, string newName, [FromBody]SearchdomainSettings? settings)
     {
-        try
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
+        if (settings is null)
+        {
+            Dictionary<string, dynamic> parameters = new()
+            {
+                {"name", newName},
+                {"id", searchdomain_.id}
+            };
+            searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name WHERE id = @id", parameters);
+        } else
         {
-            Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
             Dictionary<string, dynamic> parameters = new()
             {
                 {"name", newName},
@@ -97,85 +130,57 @@ public class SearchdomainController : ControllerBase
                 {"id", searchdomain_.id}
             };
             searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set name = @name, settings = @settings WHERE id = @id", parameters);            
-        } catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to update searchdomain {searchdomain} - not found", [searchdomain]);
-            return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update searchdomain {searchdomain} - not found" });
-        } catch (Exception ex)
-        {
-            _logger.LogError("Unable to update searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update searchdomain {searchdomain}" });
         }
         return Ok(new SearchdomainUpdateResults(){Success = true});
     }
 
-    [HttpPost("UpdateSettings")]
+    /// <summary>
-    public ActionResult<SearchdomainUpdateResults> UpdateSettings(string searchdomain, [FromBody] SearchdomainSettings request)
+    /// Gets the query cache of a searchdomain
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    [HttpGet("Queries")]
+    public ActionResult<SearchdomainQueriesResults> GetQueries([Required]string searchdomain)
     {
-        try
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
-        {
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
-            Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
+        Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.queryCache.AsDictionary();
-            Dictionary<string, dynamic> parameters = new()
+        
-            {
+        return Ok(new SearchdomainQueriesResults() { Searches = searchCache, Success = true });
-                {"settings", JsonSerializer.Serialize(request)},
-                {"id", searchdomain_.id}
-            };
-            searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters);
-            searchdomain_.settings = request;
-        } catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to update settings for searchdomain {searchdomain} - not found", [searchdomain]);
-            return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update settings for searchdomain {searchdomain} - not found" });
-        } catch (Exception ex)
-        {
-            _logger.LogError("Unable to update settings for searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainUpdateResults() { Success = false, Message = $"Unable to update settings for searchdomain {searchdomain}" });
-        }
-        return Ok(new SearchdomainUpdateResults(){Success = true});
     }
 
-    [HttpGet("GetSearches")]
+    /// <summary>
-    public ActionResult<SearchdomainSearchesResults> GetSearches(string searchdomain)
+    /// Executes a query in the searchdomain
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    /// <param name="query">Query to execute</param>
+    /// <param name="topN">Return only the top N results</param>
+    /// <param name="returnAttributes">Return the attributes of the object</param>
+    [HttpPost("Query")]
+    public ActionResult<EntityQueryResults> Query([Required]string searchdomain, [Required]string query, int? topN, bool returnAttributes = false)
     {
-        Searchdomain searchdomain_;
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
-        try
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
+        List<(float, string)> results = searchdomain_.Search(query, topN);
+        List<EntityQueryResult> queryResults = [.. results.Select(r => new EntityQueryResult
         {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
+            Name = r.Item2,
-        }
+            Value = r.Item1,
-        catch (SearchdomainNotFoundException)
+            Attributes = returnAttributes ? (searchdomain_.entityCache.FirstOrDefault(x => x.name == r.Item2)?.attributes ?? null) : null
-        {
+        })];
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
+        return Ok(new EntityQueryResults(){Results = queryResults, Success = true });
-            return Ok(new SearchdomainSearchesResults() { Searches = [], Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainSearchesResults() { Searches = [], Success = false, Message = ex.Message });
-        }
-        Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
-        
-        return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true });
     }
 
-    [HttpDelete("Searches")]
+    /// <summary>
-    public ActionResult<SearchdomainDeleteSearchResult> DeleteSearch(string searchdomain, string query)
+    /// Deletes a query from the query cache
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    /// <param name="query">Query to delete</param>
+    [HttpDelete("Query")]
+    public ActionResult<SearchdomainDeleteSearchResult> DeleteQuery([Required]string searchdomain, [Required]string query)
     {
-        Searchdomain searchdomain_;
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
-        try
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
-        {
+        EnumerableLruCache<string, DateTimedSearchResult> searchCache = searchdomain_.queryCache;
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        }
-        catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = ex.Message });
-        }
-        Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
         bool containsKey = searchCache.ContainsKey(query);
         if (containsKey)
         {
@@ -185,25 +190,18 @@ public class SearchdomainController : ControllerBase
         return Ok(new SearchdomainDeleteSearchResult() {Success = false, Message = "Query not found in search cache"});
     }
 
-    [HttpPatch("Searches")]
+    /// <summary>
-    public ActionResult<SearchdomainUpdateSearchResult> UpdateSearch(string searchdomain, string query, [FromBody]List<ResultItem> results)
+    /// Updates a query from the query cache
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    /// <param name="query">Query to update</param>
+    /// <param name="results">List of results to apply to the query</param>
+    [HttpPatch("Query")]
+    public ActionResult<SearchdomainUpdateSearchResult> UpdateQuery([Required]string searchdomain, [Required]string query, [Required][FromBody]List<ResultItem> results)
     {
-        Searchdomain searchdomain_;
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
-        try
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
-        {
+        EnumerableLruCache<string, DateTimedSearchResult> searchCache = searchdomain_.queryCache;
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        }
-        catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainUpdateSearchResult() { Success = false, Message = ex.Message });
-        }
-        Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
         bool containsKey = searchCache.ContainsKey(query);
         if (containsKey)
         {
@@ -215,95 +213,80 @@ public class SearchdomainController : ControllerBase
         return Ok(new SearchdomainUpdateSearchResult() {Success = false, Message = "Query not found in search cache"});
     }
 
-    [HttpGet("GetSettings")]
+    /// <summary>
-    public ActionResult<SearchdomainSettingsResults> GetSettings(string searchdomain)
+    /// Get the settings of a searchdomain
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    [HttpGet("Settings")]
+    public ActionResult<SearchdomainSettingsResults> GetSettings([Required]string searchdomain)
     {
-        Searchdomain searchdomain_;
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
-        try
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
-        {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
-        }
-        catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new SearchdomainSettingsResults() { Settings = null, Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainSettingsResults() { Settings = null, Success = false, Message = ex.Message });
-        }
         SearchdomainSettings settings = searchdomain_.settings;
         return Ok(new SearchdomainSettingsResults() { Settings = settings, Success = true });
     }
 
-    [HttpGet("GetSearchCacheSize")]
+    /// <summary>
-    public ActionResult<SearchdomainSearchCacheSizeResults> GetSearchCacheSize(string searchdomain)
+    /// Update the settings of a searchdomain
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    [HttpPut("Settings")]
+    public ActionResult<SearchdomainUpdateResults> UpdateSettings([Required]string searchdomain, [Required][FromBody] SearchdomainSettings request)
     {
-        Searchdomain searchdomain_;
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
-        try
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
+        Dictionary<string, dynamic> parameters = new()
         {
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
+            {"settings", JsonSerializer.Serialize(request)},
-        }
+            {"id", searchdomain_.id}
-        catch (SearchdomainNotFoundException)
+        };
-        {
+        searchdomain_.helper.ExecuteSQLNonQuery("UPDATE searchdomain set settings = @settings WHERE id = @id", parameters);
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
+        searchdomain_.settings = request;
-            return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = null, Success = false, Message = "Searchdomain not found" });
+        searchdomain_.queryCache.Capacity = request.QueryCacheSize;
-        }
+        return Ok(new SearchdomainUpdateResults(){Success = true});
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = null, Success = false, Message = ex.Message });
-        }
-        Dictionary<string, DateTimedSearchResult> searchCache = searchdomain_.searchCache;
-        long sizeInBytes = 0;
-        foreach (var entry in searchCache)
-        {
-            sizeInBytes += sizeof(int); // string length prefix
-            sizeInBytes += entry.Key.Length * sizeof(char); // string characters
-            sizeInBytes += entry.Value.EstimateSize();
-        }
-        return Ok(new SearchdomainSearchCacheSizeResults() { SearchCacheSizeBytes = sizeInBytes, Success = true });
     }
 
-    [HttpGet("ClearSearchCache")]
+    /// <summary>
-    public ActionResult<SearchdomainInvalidateCacheResults> InvalidateSearchCache(string searchdomain)
+    /// Get the query cache size of a searchdomain
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    [HttpGet("QueryCache/Size")]
+    public ActionResult<SearchdomainQueryCacheSizeResults> GetQueryCacheSize([Required]string searchdomain)
     {
-        try
+        if (!SearchdomainHelper.IsSearchdomainLoaded(_domainManager, searchdomain))
         {
-            Searchdomain searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
+            return Ok(new SearchdomainQueryCacheSizeResults() { SizeBytes = 0, ElementCount = 0, ElementMaxCount = 0, Success = true });
-            searchdomain_.InvalidateSearchCache();
-        } catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to invalidate search cache for searchdomain {searchdomain} - not found", [searchdomain]);
-            return Ok(new SearchdomainInvalidateCacheResults() { Success = false, Message = $"Unable to invalidate search cache for searchdomain {searchdomain} - not found" });
-        } catch (Exception ex)
-        {
-            _logger.LogError("Unable to invalidate search cache for searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainInvalidateCacheResults() { Success = false, Message = $"Unable to invalidate search cache for searchdomain {searchdomain}" });
         }
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
+        int elementCount = searchdomain_.queryCache.Count;
+        int ElementMaxCount = searchdomain_.settings.QueryCacheSize;
+        return Ok(new SearchdomainQueryCacheSizeResults() { SizeBytes = searchdomain_.GetSearchCacheSize(), ElementCount = elementCount, ElementMaxCount = ElementMaxCount, Success = true });
+    }
+
+    /// <summary>
+    /// Clear the query cache of a searchdomain
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    [HttpPost("QueryCache/Clear")]
+    public ActionResult<SearchdomainInvalidateCacheResults> InvalidateSearchCache([Required]string searchdomain)
+    {
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
+        searchdomain_.InvalidateSearchCache();
         return Ok(new SearchdomainInvalidateCacheResults(){Success = true});
     }
 
-    [HttpGet("GetDatabaseSize")]
+    /// <summary>
-    public ActionResult<SearchdomainGetDatabaseSizeResult> GetDatabaseSize(string searchdomain)
+    /// Get the disk size of a searchdomain in bytes
+    /// </summary>
+    /// <param name="searchdomain">Name of the searchdomain</param>
+    [HttpGet("Database/Size")]
+    public ActionResult<SearchdomainGetDatabaseSizeResult> GetDatabaseSize([Required]string searchdomain)
     {
-        Searchdomain searchdomain_;
+        (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_domainManager, searchdomain, _logger);
-        try
+        if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new SearchdomainUpdateResults(){Success = false, Message = message});
-        {
+        long EmbeddingCacheUtilization = DatabaseHelper.GetSearchdomainDatabaseSize(searchdomain_.helper, searchdomain);
-            searchdomain_ = _domainManager.GetSearchdomain(searchdomain);
+        return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = EmbeddingCacheUtilization, Success = true });        
-        }
-        catch (SearchdomainNotFoundException)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]);
-            return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = null, Success = false, Message = "Searchdomain not found" });
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
-            return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = null, Success = false, Message = ex.Message });
-        }
-        long sizeInBytes = DatabaseHelper.GetSearchdomainDatabaseSize(searchdomain_.helper, searchdomain);
-        return Ok(new SearchdomainGetDatabaseSizeResult() { SearchdomainDatabaseSizeBytes = sizeInBytes, Success = true });        
     }
 }

src/Server/Controllers/ServerController.cs

@@ -1,10 +1,14 @@
 namespace Server.Controllers;
 
+using System.Reflection;
 using System.Text.Json;
+using AdaptiveExpressions;
 using ElmahCore;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Options;
 using Server.Exceptions;
 using Server.Helper;
+using Server.Models;
 using Shared.Models;
 
 [ApiController]
@@ -14,15 +18,25 @@ public class ServerController : ControllerBase
     private readonly ILogger<ServerController> _logger;
     private readonly IConfiguration _config;
     private AIProvider _aIProvider;
+    private readonly SearchdomainManager _searchdomainManager;
+    private readonly IOptions<EmbeddingSearchOptions> _options;
 
-    public ServerController(ILogger<ServerController> logger, IConfiguration config, AIProvider aIProvider)
+    public ServerController(ILogger<ServerController> logger, IConfiguration config, AIProvider aIProvider, SearchdomainManager searchdomainManager, IOptions<EmbeddingSearchOptions> options)
     {
         _logger = logger;
         _config = config;
         _aIProvider = aIProvider;
+        _searchdomainManager = searchdomainManager;
+        _options = options;
     }
 
-    [HttpGet("GetModels")]
+    /// <summary>
+    /// Lists the models available to the server
+    /// </summary>
+    /// <remarks>
+    /// Returns ALL models available to the server - not only the embedding models.
+    /// </remarks>
+    [HttpGet("Models")]
     public ActionResult<ServerGetModelsResult> GetModels()
     {
         try
@@ -35,4 +49,93 @@ public class ServerController : ControllerBase
             return new ServerGetModelsResult() { Success = false, Message = ex.Message};
         }
     }
+
+    /// <summary>
+    /// Gets numeric info regarding the searchdomains
+    /// </summary>
+    [HttpGet("Stats")]
+    public async Task<ActionResult<ServerGetStatsResult>> Stats()
+    {
+        try
+        {
+            long size = 0;
+            long elementCount = 0;
+            long embeddingsCount = 0;
+            LRUCache<string, Dictionary<string, float[]>> embeddingCache = _searchdomainManager.embeddingCache;
+            var cacheListField = embeddingCache.GetType()
+                .GetField("_cacheList", BindingFlags.Instance | BindingFlags.NonPublic) ?? throw new InvalidOperationException("_cacheList field not found"); // TODO Remove this unsafe reflection atrocity
+            LinkedList<string> cacheListOriginal = (LinkedList<string>)cacheListField.GetValue(embeddingCache)!;
+            LinkedList<string> cacheList = new(cacheListOriginal);
+
+            foreach (string key in cacheList)
+            {
+                if (!embeddingCache.TryGet(key, out var entry))
+                    continue;
+
+                // estimate size
+                size += EstimateEntrySize(key, entry);
+                elementCount++;
+                embeddingsCount += entry.Keys.Count;
+            }
+            var sqlHelper = DatabaseHelper.GetSQLHelper(_options.Value);
+            Task<long> entityCountTask = DatabaseHelper.CountEntities(sqlHelper);
+            long queryCacheUtilization = 0;
+            long queryCacheElementCount = 0;
+            long queryCacheMaxElementCountAll = 0;
+            long queryCacheMaxElementCountLoadedSearchdomainsOnly = 0;
+            foreach (string searchdomain in _searchdomainManager.ListSearchdomains())
+            {
+                if (SearchdomainHelper.IsSearchdomainLoaded(_searchdomainManager, searchdomain))
+                {
+                    (Searchdomain? searchdomain_, int? httpStatusCode, string? message) = SearchdomainHelper.TryGetSearchdomain(_searchdomainManager, searchdomain, _logger);
+                    if (searchdomain_ is null || httpStatusCode is not null) return StatusCode(httpStatusCode ?? 500, new ServerGetStatsResult(){Success = false, Message = message});
+                    queryCacheUtilization += searchdomain_.GetSearchCacheSize();
+                    queryCacheElementCount += searchdomain_.queryCache.Count;
+                    queryCacheMaxElementCountAll += searchdomain_.queryCache.Capacity;
+                    queryCacheMaxElementCountLoadedSearchdomainsOnly += searchdomain_.queryCache.Capacity;
+                } else
+                {
+                    var searchdomainSettings = DatabaseHelper.GetSearchdomainSettings(sqlHelper, searchdomain);
+                    queryCacheMaxElementCountAll += searchdomainSettings.QueryCacheSize;
+                }
+            };
+            long entityCount = await entityCountTask;
+            
+            return new ServerGetStatsResult() {
+                Success = true,
+                EntityCount = entityCount,
+                QueryCacheUtilization = queryCacheUtilization,
+                QueryCacheElementCount = queryCacheElementCount,
+                QueryCacheMaxElementCountAll = queryCacheMaxElementCountAll,
+                QueryCacheMaxElementCountLoadedSearchdomainsOnly = queryCacheMaxElementCountLoadedSearchdomainsOnly,
+                EmbeddingCacheUtilization = size,
+                EmbeddingCacheMaxElementCount = _searchdomainManager.EmbeddingCacheMaxCount,
+                EmbeddingCacheElementCount = elementCount,
+                EmbeddingsCount = embeddingsCount
+            };
+        } catch (Exception ex)
+        {
+            ElmahExtensions.RaiseError(ex);
+            return StatusCode(500, new ServerGetStatsResult(){Success = false, Message = ex.Message});
+        }
+    }
+
+    private static long EstimateEntrySize(string key, Dictionary<string, float[]> value)
+    {
+        int stringOverhead = MemorySizes.Align(MemorySizes.ObjectHeader + sizeof(int));
+        int arrayOverhead = MemorySizes.ArrayHeader;
+        int dictionaryOverhead = MemorySizes.ObjectHeader;
+        long size = 0;
+
+        size += stringOverhead + key.Length * sizeof(char);
+        size += dictionaryOverhead;
+
+        foreach (var kv in value)
+        {
+            size += stringOverhead + kv.Key.Length * sizeof(char);
+            size += arrayOverhead + kv.Value.Length * sizeof(float);
+        }
+
+        return size;
+    }
 }

src/Server/Datapoint.cs

@@ -1,3 +1,4 @@
+using AdaptiveExpressions;
 using OllamaSharp;
 using OllamaSharp.Models;
 
@@ -27,74 +28,31 @@ public class Datapoint
 
     public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider)
     {
-        return GenerateEmbeddings(content, models, aIProvider, []);
+        return GenerateEmbeddings(content, models, aIProvider, new());
     }
 
-    public static Dictionary<string, float[]> GenerateEmbeddings(List<string> contents, string model, OllamaApiClient ollama, Dictionary<string, Dictionary<string, float[]>> embeddingCache)
+    public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider, LRUCache<string, Dictionary<string, float[]>> embeddingCache)
-    {
-        Dictionary<string, float[]> retVal = [];
-
-        List<string> remainingContents = new List<string>(contents);
-        for (int i = contents.Count - 1; i >= 0; i--) // Compare against cache and remove accordingly
-        {
-            string content = contents[i];
-            if (embeddingCache.ContainsKey(model) && embeddingCache[model].ContainsKey(content))
-            {
-                retVal[content] = embeddingCache[model][content];
-                remainingContents.RemoveAt(i);
-            }
-        }
-        if (remainingContents.Count == 0)
-        {
-            return retVal;
-        } 
-
-        EmbedRequest request = new()
-        {
-            Model = model,
-            Input = remainingContents
-        };
-
-        EmbedResponse response = ollama.EmbedAsync(request).Result;
-        for (int i = 0; i < response.Embeddings.Count; i++)
-        {
-            string content = remainingContents.ElementAt(i);
-            float[] embeddings = response.Embeddings.ElementAt(i);
-            retVal[content] = embeddings;
-            if (!embeddingCache.ContainsKey(model))
-            {
-                embeddingCache[model] = [];
-            }
-            if (!embeddingCache[model].ContainsKey(content))
-            {
-                embeddingCache[model][content] = embeddings;
-            }
-        }
-
-        return retVal;
-    }
-
-    public static Dictionary<string, float[]> GenerateEmbeddings(string content, List<string> models, AIProvider aIProvider, Dictionary<string, Dictionary<string, float[]>> embeddingCache)
     {
         Dictionary<string, float[]> retVal = [];
         foreach (string model in models)
         {
-            if (embeddingCache.ContainsKey(model) && embeddingCache[model].ContainsKey(content))
+            bool embeddingCacheHasModel = embeddingCache.TryGet(model, out var embeddingCacheForModel);
+            if (embeddingCacheHasModel && embeddingCacheForModel.ContainsKey(content))
             {
-                retVal[model] = embeddingCache[model][content];
+                retVal[model] = embeddingCacheForModel[content];
                 continue;
             }
             var response = aIProvider.GenerateEmbeddings(model, [content]);
             if (response is not null)
             {
                 retVal[model] = response;
-                if (!embeddingCache.ContainsKey(model))
+                if (!embeddingCacheHasModel)
                 {
-                    embeddingCache[model] = [];
+                    embeddingCacheForModel = [];
                 }
-                if (!embeddingCache[model].ContainsKey(content))
+                if (!embeddingCacheForModel.ContainsKey(content))
                 {
-                    embeddingCache[model][content] = response;
+                    embeddingCacheForModel[content] = response;
                 }
             }
         }

src/Server/Exceptions/GeneralExceptions.cs

@@ -1,7 +1,9 @@
+using Shared.Models;
+
 namespace Server.Exceptions;
 
-public class ProbMethodNotFoundException(string probMethod) : Exception($"Unknown probMethod name {probMethod}") { }
+public class ProbMethodNotFoundException(ProbMethodEnum probMethod) : Exception($"Unknown probMethod name {probMethod}") { }
 
-public class SimilarityMethodNotFoundException(string similarityMethod) : Exception($"Unknown similarityMethod name \"{similarityMethod}\"") { }
+public class SimilarityMethodNotFoundException(SimilarityMethodEnum similarityMethod) : Exception($"Unknown similarityMethod name \"{similarityMethod}\"") { }
 
 public class JSONPathSelectionException(string path, string testedContent) : Exception($"Unable to select tokens using JSONPath {path} for string: {testedContent}.") { }

src/Server/Helper/DatabaseHelper.cs

@@ -1,6 +1,10 @@
+using System.Configuration;
 using System.Data.Common;
 using System.Text;
+using System.Text.Json;
+using MySql.Data.MySqlClient;
 using Server.Exceptions;
+using Server.Models;
 using Shared.Models;
 
 namespace Server.Helper;
@@ -9,6 +13,14 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
 {
     private readonly ILogger<DatabaseHelper> _logger = logger;
 
+    public static SQLHelper GetSQLHelper(EmbeddingSearchOptions embeddingSearchOptions)
+    {
+        string connectionString = embeddingSearchOptions.ConnectionStrings.SQL;
+        MySqlConnection connection = new(connectionString);
+        connection.Open();
+        return new SQLHelper(connection, connectionString);
+    }
+
     public static void DatabaseInsertEmbeddingBulk(SQLHelper helper, int id_datapoint, List<(string model, byte[] embedding)> data)
     {
         Dictionary<string, object> parameters = [];
@@ -38,12 +50,12 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
         return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO searchdomain (name, settings) VALUES (@name, @settings)", parameters);
     }
 
-    public static int DatabaseInsertEntity(SQLHelper helper, string name, string probmethod, int id_searchdomain)
+    public static int DatabaseInsertEntity(SQLHelper helper, string name, ProbMethodEnum probmethod, int id_searchdomain)
     {
         Dictionary<string, dynamic> parameters = new()
         {
             { "name", name },
-            { "probmethod", probmethod },
+            { "probmethod", probmethod.ToString() },
             { "id_searchdomain", id_searchdomain }
         };
         return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO entity (name, probmethod, id_searchdomain) VALUES (@name, @probmethod, @id_searchdomain)", parameters);
@@ -60,13 +72,13 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
         return helper.ExecuteSQLCommandGetInsertedID("INSERT INTO attribute (attribute, value, id_entity) VALUES (@attribute, @value, @id_entity)", parameters);
     }
 
-    public static int DatabaseInsertDatapoint(SQLHelper helper, string name, string probmethod_embedding, string similarityMethod, string hash, int id_entity)
+    public static int DatabaseInsertDatapoint(SQLHelper helper, string name, ProbMethodEnum probmethod_embedding, SimilarityMethodEnum similarityMethod, string hash, int id_entity)
     {
         Dictionary<string, dynamic> parameters = new()
         {
             { "name", name },
-            { "probmethod_embedding", probmethod_embedding },
+            { "probmethod_embedding", probmethod_embedding.ToString() },
-            { "similaritymethod", similarityMethod },
+            { "similaritymethod", similarityMethod.ToString() },
             { "hash", hash },
             { "id_entity", id_entity }
         };
@@ -211,4 +223,44 @@ public class DatabaseHelper(ILogger<DatabaseHelper> logger)
 
         return result;
     }
+
+    public static async Task<long> CountEntities(SQLHelper helper)
+    {
+        DbDataReader searchdomainSumReader = helper.ExecuteSQLCommand("SELECT COUNT(*) FROM entity;", []);
+        bool success = searchdomainSumReader.Read();
+        long result = success && !searchdomainSumReader.IsDBNull(0) ? searchdomainSumReader.GetInt64(0) : 0;
+        searchdomainSumReader.Close();
+        return result;
+    }
+
+    public static long CountEntitiesForSearchdomain(SQLHelper helper, string searchdomain)
+    {
+        Dictionary<string, dynamic> parameters = new()
+        {
+            { "searchdomain", searchdomain}
+        };
+        DbDataReader searchdomainSumReader = helper.ExecuteSQLCommand("SELECT COUNT(*) FROM entity e JOIN searchdomain s on e.id_searchdomain = s.id WHERE e.id_searchdomain = s.id AND s.name = @searchdomain;", parameters);
+        bool success = searchdomainSumReader.Read();
+        long result = success && !searchdomainSumReader.IsDBNull(0) ? searchdomainSumReader.GetInt64(0) : 0;
+        searchdomainSumReader.Close();
+        return result;
+    }
+
+    public static SearchdomainSettings GetSearchdomainSettings(SQLHelper helper, string searchdomain)
+    {
+        Dictionary<string, dynamic> parameters = new()
+        {
+            ["name"] = searchdomain
+        };
+        DbDataReader reader = helper.ExecuteSQLCommand("SELECT settings from searchdomain WHERE name = @name", parameters);
+        try
+        {
+            reader.Read();
+            string settingsString = reader.GetString(0);
+            return JsonSerializer.Deserialize<SearchdomainSettings>(settingsString);
+        } finally
+        {
+            reader.Close();
+        }
+    }
 }

src/Server/Helper/SQLHelper.cs

@@ -1,3 +1,4 @@
+using System.Data;
 using System.Data.Common;
 using MySql.Data.MySqlClient;
 
@@ -6,6 +7,7 @@ namespace Server.Helper;
 public class SQLHelper:IDisposable
 {
     public MySqlConnection connection;
+    public DbDataReader? dbDataReader;
     public string connectionString;
     public SQLHelper(MySqlConnection connection, string connectionString)
     {
@@ -30,13 +32,15 @@ public class SQLHelper:IDisposable
         lock (connection)
         {
             EnsureConnected();
+            EnsureDbReaderIsClosed();
             using MySqlCommand command = connection.CreateCommand();
             command.CommandText = query;
             foreach (KeyValuePair<string, dynamic> parameter in parameters)
             {
                 command.Parameters.AddWithValue($"@{parameter.Key}", parameter.Value);
             }
-            return command.ExecuteReader();
+            dbDataReader = command.ExecuteReader();
+            return dbDataReader;
         }
     }
 
@@ -45,6 +49,7 @@ public class SQLHelper:IDisposable
         lock (connection)
         {
             EnsureConnected();
+            EnsureDbReaderIsClosed();
             using MySqlCommand command = connection.CreateCommand();
 
             command.CommandText = query;
@@ -61,6 +66,7 @@ public class SQLHelper:IDisposable
         lock (connection)
         {
             EnsureConnected();
+            EnsureDbReaderIsClosed();
             using MySqlCommand command = connection.CreateCommand();
 
             command.CommandText = query;
@@ -83,11 +89,29 @@ public class SQLHelper:IDisposable
                 connection.Close();
                 connection.Open();
             }
-            catch (Exception)
+            catch (Exception ex)
             {
-                throw; // TODO add logging here
+                ElmahCore.ElmahExtensions.RaiseError(ex);
+                throw;
             }
         }
         return true;
     }
+
+    public void EnsureDbReaderIsClosed()
+    {
+        int counter = 0;
+        int sleepTime = 10;
+        int timeout = 5000;
+        while (!(dbDataReader?.IsClosed ?? true))
+        {
+            if (counter > timeout / sleepTime)
+            {
+                TimeoutException ex = new("Unable to ensure dbDataReader is closed");
+                ElmahCore.ElmahExtensions.RaiseError(ex);
+                throw ex;
+            }
+            Thread.Sleep(sleepTime);
+        }
+    }
 }

src/Server/Helper/SearchdomainHelper.cs

@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
 using System.Security.Cryptography;
 using System.Text;
 using System.Text.Json;
+using AdaptiveExpressions;
 using Server.Exceptions;
 using Shared.Models;
 
@@ -46,7 +47,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
 
     public List<Entity>? EntitiesFromJSON(SearchdomainManager searchdomainManager, ILogger logger, string json)
     {
-        Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
+        LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomainManager.embeddingCache;
         AIProvider aIProvider = searchdomainManager.aIProvider;
         SQLHelper helper = searchdomainManager.helper;
 
@@ -87,12 +88,13 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
 
     public Entity? EntityFromJSON(SearchdomainManager searchdomainManager, ILogger logger, JSONEntity jsonEntity) //string json)
     {
-        SQLHelper helper = searchdomainManager.helper.DuplicateConnection();
+        using SQLHelper helper = searchdomainManager.helper.DuplicateConnection();
         Searchdomain searchdomain = searchdomainManager.GetSearchdomain(jsonEntity.Searchdomain);
         List<Entity> entityCache = searchdomain.entityCache;
         AIProvider aIProvider = searchdomain.aIProvider;
-        Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
+        LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
         Entity? preexistingEntity = entityCache.FirstOrDefault(entity => entity.name == jsonEntity.Name);
+        bool invalidateSearchCache = false;
 
         if (preexistingEntity is not null)
         {
@@ -147,8 +149,9 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             }
 
             // Datapoint
-            foreach (Datapoint datapoint in preexistingEntity.datapoints.ToList())
+            foreach (Datapoint datapoint_ in preexistingEntity.datapoints.ToList())
             {
+                Datapoint datapoint = datapoint_; // To enable replacing the datapoint reference as foreach iterators cannot be overwritten
                 bool newEntityHasDatapoint = jsonEntity.Datapoints.Any(x => x.Name == datapoint.name);
                 if (!newEntityHasDatapoint)
                 {
@@ -161,6 +164,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
                     helper.ExecuteSQLNonQuery("DELETE e FROM embedding e JOIN datapoint d ON e.id_datapoint=d.id WHERE d.name=@datapointName AND d.id_entity=@entityId", parameters);
                     helper.ExecuteSQLNonQuery("DELETE FROM datapoint WHERE id_entity=@entityId AND name=@datapointName", parameters);
                     preexistingEntity.datapoints.Remove(datapoint);
+                    invalidateSearchCache = true;
                 } else
                 {
                     JSONDatapoint? newEntityDatapoint = jsonEntity.Datapoints.FirstOrDefault(x => x.Name == datapoint.name);
@@ -177,22 +181,24 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
                         preexistingEntity.datapoints.Remove(datapoint);
                         Datapoint newDatapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, newEntityDatapoint, (int)preexistingEntityID);
                         preexistingEntity.datapoints.Add(newDatapoint);
-
+                        datapoint = newDatapoint;
+                        invalidateSearchCache = true;
                     }
-                    if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.name || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.name))
+                    if (newEntityDatapoint is not null && (newEntityDatapoint.Probmethod_embedding != datapoint.probMethod.probMethodEnum || newEntityDatapoint.SimilarityMethod != datapoint.similarityMethod.similarityMethodEnum))
                     {
                         // Datapoint - Updated (probmethod or similaritymethod)
                         Dictionary<string, dynamic> parameters = new()
                         {
-                            { "probmethod", newEntityDatapoint.Probmethod_embedding },
+                            { "probmethod", newEntityDatapoint.Probmethod_embedding.ToString() },
-                            { "similaritymethod", newEntityDatapoint.SimilarityMethod },
+                            { "similaritymethod", newEntityDatapoint.SimilarityMethod.ToString() },
                             { "datapointName", datapoint.name },
                             { "entityId", preexistingEntityID}
                         };
                         helper.ExecuteSQLNonQuery("UPDATE datapoint SET probmethod_embedding=@probmethod, similaritymethod=@similaritymethod WHERE id_entity=@entityId AND name=@datapointName", parameters);
                         Datapoint preexistingDatapoint = preexistingEntity.datapoints.First(x => x == datapoint); // The for loop is a copy. This retrieves the original such that it can be updated.
-                        preexistingDatapoint.probMethod = datapoint.probMethod;
+                        preexistingDatapoint.probMethod = new(newEntityDatapoint.Probmethod_embedding, _logger);
-                        preexistingDatapoint.similarityMethod = datapoint.similarityMethod;
+                        preexistingDatapoint.similarityMethod = new(newEntityDatapoint.SimilarityMethod, _logger);
+                        invalidateSearchCache = true;
                     }
                 }
             }
@@ -204,10 +210,15 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
                     // Datapoint - New
                     Datapoint datapoint = DatabaseInsertDatapointWithEmbeddings(helper, searchdomain, jsonDatapoint, (int)preexistingEntityID);
                     preexistingEntity.datapoints.Add(datapoint);
+                    invalidateSearchCache = true;
                 }
             }
 
-
+            if (invalidateSearchCache)
+            {
+                searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(preexistingEntity);
+            }
+            searchdomain.UpdateModelsInUse();
             return preexistingEntity;
         }
         else
@@ -227,11 +238,13 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             }
 
             var probMethod = Probmethods.GetMethod(jsonEntity.Probmethod) ?? throw new ProbMethodNotFoundException(jsonEntity.Probmethod);
-            Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod, datapoints, jsonEntity.Name)
+            Entity entity = new(jsonEntity.Attributes, probMethod, jsonEntity.Probmethod.ToString(), datapoints, jsonEntity.Name)
             {
                 id = id_entity
             };
             entityCache.Add(entity);
+            searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(entity);
+            searchdomain.UpdateModelsInUse();
             return entity;
         }
     }
@@ -261,7 +274,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             throw new Exception("jsonDatapoint.Text must not be null at this point");
         }
         using SQLHelper helper = searchdomain.helper.DuplicateConnection();
-        Dictionary<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
+        LRUCache<string, Dictionary<string, float[]>> embeddingCache = searchdomain.embeddingCache;
         hash ??= Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes(jsonDatapoint.Text)));
         DatabaseHelper.DatabaseInsertDatapoint(helper, jsonDatapoint.Name, jsonDatapoint.Probmethod_embedding, jsonDatapoint.SimilarityMethod, hash, entityId);
         Dictionary<string, float[]> embeddings = Datapoint.GenerateEmbeddings(jsonDatapoint.Text, [.. jsonDatapoint.Model], searchdomain.aIProvider, embeddingCache);
@@ -269,4 +282,26 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
         var similarityMethod = new SimilarityMethod(jsonDatapoint.SimilarityMethod, logger) ?? throw new SimilarityMethodNotFoundException(jsonDatapoint.SimilarityMethod);
         return new Datapoint(jsonDatapoint.Name, probMethod_embedding, similarityMethod, hash, [.. embeddings.Select(kv => (kv.Key, kv.Value))]);
     }
+
+    public static (Searchdomain?, int?, string?) TryGetSearchdomain(SearchdomainManager searchdomainManager, string searchdomain, ILogger logger)
+    {
+        try
+        {
+            Searchdomain searchdomain_ = searchdomainManager.GetSearchdomain(searchdomain);
+            return (searchdomain_, null, null);
+        } catch (SearchdomainNotFoundException)
+        {
+            logger.LogError("Unable to update searchdomain {searchdomain} - not found", [searchdomain]);
+            return (null, 500, $"Unable to update searchdomain {searchdomain} - not found");
+        } catch (Exception ex)
+        {
+            logger.LogError("Unable to update searchdomain {searchdomain} - Exception: {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]);
+            return (null, 404, $"Unable to update searchdomain {searchdomain}");
+        }
+    }
+
+    public static bool IsSearchdomainLoaded(SearchdomainManager searchdomainManager, string name)
+    {
+        return searchdomainManager.IsSearchdomainLoaded(name);
+    }
 }

src/Server/Migrations/DatabaseMigrations.cs

@@ -91,4 +91,10 @@ public static class DatabaseMigrations
         helper.ExecuteSQLNonQuery("ALTER TABLE datapoint ADD COLUMN similaritymethod VARCHAR(512) NULL DEFAULT 'Cosine' AFTER probmethod_embedding", []);
         return 4;
     }
+
+    public static int UpdateFrom4(SQLHelper helper)
+    {
+        helper.ExecuteSQLNonQuery("UPDATE searchdomain SET settings = JSON_SET(settings, '$.QueryCacheSize', 1000000) WHERE JSON_EXTRACT(settings, '$.QueryCacheSize') is NULL;", []); // Set QueryCacheSize to a default of 1000000
+        return 5;
+    }
 }

src/Server/Models/Auth.cs

@@ -1,13 +0,0 @@
-namespace Server.Models;
-
-public class SimpleAuthOptions
-{
-    public List<SimpleUser> Users { get; set; } = new();
-}
-
-public class SimpleUser
-{
-    public string Username { get; set; } = "";
-    public string Password { get; set; } = "";
-    public string[] Roles { get; set; } = Array.Empty<string>();
-}

src/Server/Models/ConfigModels.cs

@@ -0,0 +1,41 @@
+using System.Configuration;
+using ElmahCore;
+using Shared.Models;
+
+namespace Server.Models;
+
+public class EmbeddingSearchOptions : ApiKeyOptions
+{
+    public required ConnectionStringsOptions ConnectionStrings { get; set; }
+    public ElmahOptions? Elmah { get; set; }
+    public required long EmbeddingCacheMaxCount { get; set; }
+    public required Dictionary<string, AiProvider> AiProviders { get; set; }
+    public required SimpleAuthOptions SimpleAuth { get; set; }
+    public required bool UseHttpsRedirection { get; set; }
+}
+
+public class AiProvider
+{
+    public required string Handler { get; set; }
+    public required string BaseURL { get; set; }
+    public string? ApiKey { get; set; }
+    public required string[] Allowlist { get; set; }
+    public required string[] Denylist { get; set; }
+}
+
+public class SimpleAuthOptions
+{
+    public List<SimpleUser> Users { get; set; } = [];
+}
+
+public class SimpleUser
+{
+    public string Username { get; set; } = "";
+    public string Password { get; set; } = "";
+    public string[] Roles { get; set; } = [];
+}
+
+public class ConnectionStringsOptions
+{
+    public required string SQL { get; set; }
+}

src/Server/Probmethods.cs

@@ -1,37 +1,29 @@
 using System.Text.Json;
 using Server.Exceptions;
+using Shared.Models;
 
 namespace Server;
 
 public class ProbMethod
 {
     public Probmethods.probMethodDelegate method;
+    public ProbMethodEnum probMethodEnum;
     public string name;
 
-    public ProbMethod(string name, ILogger logger)
+    public ProbMethod(ProbMethodEnum probMethodEnum, ILogger logger)
     {
-        this.name = name;
+        this.probMethodEnum = probMethodEnum;
+        this.name = probMethodEnum.ToString();
         Probmethods.probMethodDelegate? probMethod = Probmethods.GetMethod(name);
         if (probMethod is null)
         {
             logger.LogError("Unable to retrieve probMethod {name}", [name]);
-            throw new ProbMethodNotFoundException(name);
+            throw new ProbMethodNotFoundException(probMethodEnum);
         }
         method = probMethod;
     }
 }
 
-public enum ProbMethodEnum
-{
-    Mean,
-    HarmonicMean,
-    QuadraticMean,
-    GeometricMean,
-    EVEWAvg,
-    HVEWAvg,
-    LVEWAvg,
-    DictionaryWeightedAverage
-}
 
 public static class Probmethods
 {
@@ -54,6 +46,11 @@ public static class Probmethods
         };
     }
 
+    public static probMethodDelegate? GetMethod(ProbMethodEnum probMethodEnum)
+    {
+        return GetMethod(probMethodEnum.ToString());
+    }
+
     public static probMethodDelegate? GetMethod(string name)
     {
         string methodName = name;

src/Server/Program.cs

@@ -8,12 +8,32 @@ using Server.HealthChecks;
 using Server.Helper;
 using Server.Models;
 using Server.Services;
+using System.Text.Json.Serialization;
+using System.Reflection;
+using System.Configuration;
+using Microsoft.OpenApi.Models;
+using Shared.Models;
+using Microsoft.AspNetCore.ResponseCompression;
+using System.Net;
+using System.Text;
 
 var builder = WebApplication.CreateBuilder(args);
 
-// Add services to the container.
+// Add Controllers with views & string conversion for enums
+builder.Services.AddControllersWithViews()
+    .AddJsonOptions(options =>
+    {
+        options.JsonSerializerOptions.Converters.Add(
+            new JsonStringEnumConverter()
+        );
+    });
 
-builder.Services.AddControllersWithViews();
+// Add Configuration
+IConfigurationSection configurationSection = builder.Configuration.GetSection("Embeddingsearch");
+EmbeddingSearchOptions configuration = configurationSection.Get<EmbeddingSearchOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
+
+builder.Services.Configure<EmbeddingSearchOptions>(configurationSection);
+builder.Services.Configure<ApiKeyOptions>(configurationSection);
 
 // Add Localization
 builder.Services.AddLocalization(options => options.ResourcesPath = "Resources");
@@ -30,7 +50,37 @@ builder.Services.AddScoped<LocalizationService>();
 
 // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
 builder.Services.AddEndpointsApiExplorer();
-builder.Services.AddSwaggerGen();
+builder.Services.AddSwaggerGen(c =>
+{
+    var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml";
+    var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile);
+    c.IncludeXmlComments(xmlPath);
+    if (configuration.ApiKeys is not null)
+    {
+        c.AddSecurityDefinition("ApiKey", new OpenApiSecurityScheme
+        {
+            Description = "ApiKey must appear in header",
+            Type = SecuritySchemeType.ApiKey,
+            Name = "X-API-KEY",
+            In = ParameterLocation.Header,
+            Scheme = "ApiKeyScheme"
+        });
+        var key = new OpenApiSecurityScheme()
+        {
+            Reference = new OpenApiReference
+            {
+                Type = ReferenceType.SecurityScheme,
+                Id = "ApiKey"
+            },
+            In = ParameterLocation.Header
+        };
+        var requirement = new OpenApiSecurityRequirement
+        {
+            { key, []}
+        };
+        c.AddSecurityRequirement(requirement);
+    }
+});
 Log.Logger = new LoggerConfiguration()
     .ReadFrom.Configuration(builder.Configuration)
     .CreateLogger();
@@ -40,12 +90,17 @@ builder.Services.AddSingleton<SearchdomainHelper>();
 builder.Services.AddSingleton<SearchdomainManager>();
 builder.Services.AddSingleton<AIProvider>();
 builder.Services.AddHealthChecks()
-    .AddCheck<DatabaseHealthCheck>("DatabaseHealthCheck")
+    .AddCheck<DatabaseHealthCheck>("DatabaseHealthCheck", tags: ["Database"])
-    .AddCheck<AIProviderHealthCheck>("AIProviderHealthCheck");
+    .AddCheck<AIProviderHealthCheck>("AIProviderHealthCheck", tags: ["AIProvider"]);
 
 builder.Services.AddElmah<XmlFileErrorLog>(Options =>
 {
-    Options.LogPath = builder.Configuration.GetValue<string>("Embeddingsearch:Elmah:LogFolder") ?? "~/logs";
+    Options.OnPermissionCheck = context =>
+        context.User.Claims.Any(claim =>
+            claim.Value.Equals("Admin", StringComparison.OrdinalIgnoreCase)
+            || claim.Value.Equals("Elmah", StringComparison.OrdinalIgnoreCase)
+    );
+    Options.LogPath = configuration.Elmah?.LogPath ?? "~/logs";
 });
 
 builder.Services
@@ -63,29 +118,109 @@ builder.Services.AddAuthorization(options =>
         policy => policy.RequireRole("Admin"));
 });
 
-IConfigurationSection simpleAuthSection = builder.Configuration.GetSection("Embeddingsearch:SimpleAuth");
+builder.Services.AddResponseCompression(options =>
-if (simpleAuthSection.Exists()) builder.Services.Configure<SimpleAuthOptions>(simpleAuthSection);
+{
+    options.EnableForHttps = true;
+    options.Providers.Add<GzipCompressionProvider>();
+    options.Providers.Add<BrotliCompressionProvider>();
+    options.MimeTypes =
+    [
+        "text/plain",
+        "text/css",
+        "application/javascript",
+        "text/javascript",
+        "text/html",
+        "application/xml",
+        "text/xml",
+        "application/json",
+        "image/svg+xml"
+    ];
+});
 
 var app = builder.Build();
 
-List<string>? allowedIps = builder.Configuration.GetSection("Embeddingsearch:Elmah:AllowedHosts")
+app.UseAuthentication();
-    .Get<List<string>>();
+app.UseAuthorization();
+
+// Configure Elmah
+app.Use(async (context, next) =>
+{
+    if (context.Request.Path.StartsWithSegments("/elmah"))
+    {
+        context.Response.OnStarting(() =>
+        {
+            context.Response.Headers.Append(
+                "Content-Security-Policy",
+                "default-src 'self' 'unsafe-inline' 'unsafe-eval'"
+            );
+            return Task.CompletedTask;
+        });
+    }
+
+    await next();
+});
+app.Use(async (context, next) =>
+{
+    if (!context.Request.Path.StartsWithSegments("/elmah"))
+    {
+        await next();
+        return;
+    }
+
+    var originalBody = context.Response.Body;
+    using var memStream = new MemoryStream();
+    context.Response.Body = memStream;
+
+    await next();
+
+    memStream.Position = 0;
+    var html = await new StreamReader(memStream).ReadToEndAsync();
+
+    if (context.Response.ContentType?.Contains("text/html") == true)
+    {
+        html = html.Replace(
+            "</head>",
+            """
+            <link rel="stylesheet" href="/elmah-ui/custom.css" />
+            <script src="/elmah-ui/custom.js"></script>
+            </head>
+            """
+        );
+    }
+
+    var bytes = Encoding.UTF8.GetBytes(html);
+    context.Response.ContentLength = bytes.Length;
+    await originalBody.WriteAsync(bytes);
+    context.Response.Body = originalBody;
+});
+app.UseElmah();
+
+app.MapHealthChecks("/healthz");
+app.MapHealthChecks("/healthz/Database", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions
+{
+    Predicate = c => c.Name.Contains("Database")
+});
+
+app.MapHealthChecks("/healthz/AIProvider", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions
+{
+    Predicate = c => c.Name.Contains("AIProvider")
+});
+
+bool IsDevelopment = app.Environment.IsDevelopment();
 
 app.Use(async (context, next) =>
 {
-    bool requestIsElmah = context.Request.Path.StartsWithSegments("/elmah");
+    if (context.Request.Path.StartsWithSegments("/swagger"))
-    bool requestIsSwagger = context.Request.Path.StartsWithSegments("/swagger");
-    
-    if (requestIsElmah || requestIsSwagger)
     {
-        var remoteIp = context.Connection.RemoteIpAddress?.ToString();
+        if (!context.User.Identity?.IsAuthenticated ?? true)
-        bool blockRequest = allowedIps is null
-            || remoteIp is null
-            || !allowedIps.Contains(remoteIp);
-        if (blockRequest)
         {
-            context.Response.StatusCode = 403;
+            context.Response.Redirect($"/Account/Login?ReturnUrl={WebUtility.UrlEncode("/swagger")}");
-            await context.Response.WriteAsync("Forbidden");
+            return;
+        }
+
+        if (!context.User.IsInRole("Admin"))
+        {
+            context.Response.StatusCode = StatusCodes.Status403Forbidden;
             return;
         }
     }
@@ -93,26 +228,34 @@ app.Use(async (context, next) =>
     await next();
 });
 
-app.UseElmah();
+app.UseSwagger();
-
+app.UseSwaggerUI(options =>
-app.MapHealthChecks("/healthz");
-
-bool IsDevelopment = app.Environment.IsDevelopment();
-bool useSwagger = app.Configuration.GetValue<bool>("UseSwagger");
-bool? UseMiddleware = app.Configuration.GetValue<bool?>("UseMiddleware");
-
-// Configure the HTTP request pipeline.
-if (IsDevelopment || useSwagger)
 {
-    app.UseSwagger();
+    options.EnablePersistAuthorization();
-    app.UseSwaggerUI();
+    options.InjectStylesheet("/swagger-ui/custom.css");
-    //app.UseElmahExceptionPage(); // Messes with JSON response for API calls. Leaving this here so I don't accidentally put this in again later on.
+    options.InjectJavascript("/swagger-ui/custom.js");
-}
+});
-if (UseMiddleware == true && !IsDevelopment)
+//app.UseElmahExceptionPage(); // Messes with JSON response for API calls. Leaving this here so I don't accidentally put this in again later on.
+
+if (configuration.ApiKeys is not null)
 {
-    app.UseMiddleware<Shared.ApiKeyMiddleware>();
+    app.UseWhen(context =>
+    {
+        RouteData routeData = context.GetRouteData();
+        string controllerName = routeData.Values["controller"]?.ToString() ?? "StaticFile";
+        if (controllerName == "Account" || controllerName == "Home" || controllerName == "StaticFile")
+        {
+            return false;
+        }
+        return true;
+    }, appBuilder =>
+    {
+        appBuilder.UseMiddleware<Shared.ApiKeyMiddleware>();    
+    });
 }
 
+app.UseResponseCompression();
+
 // Add localization
 var supportedCultures = new[] { "de", "de-DE", "en-US" };
 var localizationOptions = new RequestLocalizationOptions()
@@ -121,10 +264,23 @@ var localizationOptions = new RequestLocalizationOptions()
     .AddSupportedUICultures(supportedCultures);
 app.UseRequestLocalization(localizationOptions);
 
-app.UseAuthentication();
-app.UseAuthorization();
-
 app.MapControllers();
-app.UseStaticFiles();
+app.UseStaticFiles(new StaticFileOptions
+{
+    OnPrepareResponse = ctx =>
+    {
+        string requestPath = ctx.Context.Request.Path.ToString();
+        string[] cachedSuffixes = [".css", ".js", ".png", ".ico", ".woff2"];
+        if (cachedSuffixes.Any(suffix => requestPath.EndsWith(suffix)))
+        {
+            ctx.Context.Response.GetTypedHeaders().CacheControl =
+                new Microsoft.Net.Http.Headers.CacheControlHeaderValue()
+                {
+                    Public = true,
+                    MaxAge = TimeSpan.FromDays(365)
+                };
+        }
+    }
+});
 
 app.Run();

src/Server/Resources/SharedResources.de.resx

@@ -24,4 +24,295 @@
   <data name="IrreversibleActionWarning" xml:space="preserve">
     <value>Diese Aktion kann nicht rückgängig gemacht werden.</value>
   </data>
+  <data name="Searchdomain selection" xml:space="preserve">
+    <value>Searchdomain Auswahl</value>
+  </data>
+  <data name="Create" xml:space="preserve">
+    <value>Erstellen</value>
+  </data>
+  <data name="Searchdomain information and settings" xml:space="preserve">
+    <value>Searchdomain Informationen und Einstellungen</value>
+  </data>
+  <data name="Actions" xml:space="preserve">
+    <value>Aktionen</value>
+  </data>
+  <data name="Rename" xml:space="preserve">
+    <value>Umbenennen</value>
+  </data>
+  <data name="Delete" xml:space="preserve">
+    <value>Löschen</value>
+  </data>
+  <data name="Settings" xml:space="preserve">
+    <value>Einstellungen</value>
+  </data>
+  <data name="Cache reconciliation" xml:space="preserve">
+    <value>Cache Abgleich</value>
+  </data>
+  <data name="Update" xml:space="preserve">
+    <value>Anpassen</value>
+  </data>
+  <data name="Search cache" xml:space="preserve">
+    <value>Such-Cache</value>
+  </data>
+  <data name="Search cache utilization" xml:space="preserve">
+    <value>Such-Cache-Speicherauslastung</value>
+  </data>
+  <data name="Clear" xml:space="preserve">
+    <value>Leeren</value>
+  </data>
+  <data name="Database size" xml:space="preserve">
+    <value>Größe in der Datenbank</value>
+  </data>
+  <data name="Add new entity" xml:space="preserve">
+    <value>Neue Entity erstellen</value>
+  </data>
+  <data name="Entity Details" xml:space="preserve">
+    <value>Entity Details</value>
+  </data>
+  <data name="Attributes" xml:space="preserve">
+    <value>Attribute</value>
+  </data>
+  <data name="Key" xml:space="preserve">
+    <value>Schlüssel</value>
+  </data>
+  <data name="Value" xml:space="preserve">
+    <value>Wert</value>
+  </data>
+  <data name="Datapoints" xml:space="preserve">
+    <value>Datapoints</value>
+  </data>
+  <data name="Name" xml:space="preserve">
+    <value>Name</value>
+  </data>
+  <data name="ProbMethod" xml:space="preserve">
+    <value>ProbMethod</value>
+  </data>
+  <data name="SimilarityMethod" xml:space="preserve">
+    <value>SimilarityMethod</value>
+  </data>
+  <data name="Close" xml:space="preserve">
+    <value>Schließen</value>
+  </data>
+  <data name="Query Details" xml:space="preserve">
+    <value>Suchanfrage Details</value>
+  </data>
+  <data name="Access times" xml:space="preserve">
+    <value>Zugriffszeiten</value>
+  </data>
+  <data name="Results" xml:space="preserve">
+    <value>Ergebnisse</value>
+  </data>
+  <data name="Score" xml:space="preserve">
+    <value>Bewertung</value>
+  </data>
+  <data name="Query Update" xml:space="preserve">
+    <value>Suchanfrage anpassen</value>
+  </data>
+  <data name="Rename searchdomain" xml:space="preserve">
+    <value>Searchdomain umbenennen</value>
+  </data>
+  <data name="Delete searchdomain" xml:space="preserve">
+    <value>Searchdomain löschen</value>
+  </data>
+  <data name="Create searchdomain" xml:space="preserve">
+    <value>Searchdomain anlegen</value>
+  </data>
+  <data name="Searchdomain name" xml:space="preserve">
+    <value>Searchdomain Name</value>
+  </data>
+  <data name="Enable cache reconciliation" xml:space="preserve">
+    <value>Cache-Abgleich verwenden</value>
+  </data>
+  <data name="Create entity" xml:space="preserve">
+    <value>Entity erstellen</value>
+  </data>
+  <data name="Entity name" xml:space="preserve">
+    <value>Entity Name</value>
+  </data>
+  <data name="Probmethod" xml:space="preserve">
+    <value>Probmethod</value>
+  </data>
+  <data name="Add attribute" xml:space="preserve">
+    <value>Attribut hinzufügen</value>
+  </data>
+  <data name="Probmethod_embedding" xml:space="preserve">
+    <value>Probmethod_embedding</value>
+  </data>
+  <data name="Similarity method" xml:space="preserve">
+    <value>Similarity method</value>
+  </data>
+  <data name="Model" xml:space="preserve">
+    <value>Modell</value>
+  </data>
+  <data name="Add datapoint" xml:space="preserve">
+    <value>Datapoint hinzufügen</value>
+  </data>
+  <data name="Delete entity" xml:space="preserve">
+    <value>Entity löschen</value>
+  </data>
+  <data name="Update entity" xml:space="preserve">
+    <value>Entity anpassen</value>
+  </data>
+  <data name="Action" xml:space="preserve">
+    <value>Aktion</value>
+  </data>
+  <data name="Delete query" xml:space="preserve">
+    <value>Suchanfrage löschen</value>
+  </data>
+  <data name="Creating entity" xml:space="preserve">
+    <value>Erstelle Entity</value>
+  </data>
+  <data name="Entity was created successfully" xml:space="preserve">
+    <value>Entity wurde erfolgreich erstellt</value>
+  </data>
+  <data name="Failed to create entity" xml:space="preserve">
+    <value>Entity konnte nicht erstellt werden</value>
+  </data>
+  <data name="Searchdomain was created successfully" xml:space="preserve">
+    <value>Searchdomain wurde erfolgreich erstellt</value>
+  </data>
+  <data name="Failed to create searchdomain" xml:space="preserve">
+    <value>Searchdomain konnte nicht erstellt werden</value>
+  </data>
+  <data name="Searchdomain cache was cleared successfully" xml:space="preserve">
+    <value>Searchdomain-Cache wurde erfolgreich geleert</value>
+  </data>
+  <data name="Failed to clear searchdomain cache" xml:space="preserve">
+    <value>Searchdomain-Cache konnte nicht geleert werden</value>
+  </data>
+  <data name="Entity was deleted successfully" xml:space="preserve">
+    <value>Entity wurde erfolgreich gelöscht</value>
+  </data>
+  <data name="Failed to delete entity" xml:space="preserve">
+    <value>Entity konnte nicht gelöscht werden</value>
+  </data>
+  <data name="Updating entity" xml:space="preserve">
+    <value>Entity wird angepasst</value>
+  </data>
+  <data name="Entity was updated successfully" xml:space="preserve">
+    <value>Entity wurde erfolgreich angepasst</value>
+  </data>
+  <data name="Failed to update entity" xml:space="preserve">
+    <value>Entity konnte nicht angepasst werden</value>
+  </data>
+  <data name="Search query was deleted successfully" xml:space="preserve">
+    <value>Suchanfrage wurde erfolgreich gelöscht</value>
+  </data>
+  <data name="Failed to delete search query" xml:space="preserve">
+    <value>Suchanfrage konnte nicht gelöscht werden</value>
+  </data>
+  <data name="Searchdomain was created successfully" xml:space="preserve">
+    <value>Searchdomain wurde erfolgreich erstellt</value>
+  </data>
+  <data name="Updating search query failed" xml:space="preserve">
+    <value>Suchanfrage konnte nicht angepasst werden</value>
+  </data>
+  <data name="Searchdomain was deleted successfully" xml:space="preserve">
+    <value>Searchdomain wurde erfolgreich gelöscht</value>
+  </data>
+  <data name="Failed to delete searchdomain" xml:space="preserve">
+    <value>Konnte Searchdomain nicht löschen</value>
+  </data>
+  <data name="Searchdomain was renamed successfully" xml:space="preserve">
+    <value>Searchdomain wurde erfolgreich umbenannt</value>
+  </data>
+  <data name="Failed to rename searchdomain" xml:space="preserve">
+    <value>Searchdomain konnte nicht umbenannt werden</value>
+  </data>
+  <data name="Searchdomain settings were updated successfully" xml:space="preserve">
+    <value>Searchdomain Einstellungen wurden erfolgreich angepasst</value>
+  </data>
+  <data name="Updating searchdomain settings failed" xml:space="preserve">
+    <value>Searchdomain Einstellungen konnten nicht angepasst werden</value>
+  </data>
+  <data name="Unable to fetch searchdomain config" xml:space="preserve">
+    <value>Searchdomain Einstellungen konnten nicht abgerufen werden</value>
+  </data>
+  <data name="Unable to fetch searchdomain cache utilization" xml:space="preserve">
+    <value>Searchdomain-Cache-Auslastung konnte nicht abgerufen werden</value>
+  </data>
+  <data name="Details" xml:space="preserve">
+    <value>Details</value>
+  </data>
+  <data name="Remove attribute" xml:space="preserve">
+    <value>Attribut entfernen</value>
+  </data>
+  <data name="Remove" xml:space="preserve">
+    <value>Entfernen</value>
+  </data>
+  <data name="Close alert" xml:space="preserve">
+    <value>Benachrichtigung schließen</value>
+  </data>
+  <data name="Recent queries" xml:space="preserve">
+    <value>Letzte Queries</value>
+  </data>
+  <data name="Home" xml:space="preserve">
+    <value>Dashboard</value>
+  </data>
+  <data name="Searchdomains" xml:space="preserve">
+    <value>Searchdomains</value>
+  </data>
+  <data name="Swagger" xml:space="preserve">
+    <value>Swagger</value>
+  </data>
+  <data name="Elmah" xml:space="preserve">
+    <value>Elmah</value>
+  </data>
+  <data name="Hi!" xml:space="preserve">
+    <value>Hallo!</value>
+  </data>
+  <data name="Hi, {0}!" xml:space="preserve">
+    <value>Hallo {0}!</value>
+  </data>
+  <data name="Embedding Cache" xml:space="preserve">
+    <value>Embedding-Cache</value>
+  </data>
+  <data name="Size" xml:space="preserve">
+    <value>Größe</value>
+  </data>
+  <data name="Strings" xml:space="preserve">
+    <value>Zeichenketten</value>
+  </data>
+  <data name="stringsCountInfo" xml:space="preserve">
+    <value>Die Anzahl der Zeichenketten, für die Embeddings vorliegen. D.h. wenn zwei Modelle verwendet werden, ist die Zahl der Embeddings zweimal so hoch.</value>
+  </data>
+  <data name="Embeddings" xml:space="preserve">
+    <value>Embeddings</value>
+  </data>
+  <data name="Health Checks" xml:space="preserve">
+    <value>Health Checks</value>
+  </data>
+  <data name="Server" xml:space="preserve">
+    <value>Server</value>
+  </data>
+  <data name="AI Providers" xml:space="preserve">
+    <value>AI Providers</value>
+  </data>
+  <data name="Count" xml:space="preserve">
+    <value>Anzahl</value>
+  </data>
+  <data name="Total Entities" xml:space="preserve">
+    <value>Entities insgesamt</value>
+  </data>
+  <data name="Total query cache utilization" xml:space="preserve">
+    <value>Query-Cache-Verwendung insgesamt</value>
+  </data>
+  <data name="Unable to fetch searchdomain database utilization" xml:space="preserve">
+    <value>Searchdomain Datenbank-Auslastung konnte nicht abgerufen werden</value>
+  </data>
+  <data name="Query cache entry count" xml:space="preserve">
+    <value>Query-Cache Einträge</value>
+  </data>
+  <data name="Query cache capacity (all)" xml:space="preserve">
+    <value>Query-Cache Kapazität (alle)</value>
+  </data>
+  <data name="queryCacheEntryCountAllInfo" xml:space="preserve">
+    <value>Anzahl der Einträge, die insgesamt in den Query-Cache passen. Ungeladene Searchdomains werden berücksichtigt.</value>
+  </data>
+  <data name="Query cache capacity (loaded)" xml:space="preserve">
+    <value>Query-Cache Kapazität (geladen)</value>
+  </data>
+  <data name="queryCacheEntryCountLoadedInfo" xml:space="preserve">
+    <value>Anzahl der Einträge, die insgesamt in den Query-Cache der geladenen Searchdomains passen.</value>
+  </data>
 </root>

src/Server/Resources/SharedResources.en.resx

@@ -24,4 +24,295 @@
   <data name="IrreversibleActionWarning" xml:space="preserve">
     <value>This action cannot be undone.</value>
   </data>
+  <data name="Searchdomain selection" xml:space="preserve">
+    <value>Searchdomain selection</value>
+  </data>
+  <data name="Create" xml:space="preserve">
+    <value>Create</value>
+  </data>
+  <data name="Searchdomain information and settings" xml:space="preserve">
+    <value>Searchdomain information and settings</value>
+  </data>
+  <data name="Actions" xml:space="preserve">
+    <value>Actions</value>
+  </data>
+  <data name="Rename" xml:space="preserve">
+    <value>Rename</value>
+  </data>
+  <data name="Delete" xml:space="preserve">
+    <value>Delete</value>
+  </data>
+  <data name="Settings" xml:space="preserve">
+    <value>Settings</value>
+  </data>
+  <data name="Cache reconciliation" xml:space="preserve">
+    <value>Cache reconciliation</value>
+  </data>
+  <data name="Update" xml:space="preserve">
+    <value>Update</value>
+  </data>
+  <data name="Search cache" xml:space="preserve">
+    <value>Search cache</value>
+  </data>
+  <data name="Search cache utilization" xml:space="preserve">
+    <value>Search cache utilization</value>
+  </data>
+  <data name="Clear" xml:space="preserve">
+    <value>Clear</value>
+  </data>
+  <data name="Database size" xml:space="preserve">
+    <value>Database size</value>
+  </data>
+  <data name="Add new entity" xml:space="preserve">
+    <value>Add new entity</value>
+  </data>
+  <data name="Entity Details" xml:space="preserve">
+    <value>Entity Details</value>
+  </data>
+  <data name="Attributes" xml:space="preserve">
+    <value>Attributes</value>
+  </data>
+  <data name="Key" xml:space="preserve">
+    <value>Key</value>
+  </data>
+  <data name="Value" xml:space="preserve">
+    <value>Value</value>
+  </data>
+  <data name="Datapoints" xml:space="preserve">
+    <value>Datapoints</value>
+  </data>
+  <data name="Name" xml:space="preserve">
+    <value>Name</value>
+  </data>
+  <data name="ProbMethod" xml:space="preserve">
+    <value>ProbMethod</value>
+  </data>
+  <data name="SimilarityMethod" xml:space="preserve">
+    <value>SimilarityMethod</value>
+  </data>
+  <data name="Close" xml:space="preserve">
+    <value>Close</value>
+  </data>
+  <data name="Query Details" xml:space="preserve">
+    <value>Query Details</value>
+  </data>
+  <data name="Access times" xml:space="preserve">
+    <value>Access times</value>
+  </data>
+  <data name="Results" xml:space="preserve">
+    <value>Results</value>
+  </data>
+  <data name="Score" xml:space="preserve">
+    <value>Score</value>
+  </data>
+  <data name="Query Update" xml:space="preserve">
+    <value>Query Update</value>
+  </data>
+  <data name="Rename searchdomain" xml:space="preserve">
+    <value>Rename searchdomain</value>
+  </data>
+  <data name="Delete searchdomain" xml:space="preserve">
+    <value>Delete searchdomain</value>
+  </data>
+  <data name="Create searchdomain" xml:space="preserve">
+    <value>Create searchdomain</value>
+  </data>
+  <data name="Searchdomain name" xml:space="preserve">
+    <value>Searchdomain name</value>
+  </data>
+  <data name="Enable cache reconciliation" xml:space="preserve">
+    <value>Enable cache reconciliation</value>
+  </data>
+  <data name="Create entity" xml:space="preserve">
+    <value>Create entity</value>
+  </data>
+  <data name="Entity name" xml:space="preserve">
+    <value>Entity name</value>
+  </data>
+  <data name="Probmethod" xml:space="preserve">
+    <value>Probmethod</value>
+  </data>
+  <data name="Add attribute" xml:space="preserve">
+    <value>Add attribute</value>
+  </data>
+  <data name="Probmethod_embedding" xml:space="preserve">
+    <value>Probmethod_embedding</value>
+  </data>
+  <data name="Similarity method" xml:space="preserve">
+    <value>Similarity method</value>
+  </data>
+  <data name="Model" xml:space="preserve">
+    <value>Model</value>
+  </data>
+  <data name="Add datapoint" xml:space="preserve">
+    <value>Add datapoint</value>
+  </data>
+  <data name="Delete entity" xml:space="preserve">
+    <value>Delete entity</value>
+  </data>
+  <data name="Update entity" xml:space="preserve">
+    <value>Update entity</value>
+  </data>
+  <data name="Action" xml:space="preserve">
+    <value>Action</value>
+  </data>
+  <data name="Delete query" xml:space="preserve">
+    <value>Delete query</value>
+  </data>
+  <data name="Creating entity" xml:space="preserve">
+    <value>Creating entity</value>
+  </data>
+  <data name="Entity was created successfully" xml:space="preserve">
+    <value>Entity was created successfully</value>
+  </data>
+  <data name="Failed to create entity" xml:space="preserve">
+    <value>Failed to create entity</value>
+  </data>
+  <data name="Searchdomain was created successfully" xml:space="preserve">
+    <value>Searchdomain was created successfully</value>
+  </data>
+  <data name="Failed to create searchdomain" xml:space="preserve">
+    <value>Failed to create searchdomain</value>
+  </data>
+  <data name="Searchdomain cache was cleared successfully" xml:space="preserve">
+    <value>Searchdomain cache was cleared successfully</value>
+  </data>
+  <data name="Failed to clear searchdomain cache" xml:space="preserve">
+    <value>Failed to clear searchdomain cache</value>
+  </data>
+  <data name="Entity was deleted successfully" xml:space="preserve">
+    <value>Entity was deleted successfully</value>
+  </data>
+  <data name="Failed to delete entity" xml:space="preserve">
+    <value>Failed to delete entity</value>
+  </data>
+  <data name="Updating entity" xml:space="preserve">
+    <value>Updating entity</value>
+  </data>
+  <data name="Entity was updated successfully" xml:space="preserve">
+    <value>Entity was updated successfully</value>
+  </data>
+  <data name="Failed to update entity" xml:space="preserve">
+    <value>Failed to update entity</value>
+  </data>
+  <data name="Search query was deleted successfully" xml:space="preserve">
+    <value>Search query was deleted successfully</value>
+  </data>
+  <data name="Failed to delete search query" xml:space="preserve">
+    <value>Failed to delete search query</value>
+  </data>
+  <data name="Searchdomain was created successfully" xml:space="preserve">
+    <value>Searchdomain was created successfully</value>
+  </data>
+  <data name="Updating search query failed" xml:space="preserve">
+    <value>Updating search query failed</value>
+  </data>
+  <data name="Searchdomain was deleted successfully" xml:space="preserve">
+    <value>Searchdomain was deleted successfully</value>
+  </data>
+  <data name="Failed to delete searchdomain" xml:space="preserve">
+    <value>Failed to delete searchdomain</value>
+  </data>
+  <data name="Searchdomain was renamed successfully" xml:space="preserve">
+    <value>Searchdomain was renamed successfully</value>
+  </data>
+  <data name="Failed to rename searchdomain" xml:space="preserve">
+    <value>Failed to rename searchdomain</value>
+  </data>
+  <data name="Searchdomain settings were updated successfully" xml:space="preserve">
+    <value>Searchdomain settings were updated successfully</value>
+  </data>
+  <data name="Updating searchdomain settings failed" xml:space="preserve">
+    <value>Updating searchdomain settings failed</value>
+  </data>
+  <data name="Unable to fetch searchdomain config" xml:space="preserve">
+    <value>Unable to fetch searchdomain config</value>
+  </data>
+  <data name="Unable to fetch searchdomain cache utilization" xml:space="preserve">
+    <value>"Unable to fetch searchdomain cache utilization</value>
+  </data>
+  <data name="Details" xml:space="preserve">
+    <value>Details</value>
+  </data>
+  <data name="Remove attribute" xml:space="preserve">
+    <value>Remove attribute</value>
+  </data>
+  <data name="Remove" xml:space="preserve">
+    <value>Remove</value>
+  </data>
+  <data name="Close alert" xml:space="preserve">
+    <value>Close alert</value>
+  </data>
+  <data name="Recent queries" xml:space="preserve">
+    <value>Recent queries</value>
+  </data>
+  <data name="Home" xml:space="preserve">
+    <value>Dashboard</value>
+  </data>
+  <data name="Searchdomains" xml:space="preserve">
+    <value>Searchdomains</value>
+  </data>
+  <data name="Swagger" xml:space="preserve">
+    <value>Swagger</value>
+  </data>
+  <data name="Elmah" xml:space="preserve">
+    <value>Elmah</value>
+  </data>
+  <data name="Hi!" xml:space="preserve">
+    <value>Hi!</value>
+  </data>
+  <data name="Hi, {0}!" xml:space="preserve">
+    <value>Hi {0}!</value>
+  </data>
+  <data name="Embedding Cache" xml:space="preserve">
+    <value>Embedding Cache</value>
+  </data>
+  <data name="Size" xml:space="preserve">
+    <value>Size</value>
+  </data>
+  <data name="Strings" xml:space="preserve">
+    <value>Strings</value>
+  </data>
+  <data name="stringsCountInfo" xml:space="preserve">
+    <value>The number of strings for which there are embeddings. I.e. If you use two models, the amount of embeddings will be twice this number.</value>
+  </data>
+  <data name="Embeddings" xml:space="preserve">
+    <value>Embeddings</value>
+  </data>
+  <data name="Health Checks" xml:space="preserve">
+    <value>Health Checks</value>
+  </data>
+  <data name="Server" xml:space="preserve">
+    <value>Server</value>
+  </data>
+  <data name="AI Providers" xml:space="preserve">
+    <value>AI Providers</value>
+  </data>
+  <data name="Count" xml:space="preserve">
+    <value>Count</value>
+  </data>
+  <data name="Total Entities" xml:space="preserve">
+    <value>Total Entities</value>
+  </data>
+  <data name="Total query cache utilization" xml:space="preserve">
+    <value>Total query cache utilization</value>
+  </data>
+  <data name="Unable to fetch searchdomain database utilization" xml:space="preserve">
+    <value>Unable to fetch searchdomain database utilization</value>
+  </data>
+  <data name="Query cache entry count" xml:space="preserve">
+    <value>Query cache entry count</value>
+  </data>
+  <data name="Query cache capacity (all)" xml:space="preserve">
+    <value>Query cache capacity (all)</value>
+  </data>
+  <data name="queryCacheEntryCountAllInfo" xml:space="preserve">
+    <value>Number of query cache entries that can be stored in the query cache, including searchdomains that are currently not loaded.</value>
+  </data>
+  <data name="Query cache capacity (loaded)" xml:space="preserve">
+    <value>Query cache capacity (loaded)</value>
+  </data>
+  <data name="queryCacheEntryCountLoadedInfo" xml:space="preserve">
+    <value>Number of query cache entries that can be stored in the query cache of all loaded searchdomains.</value>
+  </data>
 </root>

src/Server/Searchdomain.cs

@@ -4,7 +4,9 @@ using System.Text.Json;
 using ElmahCore.Mvc.Logger;
 using MySql.Data.MySqlClient;
 using Server.Helper;
+using Shared;
 using Shared.Models;
+using AdaptiveExpressions;
 
 namespace Server;
 
@@ -16,16 +18,15 @@ public class Searchdomain
     public string searchdomain;
     public int id;
     public SearchdomainSettings settings;
-    public Dictionary<string, DateTimedSearchResult> searchCache; // Key: query, Value: Search results for that query (with timestamp)
+    public EnumerableLruCache<string, DateTimedSearchResult> queryCache; // Key: query, Value: Search results for that query (with timestamp)
     public List<Entity> entityCache;
     public List<string> modelsInUse;
-    public Dictionary<string, Dictionary<string, float[]>> embeddingCache;
+    public LRUCache<string, Dictionary<string, float[]>> embeddingCache;
-    public int embeddingCacheMaxSize = 10000000;
     private readonly MySqlConnection connection;
     public SQLHelper helper;
     private readonly ILogger _logger;
 
-    public Searchdomain(string searchdomain, string connectionString, AIProvider aIProvider, Dictionary<string, Dictionary<string, float[]>> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false)
+    public Searchdomain(string searchdomain, string connectionString, AIProvider aIProvider, LRUCache<string, Dictionary<string, float[]>> embeddingCache, ILogger logger, string provider = "sqlserver", bool runEmpty = false)
     {
         _connectionString = connectionString;
         _provider = provider.ToLower();
@@ -33,12 +34,12 @@ public class Searchdomain
         this.aIProvider = aIProvider;
         this.embeddingCache = embeddingCache;
         this._logger = logger;
-        searchCache = [];
         entityCache = [];
         connection = new MySqlConnection(connectionString);
         connection.Open();
         helper = new SQLHelper(connection, connectionString);
         settings = GetSettings();
+        queryCache = new(settings.QueryCacheSize);
         modelsInUse = []; // To make the compiler shut up - it is set in UpdateSearchDomain() don't worry // yeah, about that...
         if (!runEmpty)
         {
@@ -96,8 +97,16 @@ public class Searchdomain
             string probmethodString = datapointReader.GetString(3);
             string similarityMethodString = datapointReader.GetString(4);
             string hash = datapointReader.GetString(5);
-            ProbMethod probmethod = new(probmethodString, _logger);
+            ProbMethodEnum probmethodEnum = (ProbMethodEnum)Enum.Parse(
-            SimilarityMethod similarityMethod = new(similarityMethodString, _logger);
+                typeof(ProbMethodEnum),
+                probmethodString
+            );
+            SimilarityMethodEnum similairtyMethodEnum = (SimilarityMethodEnum)Enum.Parse(
+                typeof(SimilarityMethodEnum),
+                similarityMethodString
+            );
+            ProbMethod probmethod = new(probmethodEnum, _logger);
+            SimilarityMethod similarityMethod = new(similairtyMethodEnum, _logger);
             if (embedding_unassigned.TryGetValue(id, out Dictionary<string, float[]>? embeddings) && probmethod is not null)
             {
                 embedding_unassigned.Remove(id);
@@ -151,44 +160,23 @@ public class Searchdomain
         }
         entityReader.Close();
         modelsInUse = GetModels(entityCache);
-        embeddingCache = []; // TODO remove this and implement proper remediation to improve performance
     }
 
     public List<(float, string)> Search(string query, int? topN = null)
     {
-        if (searchCache.TryGetValue(query, out DateTimedSearchResult cachedResult))
+        if (queryCache.TryGetValue(query, out DateTimedSearchResult cachedResult))
         {
             cachedResult.AccessDateTimes.Add(DateTime.Now);
             return [.. cachedResult.Results.Select(r => (r.Score, r.Name))];
         }
 
-        if (!embeddingCache.TryGetValue(query, out Dictionary<string, float[]>? queryEmbeddings))
+        Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query);
-        {
-            queryEmbeddings = Datapoint.GenerateEmbeddings(query, modelsInUse, aIProvider);
-            if (embeddingCache.Count < embeddingCacheMaxSize) // TODO add better way of managing cache limit hits
-            { // Idea: Add access count to each entry. On limit hit, sort the entries by access count and remove the bottom 10% of entries
-                embeddingCache.Add(query, queryEmbeddings);
-            }
-        } // TODO implement proper cache remediation for embeddingCache here
 
         List<(float, string)> result = [];
 
         foreach (Entity entity in entityCache)
         {
-            List<(string, float)> datapointProbs = [];
+            result.Add((EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings), entity.name));
-            foreach (Datapoint datapoint in entity.datapoints)
-            {
-                SimilarityMethod similarityMethod = datapoint.similarityMethod;
-                List<(string, float)> list = [];
-                foreach ((string, float[]) embedding in datapoint.embeddings)
-                {
-                    string key = embedding.Item1;
-                    float value = similarityMethod.method(queryEmbeddings[embedding.Item1], embedding.Item2);
-                    list.Add((key, value));
-                }
-                datapointProbs.Add((datapoint.name, datapoint.probMethod.method(list)));
-            }
-            result.Add((entity.probMethod(datapointProbs), entity.name));
         }
         IEnumerable<(float, string)> sortedResults = result.OrderByDescending(s => s.Item1);
         if (topN is not null)
@@ -200,23 +188,74 @@ public class Searchdomain
             [.. sortedResults.Select(r =>
                 new ResultItem(r.Item1, r.Item2 ))]
         );
-        searchCache[query] = new DateTimedSearchResult(DateTime.Now, searchResult);
+        queryCache.Set(query, new DateTimedSearchResult(DateTime.Now, searchResult));
         return results;
     }
 
+    public Dictionary<string, float[]> GetQueryEmbeddings(string query)
+    {
+        bool hasQuery = embeddingCache.TryGet(query, out Dictionary<string, float[]> queryEmbeddings);
+        bool allModelsInQuery = queryEmbeddings is not null && modelsInUse.All(model => queryEmbeddings.ContainsKey(model));
+        if (!(hasQuery && allModelsInQuery) || queryEmbeddings is null)
+        {
+            queryEmbeddings = Datapoint.GenerateEmbeddings(query, modelsInUse, aIProvider, embeddingCache);
+            if (!embeddingCache.TryGet(query, out var embeddingCacheForCurrentQuery))
+            {
+                embeddingCache.Set(query, queryEmbeddings);
+            }
+            else // embeddingCache already has an entry for this query, so the missing model-embedding pairs have to be filled in
+            {
+                foreach (KeyValuePair<string, float[]> kvp in queryEmbeddings) // kvp.Key = model, kvp.Value = embedding
+                {
+                    if (!embeddingCache.TryGet(kvp.Key, out var _))
+                    {
+                        embeddingCacheForCurrentQuery[kvp.Key] = kvp.Value;
+                    }
+                }
+            }
+        }
+        return queryEmbeddings;
+    }
+
+    public void UpdateModelsInUse()
+    {
+        modelsInUse = GetModels([.. entityCache]);
+    }
+
+    private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary<string, float[]> queryEmbeddings)
+    {
+        List<(string, float)> datapointProbs = [];
+        foreach (Datapoint datapoint in entity.datapoints)
+        {
+            SimilarityMethod similarityMethod = datapoint.similarityMethod;
+            List<(string, float)> list = [];
+            foreach ((string, float[]) embedding in datapoint.embeddings)
+            {
+                string key = embedding.Item1;
+                float value = similarityMethod.method(queryEmbeddings[embedding.Item1], embedding.Item2);
+                list.Add((key, value));
+            }
+            datapointProbs.Add((datapoint.name, datapoint.probMethod.method(list)));
+        }
+        return entity.probMethod(datapointProbs);
+    }
+
     public static List<string> GetModels(List<Entity> entities)
     {
         List<string> result = [];
-        foreach (Entity entity in entities)
+        lock (entities)
         {
-            foreach (Datapoint datapoint in entity.datapoints)
+            foreach (Entity entity in entities)
             {
-                foreach ((string, float[]) tuple in datapoint.embeddings)
+                foreach (Datapoint datapoint in entity.datapoints)
                 {
-                    string model = tuple.Item1;
+                    foreach ((string, float[]) tuple in datapoint.embeddings)
-                    if (!result.Contains(model))
                     {
-                        result.Add(model);
+                        string model = tuple.Item1;
+                        if (!result.Contains(model))
+                        {
+                            result.Add(model);
+                        }
                     }
                 }
             }
@@ -239,19 +278,70 @@ public class Searchdomain
 
     public SearchdomainSettings GetSettings()
     {
-        Dictionary<string, dynamic> parameters = new()
+        return DatabaseHelper.GetSearchdomainSettings(helper, searchdomain);
+    }
+
+    public void ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(Entity entity)
+    {
+        if (settings.CacheReconciliation)
         {
-            ["name"] = searchdomain
+            foreach (var element in queryCache)
-        };
+            {
-        DbDataReader reader = helper.ExecuteSQLCommand("SELECT settings from searchdomain WHERE name = @name", parameters);
+                string query = element.Key;
-        reader.Read();
+                DateTimedSearchResult searchResult = element.Value;
-        string settingsString = reader.GetString(0);
+
-        reader.Close();
+                Dictionary<string, float[]> queryEmbeddings = GetQueryEmbeddings(query);
-        return JsonSerializer.Deserialize<SearchdomainSettings>(settingsString);
+                float evaluationResult = EvaluateEntityAgainstQueryEmbeddings(entity, queryEmbeddings);
+
+                searchResult.Results.RemoveAll(x => x.Name == entity.name); // If entity already exists in that results list: remove it.
+
+                ResultItem newItem = new(evaluationResult, entity.name);
+                int index = searchResult.Results.BinarySearch(
+                    newItem,
+                    Comparer<ResultItem>.Create((a, b) => b.Score.CompareTo(a.Score)) // Invert searching order
+                );
+                if (index < 0) // If not found, BinarySearch gives the bitwise complement
+                    index = ~index;
+                searchResult.Results.Insert(index, newItem);
+            }
+        }
+        else
+        {
+            InvalidateSearchCache();
+        }
+    }
+
+    public void ReconciliateOrInvalidateCacheForDeletedEntity(Entity entity)
+    {
+        if (settings.CacheReconciliation)
+        {
+            foreach (KeyValuePair<string, DateTimedSearchResult> element in queryCache)
+            {
+                string query = element.Key;
+                DateTimedSearchResult searchResult = element.Value;
+                searchResult.Results.RemoveAll(x => x.Name == entity.name);
+            }
+        }
+        else
+        {
+            InvalidateSearchCache();
+        }
     }
 
     public void InvalidateSearchCache()
     {
-        searchCache = [];
+        queryCache = new(settings.QueryCacheSize);
+    }
+
+    public long GetSearchCacheSize()
+    {
+        long EmbeddingCacheUtilization = 0;
+        foreach (var entry in queryCache)
+        {
+            EmbeddingCacheUtilization += sizeof(int); // string length prefix
+            EmbeddingCacheUtilization += entry.Key.Length * sizeof(char); // string characters
+            EmbeddingCacheUtilization += entry.Value.EstimateSize();
+        }
+        return EmbeddingCacheUtilization;
     }
 }

src/Server/SearchdomainManager.cs

@@ -3,6 +3,11 @@ using System.Data.Common;
 using Server.Migrations;
 using Server.Helper;
 using Server.Exceptions;
+using AdaptiveExpressions;
+using Shared.Models;
+using System.Text.Json;
+using Microsoft.Extensions.Options;
+using Server.Models;
 
 namespace Server;
 
@@ -10,22 +15,24 @@ public class SearchdomainManager
 {
     private Dictionary<string, Searchdomain> searchdomains = [];
     private readonly ILogger<SearchdomainManager> _logger;
-    private readonly IConfiguration _config;
+    private readonly EmbeddingSearchOptions _options;
     public readonly AIProvider aIProvider;
     private readonly DatabaseHelper _databaseHelper;
     private readonly string connectionString;
     private MySqlConnection connection;
     public SQLHelper helper;
-    public Dictionary<string, Dictionary<string, float[]>> embeddingCache;
+    public LRUCache<string, Dictionary<string, float[]>> embeddingCache;
+    public long EmbeddingCacheMaxCount;
 
-    public SearchdomainManager(ILogger<SearchdomainManager> logger, IConfiguration config, AIProvider aIProvider, DatabaseHelper databaseHelper)
+    public SearchdomainManager(ILogger<SearchdomainManager> logger, IOptions<EmbeddingSearchOptions> options, AIProvider aIProvider, DatabaseHelper databaseHelper)
     {
         _logger = logger;
-        _config = config;
+        _options = options.Value;
         this.aIProvider = aIProvider;
         _databaseHelper = databaseHelper;
-        embeddingCache = [];
+        EmbeddingCacheMaxCount = _options.EmbeddingCacheMaxCount;
-        connectionString = _config.GetSection("Embeddingsearch").GetConnectionString("SQL") ?? "";
+        embeddingCache = new((int)EmbeddingCacheMaxCount);
+        connectionString = _options.ConnectionStrings.SQL;
         connection = new MySqlConnection(connectionString);
         connection.Open();
         helper = new SQLHelper(connection, connectionString);
@@ -66,7 +73,7 @@ public class SearchdomainManager
     {
         var searchdomain = GetSearchdomain(searchdomainName);
         searchdomain.UpdateEntityCache();
-        searchdomain.InvalidateSearchCache(); // TODO implement cache remediation (Suggestion: searchdomain-wide setting for cache remediation / invalidation - )
+        searchdomain.InvalidateSearchCache();
     }
 
     public List<string> ListSearchdomains()
@@ -75,15 +82,25 @@ public class SearchdomainManager
         {
             DbDataReader reader = helper.ExecuteSQLCommand("SELECT name FROM searchdomain", []);
             List<string> results = [];
-            while (reader.Read())
+            try
             {
-                results.Add(reader.GetString(0));
+                while (reader.Read())
+                {
+                    results.Add(reader.GetString(0));
+                }
+                return results;                
+            }
+            finally
+            {
+                reader.Close();
             }
-            reader.Close();
-            return results;
         }
     }
 
+    public int CreateSearchdomain(string searchdomain, SearchdomainSettings settings)
+    {
+        return CreateSearchdomain(searchdomain, JsonSerializer.Serialize(settings));
+    }
     public int CreateSearchdomain(string searchdomain, string settings = "{}")
     {
         if (searchdomains.TryGetValue(searchdomain, out Searchdomain? value))
@@ -113,4 +130,9 @@ public class SearchdomainManager
         searchdomains[name] = searchdomain;
         return searchdomain;
     }
+
+    public bool IsSearchdomainLoaded(string name)
+    {
+        return searchdomains.ContainsKey(name);
+    }
 }

src/Server/Server.csproj

@@ -6,7 +6,13 @@
     <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
+    <NoWarn>$(NoWarn);1591</NoWarn>
+  </PropertyGroup>
+
   <ItemGroup>
+    <PackageReference Include="AdaptiveExpressions" Version="4.23.0" />
     <PackageReference Include="ElmahCore" Version="2.1.2" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />

src/Server/SimilarityMethods.cs

@@ -1,16 +1,18 @@
 using System.Numerics.Tensors;
-using System.Text.Json;
+using Shared.Models;
 
 namespace Server;
 
 public class SimilarityMethod
 {
     public SimilarityMethods.similarityMethodDelegate method;
+    public SimilarityMethodEnum similarityMethodEnum;
     public string name;
 
-    public SimilarityMethod(string name, ILogger logger)
+    public SimilarityMethod(SimilarityMethodEnum similarityMethodEnum, ILogger logger)
     {
-        this.name = name;
+        this.similarityMethodEnum = similarityMethodEnum;
+        this.name = similarityMethodEnum.ToString();
         SimilarityMethods.similarityMethodDelegate? probMethod = SimilarityMethods.GetMethod(name);
         if (probMethod is null)
         {
@@ -21,14 +23,6 @@ public class SimilarityMethod
     }
 }
 
-public enum SimilarityMethodEnum
-{
-    Cosine,
-    Euclidian,
-    Manhattan,
-    Pearson
-}
-
 public static class SimilarityMethods
 {
     public delegate float similarityMethodProtoDelegate(float[] vector1, float[] vector2);

src/Server/Tools/CriticalCSS/CriticalCSSGenerator.js

@@ -0,0 +1,127 @@
+import { generate } from 'critical';
+import fs from 'fs';
+import path from 'path';
+
+import puppeteer from 'puppeteer';
+
+const browser = await puppeteer.launch();
+const page = await browser.newPage();
+
+// Login
+await page.goto('http://localhost:5146/Account/Login');
+await page.type('#username', 'admin');
+await page.type('#password', 'UnsafePractice.67');
+await page.click('button[type=submit]');
+await page.waitForNavigation();
+
+// Extract cookies
+const cookies = await page.cookies();
+await browser.close();
+
+async function generateCriticalCSSForViews() {
+  const viewsDir = '../../Views';
+  
+  // Helper function to get all .cshtml files recursively
+  function getAllCshtmlFiles(dir) {
+    let results = [];
+    const list = fs.readdirSync(dir);
+    
+    list.forEach(file => {
+      const filePath = path.join(dir, file);
+      const stat = fs.statSync(filePath);
+      if (stat && stat.isDirectory()) {
+        // Recursively get files from subdirectories
+        results = results.concat(getAllCshtmlFiles(filePath));
+      } else if (file.endsWith('.cshtml') && filePath.search("/_") == -1) {
+        results.push(filePath);
+      }
+    });
+    
+    return results;
+  }
+  
+  // Helper function to convert file path to URL path
+  function filePathToUrlPath(filePath) {
+    // Remove 'Views/' prefix
+    let relativePath = filePath.replace(/^Views[\/\\]/, '');
+    
+    // Remove .cshtml extension
+    relativePath = relativePath.replace(/\.cshtml$/, '');
+    
+    // Convert to URL format (replace \ with / and capitalize first letter)
+    const urlPath = relativePath
+      .split(/[\/\\]/)
+      .map((segment, index) => 
+        index === 0 ? segment : segment.charAt(0).toUpperCase() + segment.slice(1)
+      )
+      .join('/');
+    
+    // Handle the case where we have a single file (like Index.cshtml)
+    if (relativePath.includes('/')) {
+      // Convert to URL path format: Views/Home/Index.cshtml -> /Home/Index
+      return '/' + relativePath.replace(/\\/g, '/').replace(/\.cshtml$/, '');
+    } else {
+      // For files directly in Views folder (like Views/Index.cshtml)
+      return '/' + relativePath.replace(/\.cshtml$/, '');
+    }
+  }
+  
+  // Get all .cshtml files
+  const cshtmlFiles = getAllCshtmlFiles(viewsDir);
+  const criticalCssDir = '.';
+//   if (!fs.existsSync(criticalCssDir)) {
+//     fs.mkdirSync(criticalCssDir, { recursive: true });
+//   }
+  
+  // Process each file
+  for (const file of cshtmlFiles) {
+    try {
+      const urlPath = filePathToUrlPath(file).replace("../", "").replace("../", "").replace("/Views", "");
+      
+      // Generate critical CSS
+      await generate({
+        src: `http://localhost:5146${urlPath}?noCriticalCSS`,
+        inline: false,
+        width: 1920,
+        height: 1080,
+        penthouse: {
+          customHeaders: {
+            cookie: cookies.map(c => `${c.name}=${c.value}`).join('; ')
+          },
+          forceExclude: ['.btn'], // Otherwise buttons end up colorless and .btn overrides other classes like .btn-warning, etc. - so it has to be force-excluded here and re-added later
+          forceInclude: [
+            '[data-bs-theme="dark"]', '[data-bs-theme="dark"] body', '[data-bs-theme="dark"] .navbar', '[data-bs-theme="dark"] .card', '[data-bs-theme="dark"] .btn',
+            '.navbar',
+            '.col-md-4',
+            '.visually-hidden', // visually hidden headings
+            '.bi-info-circle-fill', '.text-info', // info icon
+            '.container', '.col-md-6', '.row', '.g-4', '.row>*',
+            'p', '.fs-3', '.py-4', // title
+            '.mb-4',
+            '.card', '.card-body', '.p-2', // card
+            'h2', '.card-title', '.fs-5', // card - title
+            '.d-flex', '.justify-content-between', '.mt-2', // card - content
+            '.progress', '.mt-3', // card - progress bar
+            '.list-group', '.list-group-flush', '.list-group-item', '.list-group-flush>.list-group-item', '.list-group-flush>.list-group-item:last-child', '.badge', '.bg-warning', '.bg-success', '.h-100', // card - health check list
+            '.btn-primary', '.btn-warning', '.btn-danger', '.btn-info', // Searchdomains buttons
+            '.col-md-8', '.sidebar',
+            '.mb-0', '.mb-2', '.align-items-center',
+            'h3', '.col-md-3', '.col-md-2', '.text-nowrap', '.overflow-auto'
+          ]
+        },
+        target: {
+          css: path.join(criticalCssDir, "../../CriticalCSS/" + urlPath.replace(/\//g, '.').replace(/^\./, '').replace("...", "") + '.css')
+        }
+      });
+      
+      console.log(`Critical CSS generated for: ${urlPath}`);
+    } catch (err) {
+      console.error(`Error processing ${file}:`, err);
+    }
+  }
+  
+  console.log('All critical CSS files generated!');
+}
+
+// Run the function
+generateCriticalCSSForViews().catch(console.error);

src/Server/Tools/CriticalCSS/README.md

@@ -0,0 +1,11 @@
+# How to use CriticalCSS
+1. Install it here
+```bash
+npm i -D critical
+npm install puppeteer
+```
+2. Run the css generator:
+```bash
+node CriticalCSSGenerator.js
+```
+3. Move the `.css` files from the current directory to the `CriticalCSS/` folder (overwrite existing files)

src/Server/Tools/LocalizationChecker/LocalizationChecker.py

@@ -0,0 +1,78 @@
+import re
+import sys
+
+def extract_translations_from_View(view_path):
+    """Extract all translation strings from file A"""
+    translations = {}
+    
+    try:
+        with open(view_path, 'r', encoding='utf-8') as file_a:
+            for line_num, line in enumerate(file_a, 1):
+                # Match T["..."] patterns
+                matches = re.findall(r'T\["([^"]*)"\]', line)
+                for match in matches:
+                    translations[match] = line_num
+    except FileNotFoundError:
+        print(f"Error: File {view_path} not found")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error reading file {view_path}: {e}")
+        sys.exit(1)
+    
+    return translations
+
+def extract_localizations_from_resource_file(file_b_path):
+    """Extract all translation strings from file B"""
+    translations = set()
+    
+    try:
+        with open(file_b_path, 'r', encoding='utf-8') as file_b:
+            for line in file_b:
+                # Match the pattern in file B
+                match = re.search(r'<data name="([^"]*)"', line)
+                if match:
+                    translations.add(match.group(1))
+    except FileNotFoundError:
+        print(f"Error: File {file_b_path} not found")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error reading file {file_b_path}: {e}")
+        sys.exit(1)
+    
+    return translations
+
+def find_missing_translations(view, resource):
+    """Find translations in file A that don't exist in file B"""
+    # Extract translations from both files
+    file_a_translations = extract_translations_from_View(view)
+    file_b_translations = extract_localizations_from_resource_file(resource)
+    
+    # Find missing translations
+    missing_translations = []
+    
+    for translation_text, line_number in file_a_translations.items():
+        if translation_text not in file_b_translations:
+            missing_translations.append((translation_text, line_number))
+    
+    return missing_translations
+
+def main():
+    views = ["Shared/_Layout.cshtml", "Home/Index.cshtml", "Home/Searchdomains.cshtml"]
+    resources = ["SharedResources.en.resx", "SharedResources.de.resx"]
+    
+    print("Checking for missing translations...")
+    print("=" * 50)
+    for view in views:
+        for resource in resources:
+            missing = find_missing_translations("../../Views/" + view, "../../Resources/" + resource)
+            
+            if missing:
+                print(f"Found {len(missing)} missing translations in {view}:")
+                print("-" * 50)
+                for translation_text, line_number in missing:
+                    print(f"Line {line_number}: T[\"{translation_text}\"]")
+            else:
+                print(f"All localizations in {view} have a matching resource in {resource}!")
+
+if __name__ == "__main__":
+    main()

src/Server/Views/Account/Login.cshtml

@@ -1,3 +1,4 @@
+@using Microsoft.Extensions.Primitives
 @using Server.Services
 @inject LocalizationService T
 @{
@@ -9,6 +10,10 @@
     <h1>Login</h1>
     <form asp-action="Login" method="post" class="mt-4" style="max-width: 400px; margin: auto;">
         <div class="form-group mb-3">
+            @if (Context.Request.Query.TryGetValue("ReturnUrl", out StringValues returnUrl))
+            {
+                <input type="hidden" name="ReturnUrl" value="@(returnUrl)" />
+            }
             <label for="username" class="form-label">@T["Username"]</label>
             <input autofocus type="text" class="form-control" id="username" name="username" autocomplete="username" required>
         </div>

src/Server/Views/Home/Privacy.cshtml

@@ -1,6 +0,0 @@
-@{
-    ViewData["Title"] = "Privacy Policy";
-}
-<h1>@ViewData["Title"]</h1>
-
-<p>Use this page to detail your site's privacy policy.</p>

src/Server/Views/Shared/_Layout.cshtml

@@ -1,23 +1,55 @@
-@using Server.Services
+@using System.Globalization
+@using Server.Services
+@using System.Net
 @inject LocalizationService T
-
+@{
+    var currentUrl = WebUtility.HtmlEncode(Context.Request.Path);
+}
 <!DOCTYPE html>
-<html lang="en">
+<html lang="@CultureInfo.CurrentUICulture.TwoLetterISOLanguageName">
 <head>
     <meta charset="utf-8" />
+    <meta name="description" content="Embeddingsearch server" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>@ViewData["Title"] - embeddingsearch</title>
-    <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
+    @if (!Context.Request.Query.ContainsKey("renderRaw") && !Context.Request.Query.ContainsKey("noCriticalCSS"))
-    <link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
+    {
+        <link rel="preload" href="~/lib/bootstrap/dist/css/bootstrap.min.css" as="style"/>
+        <link rel="stylesheet" fetchpriority="high"
+        href="~/lib/bootstrap/dist/css/bootstrap.min.css"
+        media="print"
+        onload="this.media='all'">
+    } else if (Context.Request.Query.ContainsKey("noCriticalCSS"))
+    {
+        <link rel="preload" href="~/lib/bootstrap/dist/css/bootstrap.min.css" as="style"/>
+        <link rel="stylesheet" fetchpriority="high"
+        href="~/lib/bootstrap/dist/css/bootstrap.min.css">
+    }
+    <style>
+        @Html.Raw(File.ReadAllText(System.IO.Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "css", "site.css")))
+    </style>
+    @if (!Context.Request.Query.ContainsKey("noCriticalCSS"))
+    {
+        <style>
+            @if (Context.Request.Path.Value is not null)
+            {
+                string path = System.IO.Path.Combine("CriticalCSS", Context.Request.Path.Value.Trim('/').Replace("/", ".") + ".css");
+                if (File.Exists(path))
+                {
+                    @Html.Raw(File.ReadAllText(path));
+                }
+            }
+        </style>
+    }
     <script>
         window.appTranslations = {
             closeAlert: '@T["Close alert"]'
         };
     </script>
 </head>
-<body>
+<body data-bs-theme="dark">
     <header>
-        <nav class="navbar navbar-expand-sm navbar-toggleable-sm navbar-light bg-white border-bottom box-shadow mb-3">
+        <nav class="navbar navbar-expand-sm navbar-toggleable-sm navbar-light border-bottom box-shadow mb-3">
             <div class="container-fluid">
                 <a class="navbar-brand" asp-area="" asp-controller="Home" asp-action="Index">embeddingsearch</a>
                 <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target=".navbar-collapse" aria-controls="navbarSupportedContent"
@@ -29,16 +61,31 @@
                         @if (User.Identity?.IsAuthenticated == true)
                         {
                             <li class="nav-item">
-                                <a class="nav-link text-dark" asp-area="" asp-controller="Home" asp-action="Index">Home</a>
+                                <a class="nav-link text" asp-area="" asp-controller="Home" asp-action="Index">@T["Home"]</a>
                             </li>
                             <li class="nav-item">
-                                <a class="nav-link text-dark" asp-area="" asp-controller="Account" asp-action="Logout">Logout</a>
+                                <a class="nav-link text" asp-area="" asp-controller="Home" asp-action="Searchdomains">@T["Searchdomains"]</a>
+                            </li>
+                            @if (User.IsInRole("Admin") || User.IsInRole("Swagger"))
+                            {
+                                <li class="nav-item">
+                                    <a class="nav-link text" href="/swagger/index.html?ReturnUrl=@(currentUrl)">@T["Swagger"]</a>
+                                </li>
+                            }
+                            @if (User.IsInRole("Admin"))
+                            {
+                                <li class="nav-item">
+                                    <a class="nav-link text" href="/elmah?ReturnUrl=@(currentUrl)">@T["Elmah"]</a>
+                                </li>
+                            }
+                            <li class="nav-item">
+                                <a class="nav-link text" asp-area="" asp-controller="Account" asp-action="Logout">@T["Logout"]</a>
                             </li>
                         }
                         else
                         {
                             <li class="nav-item">
-                                <a class="nav-link text-dark" asp-area="" asp-controller="Account" asp-action="Login">Login</a>
+                                <a class="nav-link text" asp-area="" asp-controller="Account" asp-action="Login">@T["Login"]</a>
                             </li>
                         }
                     </ul>
@@ -54,12 +101,25 @@
 
     <footer class="border-top footer text-muted">
         <div class="container">
-            &copy; 2025 - embeddingsearch - <a asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>
+            &copy; 2025 - embeddingsearch
         </div>
     </footer>
-    <script src="~/lib/jquery/dist/jquery.min.js"></script>
+    <script src="~/lib/jquery/dist/jquery.min.js" defer></script>
-    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js"></script>
+    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js" defer></script>
-    <script src="~/js/site.js" asp-append-version="true"></script>
+    <script src="~/js/site.js" asp-append-version="true" defer></script>
     @await RenderSectionAsync("Scripts", required: false)
 </body>
 </html>
+<script>
+  const mediaQuery = window.matchMedia('(prefers-color-scheme: dark)');
+
+  function applyTheme(e) {
+    document.body.setAttribute(
+      'data-bs-theme',
+      e.matches ? 'dark' : 'light'
+    );
+  }
+
+  applyTheme(mediaQuery);
+  mediaQuery.addEventListener('change', applyTheme);
+</script>

src/Server/appsettings.Development.json

@@ -18,23 +18,33 @@
       "SQL": "server=localhost;database=embeddingsearch;uid=embeddingsearch;pwd=somepassword!;"
     },
     "Elmah": {
-      "AllowedHosts": [
+      "LogPath": "~/logs"
-        "127.0.0.1",
-        "::1",
-        "172.17.0.1"
-      ]
     },
+    "EmbeddingCacheMaxCount": 10000000,
     "AiProviders": {
       "ollama": {
         "handler": "ollama",
-        "baseURL": "http://localhost:11434"
+        "baseURL": "http://localhost:11434",
+        "Allowlist": [".*"],
+        "Denylist": ["qwen3-coder:latest", "qwen3:0.6b", "deepseek-v3.1:671b-cloud", "qwen3-vl", "deepseek-ocr"]
       },
       "localAI": {
         "handler": "openai",
         "baseURL": "http://localhost:8080",
-        "ApiKey": "Some API key here"
+        "ApiKey": "Some API key here",
+        "Allowlist": [".*"],
+        "Denylist": ["cross-encoder", "kitten-tts", "jina-reranker-v1-tiny-en", "whisper-small", "qwen3-vl-2b-instruct"]
       }
     },
+    "SimpleAuth": {
+      "Users": [
+        {
+          "Username": "admin",
+          "Password": "UnsafePractice.67",
+          "Roles": ["Admin"]
+        }
+      ]
+    },
     "ApiKeys": ["Some UUID here", "Another UUID here"],
     "UseHttpsRedirection": true
   }

src/Server/appsettings.json

@@ -16,14 +16,5 @@
         "Application": "Embeddingsearch.Server"
     }
   },
-  "EmbeddingsearchIndexer": {
-    "Elmah": {
-      "AllowedHosts": [
-        "127.0.0.1",
-        "::1"
-      ],
-      "LogFolder": "./logs"
-    }
-  },
   "AllowedHosts": "*"
 }

src/Server/wwwroot/css/site.css

@@ -49,3 +49,34 @@ body {
 .modal-title {
   font-size: 1.25rem;
 }
+
+/* Bootstrap icons */
+
+@font-face {
+  font-display: block;
+  font-family: "bootstrap-icons";
+  src: url("/fonts/bootstrap-icons.woff2") format("woff2"),
+url("/fonts/bootstrap-icons.woff") format("woff");
+}
+
+.bi::before,
+[class^="bi-"]::before,
+[class*=" bi-"]::before {
+  display: inline-block;
+  font-family: bootstrap-icons !important;
+  font-style: normal;
+  font-weight: normal !important;
+  font-variant: normal;
+  text-transform: none;
+  line-height: 1;
+  vertical-align: -.125em;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+.bi-info-circle-fill::before { content: "\f430"; }
+
+td.btn-group {
+  display: revert;
+  min-width: 15rem;
+}

src/Server/wwwroot/elmah-ui/custom.css

@@ -0,0 +1,56 @@
+.elmah-return-btn {
+    position: fixed;
+    top: 6px;
+    right: 24px;
+    z-index: 9999;
+
+    display: flex;
+    align-items: center;
+
+    height: 44px;
+    min-width: 44px;
+    padding: 0 14px;
+
+    background: #85ea2d;
+    color: black;
+    border-radius: 999px;
+    font-weight: 600;
+    text-decoration: none;
+    box-shadow: 0 4px 12px rgba(0,0,0,0.2);
+
+    overflow: hidden;
+    white-space: nowrap;
+
+    justify-content: center;
+    text-decoration: none !important;
+
+    transition:
+        top 0.25s ease,
+        background-color 0.2s ease;
+}
+
+/* hidden label */
+.elmah-return-btn::before {
+    content: "Return to Front-end";
+    font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+    max-width: 0;
+    opacity: 0;
+    transition:
+        max-width 0.3s ease,
+        opacity 0.2s ease;
+}
+
+/* expand on hover */
+.elmah-return-btn.show-label::before,
+.elmah-return-btn:hover::before {
+    max-width: 220px;
+    padding: 0.5rem;
+    opacity: 1;
+}
+
+/* hover colors */
+.elmah-return-btn.show-label, 
+.elmah-return-btn:hover {
+    background: #0b5ed7;
+    color: white;
+}

src/Server/wwwroot/elmah-ui/custom.js

@@ -0,0 +1,17 @@
+document.addEventListener('DOMContentLoaded', async () => {
+    const url = new URL(window.location.href);
+    const btn = document.createElement("a");
+    btn.href = url.searchParams.get('ReturnUrl') ?? "/";
+    btn.innerText = "⎋";
+    btn.setAttribute("aria-label", "Return to Front-End");
+    btn.className = "elmah-return-btn";
+
+    document.body.appendChild(btn);
+
+    const showLabelBriefly = () => {
+        btn.classList.add("show-label");
+        setTimeout(() => btn.classList.remove("show-label"), 2000);
+    };
+
+    setTimeout(showLabelBriefly, 1000);
+});

src/Server/wwwroot/swagger-ui/custom.css

@@ -0,0 +1,58 @@
+.swagger-return-btn {
+    position: fixed;
+    top: 6px;
+    left: 24px;
+    z-index: 9999;
+
+    display: flex;
+    align-items: center;
+
+    height: 44px;
+    min-width: 44px;
+    padding: 0 14px;
+
+    background: #85ea2d;
+    color: black;
+    border-radius: 999px;
+    font-weight: 600;
+    text-decoration: none;
+    box-shadow: 0 4px 12px rgba(0,0,0,0.2);
+
+    overflow: hidden;
+    white-space: nowrap;
+
+    justify-content: center;
+
+    transition:
+        top 0.25s ease,
+        background-color 0.2s ease;
+}
+
+/* hidden label */
+.swagger-return-btn::after {
+    content: "Return to Front-end";
+    font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+    max-width: 0;
+    opacity: 0;
+    transition:
+        max-width 0.3s ease,
+        opacity 0.2s ease;
+}
+
+/* expand on hover */
+.swagger-return-btn:hover::after {
+    max-width: 220px;
+    padding: 0.5rem;
+    opacity: 1;
+}
+
+/* hover colors */
+.swagger-return-btn:hover {
+    background: #0b5ed7;
+    color: white;
+}
+
+/* scrolled state */
+.swagger-return-btn.scrolled {
+    top: 24px;
+}

src/Server/wwwroot/swagger-ui/custom.js

@@ -0,0 +1,24 @@
+document.addEventListener('DOMContentLoaded', async () => {
+    const url = new URL(window.location.href);
+    const btn = document.createElement("a");
+    btn.href = url.searchParams.get('ReturnUrl') ?? "/";
+    btn.innerText = "⎋";
+    btn.setAttribute("aria-label", "Return to Front-End");
+    btn.className = "swagger-return-btn";
+
+    document.body.appendChild(btn);
+
+    const togglePosition = () => {
+        if (window.scrollY > 0) {
+            btn.classList.add("scrolled");
+        } else {
+            btn.classList.remove("scrolled");
+        }
+    };
+
+    // Initial state
+    togglePosition();
+
+    // On scroll
+    window.addEventListener("scroll", togglePosition, { passive: true });
+});

src/Shared/ApiKeyMiddleware.cs

@@ -1,38 +1,41 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Primitives;
+using Shared.Models;
 
 namespace Shared;
 
 public class ApiKeyMiddleware
 {
     private readonly RequestDelegate _next;
-    private readonly IConfiguration _configuration;
+    private readonly ApiKeyOptions _configuration;
 
-    public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration)
+    public ApiKeyMiddleware(RequestDelegate next, IOptions<ApiKeyOptions> configuration)
     {
         _next = next;
-        _configuration = configuration;
+        _configuration = configuration.Value;
     }
 
     public async Task InvokeAsync(HttpContext context)
     {
-        if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
+        if (!(context.User.Identity?.IsAuthenticated ?? false))
         {
-            context.Response.StatusCode = 401;
+            if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
-            await context.Response.WriteAsync("API Key is missing.");
+            {
-            return;
+                context.Response.StatusCode = 401;
-        }
+                await context.Response.WriteAsync("API Key is missing.");
+                return;
+            }
 
-        var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get<List<string>>();
+            string[]? validApiKeys = _configuration.ApiKeys;
-#pragma warning disable CS8604
+            if (validApiKeys == null || !validApiKeys.ToList().Contains(extractedApiKey))
-        if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be.
+            {
-        {
+                context.Response.StatusCode = 403;
-            context.Response.StatusCode = 403;
+                await context.Response.WriteAsync("Invalid API Key.");
-            await context.Response.WriteAsync("Invalid API Key.");
+                return;
-            return;
+            }
         }
-#pragma warning restore CS8604
 
         await _next(context);
     }

src/Shared/LRUCache.cs

@@ -0,0 +1,240 @@
+namespace Shared;
+
+public sealed class EnumerableLruCache<TKey, TValue> where TKey : notnull
+{
+    private sealed record CacheItem(TKey Key, TValue Value);
+
+    private readonly Dictionary<TKey, LinkedListNode<CacheItem>> _map;
+    private readonly LinkedList<CacheItem> _lruList;
+    private readonly ReaderWriterLockSlim _lock = new();
+
+    private int _capacity;
+
+    public EnumerableLruCache(int capacity)
+    {
+        if (capacity <= 0)
+            throw new ArgumentOutOfRangeException(nameof(capacity));
+
+        _capacity = capacity;
+        _map = new Dictionary<TKey, LinkedListNode<CacheItem>>(capacity);
+        _lruList = new LinkedList<CacheItem>();
+    }
+
+    public int Capacity
+    {
+        get
+        {
+            _lock.EnterReadLock();
+            try
+            {
+                return _capacity;
+            }
+            finally
+            {
+                _lock.ExitReadLock();
+            }
+        }
+        set
+        {
+            ArgumentOutOfRangeException.ThrowIfNegativeOrZero(value);
+
+            _lock.EnterWriteLock();
+            try
+            {
+                _capacity = value;
+                TrimIfNeeded();
+            }
+            finally
+            {
+                _lock.ExitWriteLock();
+            }
+        }
+    }
+
+    public int Count
+    {
+        get
+        {
+            _lock.EnterReadLock();
+            try
+            {
+                return _map.Count;
+            }
+            finally
+            {
+                _lock.ExitReadLock();
+            }
+        }
+    }
+
+    public TValue this[TKey key]
+    {
+        get
+        {
+            if (!TryGetValue(key, out var value))
+                throw new KeyNotFoundException();
+
+            return value!;
+        }
+        set => Set(key, value);
+    }
+
+    public bool TryGetValue(TKey key, out TValue? value)
+    {
+        _lock.EnterUpgradeableReadLock();
+        try
+        {
+            if (!_map.TryGetValue(key, out var node))
+            {
+                value = default;
+                return false;
+            }
+
+            value = node.Value.Value;
+
+            // LRU aktualisieren
+            _lock.EnterWriteLock();
+            try
+            {
+                _lruList.Remove(node);
+                _lruList.AddFirst(node);
+            }
+            finally
+            {
+                _lock.ExitWriteLock();
+            }
+
+            return true;
+        }
+        finally
+        {
+            _lock.ExitUpgradeableReadLock();
+        }
+    }
+
+    public void Set(TKey key, TValue value)
+    {
+        _lock.EnterWriteLock();
+        try
+        {
+            if (_map.TryGetValue(key, out var existing))
+            {
+                // Update + nach vorne
+                existing.Value = existing.Value with { Value = value };
+                _lruList.Remove(existing);
+                _lruList.AddFirst(existing);
+                return;
+            }
+
+            var item = new CacheItem(key, value);
+            var node = new LinkedListNode<CacheItem>(item);
+
+            _lruList.AddFirst(node);
+            _map[key] = node;
+
+            TrimIfNeeded();
+        }
+        finally
+        {
+            _lock.ExitWriteLock();
+        }
+    }
+
+    public bool Remove(TKey key)
+    {
+        _lock.EnterWriteLock();
+        try
+        {
+            if (!_map.TryGetValue(key, out var node))
+                return false;
+
+            _lruList.Remove(node);
+            _map.Remove(key);
+            return true;
+        }
+        finally
+        {
+            _lock.ExitWriteLock();
+        }
+    }
+
+    public bool ContainsKey(TKey key)
+    {
+        _lock.EnterReadLock();
+        try
+        {
+            return _map.ContainsKey(key);
+        }
+        finally
+        {
+            _lock.ExitReadLock();
+        }
+    }
+
+    public Dictionary<TKey, TValue> AsDictionary()
+    {
+        _lock.EnterReadLock();
+        try
+        {
+            return _map.Values.ToDictionary(
+                n => n.Value.Key,
+                n => n.Value.Value
+            );
+        }
+        finally
+        {
+            _lock.ExitReadLock();
+        }
+    }
+
+    public IEnumerable<KeyValuePair<TKey, TValue>> Items()
+    {
+        _lock.EnterReadLock();
+        try
+        {
+            foreach (var item in _lruList)
+            {
+                yield return new KeyValuePair<TKey, TValue>(item.Key, item.Value);
+            }
+        }
+        finally
+        {
+            _lock.ExitReadLock();
+        }
+    }
+
+    public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
+    {
+        List<KeyValuePair<TKey, TValue>> snapshot;
+
+        _lock.EnterReadLock();
+        try
+        {
+            snapshot = new List<KeyValuePair<TKey, TValue>>(_map.Count);
+
+            foreach (var item in _lruList)
+            {
+                snapshot.Add(new KeyValuePair<TKey, TValue>(
+                    item.Key,
+                    item.Value
+                ));
+            }
+        }
+        finally
+        {
+            _lock.ExitReadLock();
+        }
+
+        return snapshot.GetEnumerator();
+    }
+
+    private void TrimIfNeeded()
+    {
+        while (_map.Count > _capacity)
+        {
+            var lruNode = _lruList.Last!;
+            _lruList.RemoveLast();
+            _map.Remove(lruNode.Value.Key);
+        }
+    }
+}

src/Shared/Models/BaseModels.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace Shared.Models;
+
+public class SuccesMessageBaseModel
+{
+    [JsonPropertyName("Success")]
+    public required bool Success { get; set; }
+    [JsonPropertyName("Message")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Message { get; set; }
+}

src/Shared/Models/EntityResults.cs

@@ -3,14 +3,10 @@ using System.Text.Json.Serialization;
 namespace Shared.Models;
 
 
-public class EntityQueryResults
+public class EntityQueryResults : SuccesMessageBaseModel
 {
     [JsonPropertyName("Results")]
     public required List<EntityQueryResult> Results { get; set; }
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
 }
 
 public class EntityQueryResult
@@ -19,20 +15,19 @@ public class EntityQueryResult
     public required string Name { get; set; }
     [JsonPropertyName("Value")]
     public float Value { get; set; }
+    [JsonPropertyName("Attributes")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public Dictionary<string, string>? Attributes { get; set; }
 }
 
-public class EntityIndexResult
+public class EntityIndexResult : SuccesMessageBaseModel {}
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-}
 
 public class EntityListResults
 {
     [JsonPropertyName("Results")]
     public required List<EntityListResult> Results { get; set; }
+    [JsonPropertyName("Message")]
+    public string? Message { get; set; }
     [JsonPropertyName("Success")]
     public required bool Success { get; set; }
 }
@@ -77,11 +72,5 @@ public class EmbeddingResult
     public required float[] Embeddings { get; set; }
 }
 
-public class EntityDeleteResults
+public class EntityDeleteResults : SuccesMessageBaseModel {}
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-}
 

src/Shared/Models/JSONModels.cs

@@ -3,7 +3,7 @@ namespace Shared.Models;
 public class JSONEntity
 {
     public required string Name { get; set; }
-    public required string Probmethod { get; set; }
+    public required ProbMethodEnum Probmethod { get; set; }
     public required string Searchdomain { get; set; }
     public required Dictionary<string, string> Attributes { get; set; }
     public required JSONDatapoint[] Datapoints { get; set; }
@@ -13,7 +13,27 @@ public class JSONDatapoint
 {
     public required string Name { get; set; }
     public required string? Text { get; set; }
-    public required string Probmethod_embedding { get; set; }
+    public required ProbMethodEnum Probmethod_embedding { get; set; }
-    public required string SimilarityMethod { get; set; }
+    public required SimilarityMethodEnum SimilarityMethod { get; set; }
     public required string[] Model { get; set; }
 }
+
+public enum ProbMethodEnum
+{
+    Mean,
+    HarmonicMean,
+    QuadraticMean,
+    GeometricMean,
+    EVEWAvg,
+    HVEWAvg,
+    LVEWAvg,
+    DictionaryWeightedAverage
+}
+
+public enum SimilarityMethodEnum
+{
+    Cosine,
+    Euclidian,
+    Manhattan,
+    Pearson
+}

src/Shared/Models/OptionModels.cs

@@ -0,0 +1,13 @@
+namespace Shared.Models;
+
+public class ApiKeyOptions
+{
+    public string[]? ApiKeys { get; set; }
+}
+
+public class ServerOptions
+{
+    public required string BaseUri { get; set; }
+    public string? ApiKey { get; set; }
+    public string? Searchdomain { get; set; }
+}

src/Shared/Models/SearchdomainModels.cs

@@ -95,13 +95,15 @@ public struct DateTimedSearchResult(DateTime dateTime, List<ResultItem> results)
     }
 }
 
-public struct SearchdomainSettings(bool cacheReconciliation = false)
+public struct SearchdomainSettings(bool cacheReconciliation = false, int queryCacheSize = 1_000_000)
 {
     [JsonPropertyName("CacheReconciliation")]
     public bool CacheReconciliation { get; set; } = cacheReconciliation;
+    [JsonPropertyName("QueryCacheSize")]
+    public int QueryCacheSize { get; set; } = queryCacheSize;
 }
 
-internal static class MemorySizes
+public static class MemorySizes
 {
     public static readonly int PointerSize = IntPtr.Size;
     public static readonly int ObjectHeader = PointerSize * 2;

src/Shared/Models/SearchdomainResults.cs

@@ -1,4 +1,5 @@
 using System.Text.Json.Serialization;
+using Shared;
 
 namespace Shared.Models;
 
@@ -11,109 +12,50 @@ public class SearchdomainListResults
     public string? Message { get; set; }
 }
 
-public class SearchdomainCreateResults
+public class SearchdomainCreateResults : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("Id")]
     public int? Id { get; set; }
 }
 
-public class SearchdomainUpdateResults
+public class SearchdomainUpdateResults : SuccesMessageBaseModel {}
+
+public class SearchdomainDeleteResults : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-}
-
-public class SearchdomainDeleteResults
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("DeletedEntities")]
     public required int DeletedEntities { get; set; }
 }
 
-public class SearchdomainSearchesResults
+public class SearchdomainQueriesResults : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
     [JsonPropertyName("Searches")]
     public required Dictionary<string, DateTimedSearchResult> Searches { get; set; }
 }
 
-public class SearchdomainDeleteSearchResult
+public class SearchdomainDeleteSearchResult : SuccesMessageBaseModel {}
+
+public class SearchdomainUpdateSearchResult : SuccesMessageBaseModel {}
+
+public class SearchdomainSettingsResults : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }    
-}
-
-public class SearchdomainUpdateSearchResult
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }    
-}
-
-public class SearchdomainSettingsResults
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("Settings")]
     public required SearchdomainSettings? Settings { get; set; }
 }
 
-public class SearchdomainSearchCacheSizeResults
+public class SearchdomainQueryCacheSizeResults : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
+    [JsonPropertyName("ElementCount")]
-    public required bool Success { get; set; }
+    public required int? ElementCount { get; set; }
-
+    [JsonPropertyName("ElementMaxCount")]
-    [JsonPropertyName("Message")]
+    public required int? ElementMaxCount { get; set; }
-    public string? Message { get; set; }
+    [JsonPropertyName("SizeBytes")]
-
+    public required long? SizeBytes { get; set; }
-    [JsonPropertyName("SearchCacheSizeBytes")]
-    public required long? SearchCacheSizeBytes { get; set; }
 }
 
-public class SearchdomainInvalidateCacheResults
+public class SearchdomainInvalidateCacheResults : SuccesMessageBaseModel {}
+
+public class SearchdomainGetDatabaseSizeResult : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-}
-
-public class SearchdomainGetDatabaseSizeResult
-{
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("SearchdomainDatabaseSizeBytes")]
     public required long? SearchdomainDatabaseSizeBytes { get; set; }    
 }

src/Shared/Models/ServerModels.cs

@@ -2,14 +2,30 @@ using System.Text.Json.Serialization;
 
 namespace Shared.Models;
 
-public class ServerGetModelsResult
+public class ServerGetModelsResult : SuccesMessageBaseModel
 {
-    [JsonPropertyName("Success")]
-    public required bool Success { get; set; }
-
-    [JsonPropertyName("Message")]
-    public string? Message { get; set; }
-
     [JsonPropertyName("Models")]
     public string[]? Models { get; set; }    
 }
+
+public class ServerGetStatsResult : SuccesMessageBaseModel
+{
+    [JsonPropertyName("EmbeddingCacheUtilization")]
+    public long? EmbeddingCacheUtilization { get; set; }
+    [JsonPropertyName("EmbeddingCacheMaxElementCount")]
+    public long? EmbeddingCacheMaxElementCount { get; set; }
+    [JsonPropertyName("ElementCount")]
+    public long? EmbeddingCacheElementCount { get; set; }
+    [JsonPropertyName("EmbeddingsCount")]
+    public long? EmbeddingsCount { get; set; }
+    [JsonPropertyName("EntityCount")]
+    public long? EntityCount { get; set; }
+    [JsonPropertyName("QueryCacheElementCount")]
+    public long? QueryCacheElementCount { get; set; }
+    [JsonPropertyName("QueryCacheMaxElementCountAll")]
+    public long? QueryCacheMaxElementCountAll { get; set; }
+    [JsonPropertyName("QueryCacheMaxElementCountLoadedSearchdomainsOnly")]
+    public long? QueryCacheMaxElementCountLoadedSearchdomainsOnly { get; set; }
+    [JsonPropertyName("QueryCacheUtilization")]
+    public long? QueryCacheUtilization { get; set; }
+}