Merge pull request #71 from LD-Reborn/67-improve-fcp-by-defering-js-and-css

Added CriticalCSS, defered CSS and JS, fixed heading order, fixed fro…

.gitignore

@@ -18,3 +18,5 @@ src/Server/logs
 src/Shared/bin
 src/Shared/obj
 src/Server/wwwroot/logs/*
+src/Server/CriticalCSS/node_modules
+src/Server/CriticalCSS/package*.json

src/Client/Client.cs

@@ -8,6 +8,8 @@ using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Configuration;
 using System.Reflection.Metadata.Ecma335;
 using Shared.Models;
+using System.Net;
+using Microsoft.Extensions.Options;
 
 namespace Client;
 
@@ -24,12 +26,12 @@ public class Client
             this.searchdomain = searchdomain;
         }
 
-        public Client(IConfiguration configuration)
+        public Client(IOptions<ServerOptions> configuration)
         {
-            string? baseUri = configuration.GetSection("Embeddingsearch").GetValue<string>("BaseUri");
-            string? apiKey = configuration.GetSection("Embeddingsearch").GetValue<string>("ApiKey");
-            string? searchdomain = configuration.GetSection("Embeddingsearch").GetValue<string>("Searchdomain");
-            this.baseUri = baseUri ?? "";
+            string baseUri = configuration.Value.BaseUri;
+            string? apiKey = configuration.Value.ApiKey;
+            string? searchdomain = configuration.Value.Searchdomain;
+            this.baseUri = baseUri;
             this.apiKey = apiKey ?? "";
             this.searchdomain = searchdomain ?? "";
         }
@@ -41,8 +43,8 @@ public class Client
 
         public async Task<EntityListResults> EntityListAsync(string searchdomain, bool returnEmbeddings = false)
         {
-            var url = $"{baseUri}/Entities?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
-            return await GetUrlAndProcessJson<EntityListResults>(url);
+            var url = $"{baseUri}/Entities?searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
+            return await FetchUrlAndProcessJson<EntityListResults>(HttpMethod.Get, url);
         }
 
         public async Task<EntityIndexResult> EntityIndexAsync(List<JSONEntity> jsonEntity)
@@ -53,7 +55,7 @@ public class Client
         public async Task<EntityIndexResult> EntityIndexAsync(string jsonEntity)
         {
             var content = new StringContent(jsonEntity, Encoding.UTF8, "application/json");
-            return await PutUrlAndProcessJson<EntityIndexResult>(GetUrl($"{baseUri}", "Entities", apiKey, []), content);
+            return await FetchUrlAndProcessJson<EntityIndexResult>(HttpMethod.Put, GetUrl($"{baseUri}", "Entities", []), content);
         }
 
         public async Task<EntityDeleteResults> EntityDeleteAsync(string entityName)
@@ -64,12 +66,12 @@ public class Client
         public async Task<EntityDeleteResults> EntityDeleteAsync(string searchdomain, string entityName)
         {
             var url = $"{baseUri}/Entity?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&entity={HttpUtility.UrlEncode(entityName)}";
-            return await DeleteUrlAndProcessJson<EntityDeleteResults>(url);
+            return await FetchUrlAndProcessJson<EntityDeleteResults>(HttpMethod.Delete, url);
         }
 
         public async Task<SearchdomainListResults> SearchdomainListAsync()
         {
-            return await GetUrlAndProcessJson<SearchdomainListResults>(GetUrl($"{baseUri}", "Searchdomains", apiKey, []));
+            return await FetchUrlAndProcessJson<SearchdomainListResults>(HttpMethod.Get, GetUrl($"{baseUri}", "Searchdomains", []));
         }
 
         public async Task<SearchdomainCreateResults> SearchdomainCreateAsync()
@@ -79,7 +81,7 @@ public class Client
 
         public async Task<SearchdomainCreateResults> SearchdomainCreateAsync(string searchdomain, SearchdomainSettings searchdomainSettings = new())
         {
-            return await PostUrlAndProcessJson<SearchdomainCreateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainCreateResults>(HttpMethod.Post, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
             {
                 {"searchdomain", searchdomain}
             }), new StringContent(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json"));
@@ -92,7 +94,7 @@ public class Client
 
         public async Task<SearchdomainDeleteResults> SearchdomainDeleteAsync(string searchdomain)
         {
-            return await DeleteUrlAndProcessJson<SearchdomainDeleteResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainDeleteResults>(HttpMethod.Delete, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
             {
                 {"searchdomain", searchdomain}
             }));
@@ -112,7 +114,7 @@ public class Client
 
         public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string searchdomain, string newName, string settings = "{}")
         {
-            return await PutUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
             {
                 {"searchdomain", searchdomain},
                 {"newName", newName}
@@ -125,7 +127,7 @@ public class Client
             {
                 {"searchdomain", searchdomain}
             };
-            return await GetUrlAndProcessJson<SearchdomainSearchesResults>(GetUrl($"{baseUri}/Searchdomain", "Queries", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainSearchesResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Queries", parameters));
         }
 
         public async Task<EntityQueryResults> SearchdomainQueryAsync(string query)
@@ -143,7 +145,7 @@ public class Client
             if (topN is not null) parameters.Add("topN", ((int)topN).ToString());
             if (returnAttributes) parameters.Add("returnAttributes", returnAttributes.ToString());
 
-            return await PostUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters), null);
+            return await FetchUrlAndProcessJson<EntityQueryResults>(HttpMethod.Post, GetUrl($"{baseUri}/Searchdomain", "Query", parameters), null);
         }
 
         public async Task<SearchdomainDeleteSearchResult> SearchdomainDeleteQueryAsync(string searchdomain, string query)
@@ -153,7 +155,7 @@ public class Client
                 {"searchdomain", searchdomain},
                 {"query", query}
             };
-            return await DeleteUrlAndProcessJson<SearchdomainDeleteSearchResult>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainDeleteSearchResult>(HttpMethod.Delete, GetUrl($"{baseUri}/Searchdomain", "Query", parameters));
         }
 
         public async Task<SearchdomainUpdateSearchResult> SearchdomainUpdateQueryAsync(string searchdomain, string query, List<ResultItem> results)
@@ -163,8 +165,9 @@ public class Client
                 {"searchdomain", searchdomain},
                 {"query", query}
             };
-            return await PatchUrlAndProcessJson<SearchdomainUpdateSearchResult>(
-                GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters),
+            return await FetchUrlAndProcessJson<SearchdomainUpdateSearchResult>(
+                HttpMethod.Patch,
+                GetUrl($"{baseUri}/Searchdomain", "Query", parameters),
                 new StringContent(JsonSerializer.Serialize(results), Encoding.UTF8, "application/json"));
         }
 
@@ -174,7 +177,7 @@ public class Client
             {
                 {"searchdomain", searchdomain}
             };
-            return await GetUrlAndProcessJson<SearchdomainSettingsResults>(GetUrl($"{baseUri}/Searchdomain", "Settings", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainSettingsResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters));
         }
 
         public async Task<SearchdomainUpdateResults> SearchdomainUpdateSettingsAsync(string searchdomain, SearchdomainSettings searchdomainSettings)
@@ -184,7 +187,7 @@ public class Client
                 {"searchdomain", searchdomain}
             };
             StringContent content = new(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json");
-            return await PutUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}/Searchdomain", "Settings", apiKey, parameters), content);
+            return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters), content);
         }
 
         public async Task<SearchdomainSearchCacheSizeResults> SearchdomainGetQueryCacheSizeAsync(string searchdomain)
@@ -193,7 +196,7 @@ public class Client
             {
                 {"searchdomain", searchdomain}
             };
-            return await GetUrlAndProcessJson<SearchdomainSearchCacheSizeResults>(GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainSearchCacheSizeResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", parameters));
         }
 
         public async Task<SearchdomainInvalidateCacheResults> SearchdomainClearQueryCache(string searchdomain)
@@ -202,7 +205,7 @@ public class Client
             {
                 {"searchdomain", searchdomain}
             };
-            return await PostUrlAndProcessJson<SearchdomainInvalidateCacheResults>(GetUrl($"{baseUri}/Searchdomain/QueryCache", "Clear", apiKey, parameters), null);
+            return await FetchUrlAndProcessJson<SearchdomainInvalidateCacheResults>(HttpMethod.Post, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Clear", parameters), null);
         }
 
         public async Task<SearchdomainGetDatabaseSizeResult> SearchdomainGetDatabaseSizeAsync(string searchdomain)
@@ -211,74 +214,40 @@ public class Client
             {
                 {"searchdomain", searchdomain}
             };
-            return await GetUrlAndProcessJson<SearchdomainGetDatabaseSizeResult>(GetUrl($"{baseUri}/Searchdomain/Database", "Size", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainGetDatabaseSizeResult>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/Database", "Size", parameters));
         }
 
         public async Task<ServerGetModelsResult> ServerGetModelsAsync()
         {
-            return await GetUrlAndProcessJson<ServerGetModelsResult>(GetUrl($"{baseUri}/Server", "Models", apiKey, []));
+            return await FetchUrlAndProcessJson<ServerGetModelsResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server", "Models", []));
         }
 
         public async Task<ServerGetEmbeddingCacheSizeResult> ServerGetEmbeddingCacheSizeAsync()
         {
-            return await GetUrlAndProcessJson<ServerGetEmbeddingCacheSizeResult>(GetUrl($"{baseUri}/Server/EmbeddingCache", "Size", apiKey, []));
+            return await FetchUrlAndProcessJson<ServerGetEmbeddingCacheSizeResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server/EmbeddingCache", "Size", []));
         }
 
-        private static async Task<T> GetUrlAndProcessJson<T>(string url)
+        private async Task<T> FetchUrlAndProcessJson<T>(HttpMethod httpMethod, string url, HttpContent? content = null)
         {
+            HttpRequestMessage requestMessage = new(httpMethod, url)
+            {
+                Content = content,
+            };
+            requestMessage.Headers.Add("X-API-KEY", apiKey);
             using var client = new HttpClient();
-            var response = await client.GetAsync(url);
+            var response = await client.SendAsync(requestMessage);
             string responseContent = await response.Content.ReadAsStringAsync();
+            if (response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Unauthorized) throw new UnauthorizedAccessException(responseContent); // TODO implement distinct exceptions
+            if (response.StatusCode == HttpStatusCode.InternalServerError) throw new Exception($"Request was unsuccessful due to an internal server error: {responseContent}"); // TODO implement proper InternalServerErrorException
             var result = JsonSerializer.Deserialize<T>(responseContent)
                 ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
             return result;
         }
 
-        private static async Task<T> PostUrlAndProcessJson<T>(string url, HttpContent? content)
-        {
-            using var client = new HttpClient();
-            var response = await client.PostAsync(url, content);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        private static async Task<T> PutUrlAndProcessJson<T>(string url, HttpContent content)
-        {
-            using var client = new HttpClient();
-            var response = await client.PutAsync(url, content);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        private static async Task<T> PatchUrlAndProcessJson<T>(string url, HttpContent content)
-        {
-            using var client = new HttpClient();
-            var response = await client.PatchAsync(url, content);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        private static async Task<T> DeleteUrlAndProcessJson<T>(string url)
-        {
-            using var client = new HttpClient();
-            var response = await client.DeleteAsync(url);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        public static string GetUrl(string baseUri, string endpoint, string apiKey, Dictionary<string, string> parameters)
+        public static string GetUrl(string baseUri, string endpoint, Dictionary<string, string> parameters)
         {
             var uriBuilder = new UriBuilder($"{baseUri}/{endpoint}");
             var query = HttpUtility.ParseQueryString(uriBuilder.Query);
-            if (apiKey.Length > 0) query["apiKey"] = apiKey;
             foreach (var param in parameters)
             {
                 query[param.Key] = param.Value;

src/Indexer/Indexer.csproj

@@ -15,6 +15,7 @@
     <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
     <PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
+    <PackageReference Include="System.Configuration.ConfigurationManager" Version="9.0.3" />
     <PackageReference Include="Python" Version="3.13.3" />
     <PackageReference Include="Pythonnet" Version="3.0.5" />
   </ItemGroup>

src/Indexer/Models/OptionModels.cs

@@ -0,0 +1,9 @@
+using Shared.Models;
+namespace Indexer.Models;
+
+public class IndexerOptions : ApiKeyOptions
+{
+    public required WorkerConfig[] Workers { get; set; }
+    public required ServerOptions Server { get; set;}
+    public required string PythonRuntime { get; set; } = "libpython3.13.so";
+}

src/Indexer/Models/ScriptModels.cs

@@ -15,11 +15,11 @@ public class ScriptToolSet
     public Client.Client Client;
     public LoggerWrapper Logger;
     public ICallbackInfos? CallbackInfos;
-    public IConfiguration Configuration;
+    public IndexerOptions Configuration;
     public CancellationToken CancellationToken;
     public string Name;
 
-    public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IConfiguration configuration, CancellationToken cancellationToken, string name)
+    public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IndexerOptions configuration, CancellationToken cancellationToken, string name)
     {
         Configuration = configuration;
         Name = name;

src/Indexer/Program.cs

@@ -6,6 +6,8 @@ using ElmahCore.Mvc;
 using ElmahCore.Mvc.Logger;
 using Serilog;
 using Quartz;
+using System.Configuration;
+using Shared.Models;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -21,6 +23,12 @@ Log.Logger = new LoggerConfiguration()
 builder.Logging.AddSerilog();
 builder.Services.AddHttpContextAccessor();
 builder.Services.AddSingleton<IConfigurationRoot>(builder.Configuration);
+
+IConfigurationSection configurationSection = builder.Configuration.GetSection("Indexer");
+IndexerOptions configuration = configurationSection.Get<IndexerOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
+builder.Services.Configure<IndexerOptions>(configurationSection);
+builder.Services.Configure<ServerOptions>(configurationSection.GetSection("Server"));
+builder.Services.Configure<ApiKeyOptions>(configurationSection);
 builder.Services.AddSingleton<Client.Client>();
 builder.Services.AddSingleton<WorkerManager>();
 builder.Services.AddHostedService<IndexerService>();

src/Indexer/ScriptContainers/PythonScriptContainer.cs

@@ -15,11 +15,8 @@ public class PythonScriptable : IScriptContainer
     public ILogger _logger { get; set; }
     public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
     {
-        string? runtime = toolSet.Configuration.GetValue<string>("EmbeddingsearchIndexer:PythonRuntime");
-        if (runtime is not null)
-        {
-            Runtime.PythonDLL ??= runtime;
-        }
+        string runtime = toolSet.Configuration.PythonRuntime;
+        Runtime.PythonDLL ??= runtime;
         _logger = logger;
         SourceLoaded = false;
         if (!PythonEngine.IsInitialized)

src/Indexer/WorkerManager.cs

@@ -1,21 +1,22 @@
 using Indexer.Exceptions;
 using Indexer.Models;
 using Indexer.ScriptContainers;
+using Microsoft.Extensions.Options;
 
 public class WorkerManager
 {
     public Dictionary<string, Worker> Workers;
     public List<Type> types;
     private readonly ILogger<WorkerManager> _logger;
-    private readonly IConfiguration _configuration;
+    private readonly IndexerOptions _configuration;
     private readonly Client.Client client;
 
-    public WorkerManager(ILogger<WorkerManager> logger, IConfiguration configuration, Client.Client client)
+    public WorkerManager(ILogger<WorkerManager> logger, IOptions<IndexerOptions> configuration, Client.Client client)
     {
         Workers = [];
         types = [typeof(PythonScriptable), typeof(CSharpScriptable)];
         _logger = logger;
-        _configuration = configuration;
+        _configuration = configuration.Value;
         this.client = client;
     }
 
@@ -23,27 +24,12 @@ public class WorkerManager
     {
         _logger.LogInformation("Initializing workers");
         // Load and configure all workers
-        var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer");
-        if (!sectionMain.Exists())
-        {
-            _logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
-            throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
-        }
 
-        WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
-        if (sectionWorker is not null)
+        foreach (WorkerConfig workerConfig in _configuration.Workers)
         {
-            foreach (WorkerConfig workerConfig in sectionWorker.Worker)
-            {
-                CancellationTokenSource cancellationTokenSource = new();
-                ScriptToolSet toolSet = new(workerConfig.Script, client, _logger, _configuration, cancellationTokenSource.Token, workerConfig.Name);
-                InitializeWorker(toolSet, workerConfig, cancellationTokenSource);
-            }
-        }
-        else
-        {
-            _logger.LogCritical("Unable to load section \"Worker\"");
-            throw new IndexerConfigurationException("Unable to load section \"Worker\"");
+            CancellationTokenSource cancellationTokenSource = new();
+            ScriptToolSet toolSet = new(workerConfig.Script, client, _logger, _configuration, cancellationTokenSource.Token, workerConfig.Name);
+            InitializeWorker(toolSet, workerConfig, cancellationTokenSource);
         }
         _logger.LogInformation("Initialized workers");
     }

src/Indexer/appsettings.Development.json

@@ -5,46 +5,23 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
-  "Embeddingsearch": {
-    "BaseUri": "http://localhost:5146"
-  },
-  "EmbeddingsearchIndexer": {
-    "Worker":
-    [
+  "Indexer": {
+    "Workers": [
       {
         "Name": "pythonExample",
         "Script": "Scripts/example.py",
-        "Calls": [
-          {
-            "Name": "intervalCall",
-            "Type": "interval",
-            "Interval": 30000
-          }
-        ]
-      },
-      {
-        "Name": "csharpExample",
-        "Script": "Scripts/example.csx",
         "Calls": [
           {
             "Name": "runonceCall",
             "Type": "runonce"
-          },
-          {
-            "Name": "scheduleCall",
-            "Type": "schedule",
-            "Schedule": "0 0/5 * * * ?"
-          },
-          {
-            "Name": "fileupdateCall",
-            "Type": "fileupdate",
-            "Path": "./Scripts/example_content",
-            "Events": ["Created", "Changed", "Deleted", "Renamed"],
-            "Filters": ["*.md", "*.txt"],
-            "IncludeSubdirectories": true
           }
         ]
       }
-    ]
+    ],
+    "ApiKeys": ["xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"],
+    "Server": {
+      "BaseUri": "http://localhost:5146",
+      "ApiKey": "yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy"
+    }
   }
 }

src/Indexer/appsettings.json

@@ -24,7 +24,7 @@
       ],
       "LogFolder": "./logs"
     },
-    "PythonRuntime": "libpython3.12.so"
+    "PythonRuntime": "libpython3.13.so"
   },
   "AllowedHosts": "*"
 }

src/Server/AIProvider.cs

@@ -1,24 +1,25 @@
 
 using System.Text;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Server.Exceptions;
+using Server.Models;
 
 namespace Server;
 
 public class AIProvider
 {
     private readonly ILogger<AIProvider> _logger;
-    private readonly IConfiguration _configuration;
-    public AIProvidersConfiguration aIProvidersConfiguration;
+    private readonly EmbeddingSearchOptions _configuration;
+    public Dictionary<string, AiProvider> aIProvidersConfiguration;
 
-    public AIProvider(ILogger<AIProvider> logger, IConfiguration configuration)
+    public AIProvider(ILogger<AIProvider> logger, IOptions<EmbeddingSearchOptions> configuration)
     {
         _logger = logger;
-        _configuration = configuration;
-        AIProvidersConfiguration? retrievedAiProvidersConfiguration = _configuration
-            .GetSection("Embeddingsearch")
-            .Get<AIProvidersConfiguration>();
+        _configuration = configuration.Value;
+        Dictionary<string, AiProvider>? retrievedAiProvidersConfiguration = _configuration.AiProviders;
         if (retrievedAiProvidersConfiguration is null)
         {
             _logger.LogCritical("Unable to build AIProvidersConfiguration. Please check your configuration.");
@@ -35,8 +36,8 @@ public class AIProvider
         Uri uri = new(modelUri);
         string provider = uri.Scheme;
         string model = uri.AbsolutePath;
-        AIProviderConfiguration? aIProvider = aIProvidersConfiguration.AiProviders
-            .FirstOrDefault(x => String.Equals(x.Key.ToLower(), provider.ToLower()))
+        AiProvider? aIProvider = aIProvidersConfiguration
+            .FirstOrDefault(x => string.Equals(x.Key.ToLower(), provider.ToLower()))
             .Value;
         if (aIProvider is null)
         {
@@ -119,12 +120,12 @@ public class AIProvider
 
     public string[] GetModels()
     {
-        var aIProviders = aIProvidersConfiguration.AiProviders;
+        var aIProviders = aIProvidersConfiguration;
         List<string> results = [];
-        foreach (KeyValuePair<string, AIProviderConfiguration> aIProviderKV in aIProviders)
+        foreach (KeyValuePair<string, AiProvider> aIProviderKV in aIProviders)
         {
             string aIProviderName = aIProviderKV.Key;
-            AIProviderConfiguration aIProvider = aIProviderKV.Value;
+            AiProvider aIProvider = aIProviderKV.Value;
 
             using var httpClient = new HttpClient();
 
@@ -178,7 +179,12 @@ public class AIProvider
                 foreach (string? result in aIProviderResult)
                 {
                     if (result is null) continue;
-                    results.Add(aIProviderName + ":" + result);
+                    bool isInAllowList = ElementMatchesAnyRegexInList(result, aIProvider.Allowlist);
+                    bool isInDenyList =  ElementMatchesAnyRegexInList(result, aIProvider.Denylist);
+                    if (isInAllowList && !isInDenyList)
+                    {
+                        results.Add(aIProviderName + ":" + result);
+                    }
                 }
             }
             catch (Exception ex)
@@ -189,6 +195,11 @@ public class AIProvider
         }
         return [.. results];
     }
+
+    private static bool ElementMatchesAnyRegexInList(string element, string[] list)
+    {
+        return list?.Any(pattern => pattern != null && Regex.IsMatch(element, pattern)) ?? false;
+    }
 }
 
 public class AIProvidersConfiguration

src/Server/Controllers/AccountController.cs

@@ -12,9 +12,9 @@ public class AccountController : Controller
 {
     private readonly SimpleAuthOptions _options;
 
-    public AccountController(IOptions<SimpleAuthOptions> options)
+    public AccountController(IOptions<EmbeddingSearchOptions> options)
     {
-        _options = options.Value;
+        _options = options.Value.SimpleAuth;
     }
 
     [HttpGet("Login")]

src/Server/Controllers/Frontend/HomeController.cs

@@ -20,8 +20,14 @@ public class HomeController : Controller
         _domainManager = domainManager;
     }
 
-    [Authorize]
     [HttpGet("/")]
+    public IActionResult Root()
+    {
+        return Redirect("/Home/Index");
+    }
+
+    [Authorize]
+    [HttpGet("Index")]
     public IActionResult Index()
     {
         return View();

src/Server/CriticalCSS/CriticalCSSGenerator.js

@@ -0,0 +1,129 @@
+import { generate } from 'critical';
+import fs from 'fs';
+import path from 'path';
+
+import puppeteer from 'puppeteer';
+
+const browser = await puppeteer.launch();
+const page = await browser.newPage();
+
+// Login
+await page.goto('http://localhost:5146/Account/Login');
+await page.type('#username', 'admin');
+await page.type('#password', 'UnsafePractice.67');
+await page.click('button[type=submit]');
+await page.waitForNavigation();
+
+// Extract cookies
+const cookies = await page.cookies();
+await browser.close();
+
+async function generateCriticalCSSForViews() {
+  const viewsDir = '../Views';
+  
+  // Helper function to get all .cshtml files recursively
+  function getAllCshtmlFiles(dir) {
+    let results = [];
+    const list = fs.readdirSync(dir);
+    
+    list.forEach(file => {
+      const filePath = path.join(dir, file);
+      const stat = fs.statSync(filePath);
+      console.log("DEBUG@2");
+      console.log(filePath);
+      if (stat && stat.isDirectory()) {
+        // Recursively get files from subdirectories
+        results = results.concat(getAllCshtmlFiles(filePath));
+      } else if (file.endsWith('.cshtml') && filePath.search("/_") == -1) {
+        results.push(filePath);
+      }
+    });
+    
+    return results;
+  }
+  
+  // Helper function to convert file path to URL path
+  function filePathToUrlPath(filePath) {
+    // Remove 'Views/' prefix
+    let relativePath = filePath.replace(/^Views[\/\\]/, '');
+    
+    // Remove .cshtml extension
+    relativePath = relativePath.replace(/\.cshtml$/, '');
+    
+    // Convert to URL format (replace \ with / and capitalize first letter)
+    const urlPath = relativePath
+      .split(/[\/\\]/)
+      .map((segment, index) => 
+        index === 0 ? segment : segment.charAt(0).toUpperCase() + segment.slice(1)
+      )
+      .join('/');
+    
+    // Handle the case where we have a single file (like Index.cshtml)
+    if (relativePath.includes('/')) {
+      // Convert to URL path format: Views/Home/Index.cshtml -> /Home/Index
+      return '/' + relativePath.replace(/\\/g, '/').replace(/\.cshtml$/, '');
+    } else {
+      // For files directly in Views folder (like Views/Index.cshtml)
+      return '/' + relativePath.replace(/\.cshtml$/, '');
+    }
+  }
+  
+  // Get all .cshtml files
+  const cshtmlFiles = getAllCshtmlFiles(viewsDir);
+  const criticalCssDir = '.';
+//   if (!fs.existsSync(criticalCssDir)) {
+//     fs.mkdirSync(criticalCssDir, { recursive: true });
+//   }
+  
+  // Process each file
+  for (const file of cshtmlFiles) {
+    try {
+      const urlPath = filePathToUrlPath(file).replace("../", "").replace("/Views", "");
+      
+      // Generate critical CSS
+      await generate({
+        src: `http://localhost:5146${urlPath}`,
+        inline: false,
+        width: 1920,
+        height: 1080,
+        penthouse: {
+          customHeaders: {
+            cookie: cookies.map(c => `${c.name}=${c.value}`).join('; ')
+          },
+          forceExclude: ['.btn'], // Otherwise buttons end up colorless and .btn overrides other classes like .btn-warning, etc. - so it has to be force-excluded here and re-added later
+          forceInclude: [
+            '[data-bs-theme=dark]',
+            '.navbar',
+            '.col-md-4',
+            '.visually-hidden', // visually hidden headings
+            '.bi-info-circle-fill', '.text-info', // info icon
+            '.container', '.col-md-6', '.row', '.g-4', '.row>*',
+            'p', '.fs-3', '.py-4', // title
+            '.mb-4',
+            '.card', '.card-body', '.p-2', // card
+            'h2', '.card-title', '.fs-5', // card - title
+            '.d-flex', '.justify-content-between', '.mt-2', // card - content
+            '.progress', '.mt-3', // card - progress bar
+            '.list-group', '.list-group-flush', '.list-group-item', '.list-group-flush>.list-group-item', '.list-group-flush>.list-group-item:last-child', '.badge', '.bg-warning', '.bg-success', '.h-100', // card - health check list
+            '.btn', '.btn-sm', '.btn-primary', '.btn-warning', '.btn-danger', // Searchdomains buttons
+            '.col-md-8', '.sidebar',
+            '.mb-0', '.mb-2', '.align-items-center',
+            'h3', '.col-md-3', '.col-md-2', '.text-nowrap', '.overflow-auto'
+          ]
+        },
+        target: {
+          css: path.join(criticalCssDir, urlPath.replace(/\//g, '.').replace(/^\./, '').replace("...", "") + '.css')
+        }
+      });
+      
+      console.log(`Critical CSS generated for: ${urlPath}`);
+    } catch (err) {
+      console.error(`Error processing ${file}:`, err);
+    }
+  }
+  
+  console.log('All critical CSS files generated!');
+}
+
+// Run the function
+generateCriticalCSSForViews().catch(console.error);

src/Server/CriticalCSS/README.md

@@ -0,0 +1,10 @@
+# How to use CriticalCSS
+1. Install it here
+```bash
+npm i -D critical
+npm install puppeteer
+```
+2. Run the css generator:
+```bash
+node CriticalCSSGenerator.js
+```

src/Server/Helper/SearchdomainHelper.cs

@@ -218,6 +218,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             {
                 searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(preexistingEntity);
             }
+            searchdomain.UpdateModelsInUse();
             return preexistingEntity;
         }
         else
@@ -243,6 +244,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             };
             entityCache.Add(entity);
             searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(entity);
+            searchdomain.UpdateModelsInUse();
             return entity;
         }
     }

src/Server/Models/Auth.cs

@@ -1,13 +0,0 @@
-namespace Server.Models;
-
-public class SimpleAuthOptions
-{
-    public List<SimpleUser> Users { get; set; } = new();
-}
-
-public class SimpleUser
-{
-    public string Username { get; set; } = "";
-    public string Password { get; set; } = "";
-    public string[] Roles { get; set; } = Array.Empty<string>();
-}

src/Server/Models/ConfigModels.cs

@@ -0,0 +1,36 @@
+using System.Configuration;
+using ElmahCore;
+using Shared.Models;
+
+namespace Server.Models;
+
+public class EmbeddingSearchOptions : ApiKeyOptions
+{
+    public required ConnectionStringsSection ConnectionStrings { get; set; }
+    public ElmahOptions? Elmah { get; set; }
+    public required long EmbeddingCacheMaxCount { get; set; }
+    public required Dictionary<string, AiProvider> AiProviders { get; set; }
+    public required SimpleAuthOptions SimpleAuth { get; set; }
+    public required bool UseHttpsRedirection { get; set; }
+}
+
+public class AiProvider
+{
+    public required string Handler { get; set; }
+    public required string BaseURL { get; set; }
+    public string? ApiKey { get; set; }
+    public required string[] Allowlist { get; set; }
+    public required string[] Denylist { get; set; }
+}
+
+public class SimpleAuthOptions
+{
+    public List<SimpleUser> Users { get; set; } = [];
+}
+
+public class SimpleUser
+{
+    public string Username { get; set; } = "";
+    public string Password { get; set; } = "";
+    public string[] Roles { get; set; } = [];
+}

src/Server/Program.cs

@@ -10,11 +10,14 @@ using Server.Models;
 using Server.Services;
 using System.Text.Json.Serialization;
 using System.Reflection;
+using System.Configuration;
+using Microsoft.OpenApi.Models;
+using Shared.Models;
+using Microsoft.AspNetCore.ResponseCompression;
 
 var builder = WebApplication.CreateBuilder(args);
 
-// Add services to the container.
-
+// Add Controllers with views & string conversion for enums
 builder.Services.AddControllersWithViews()
     .AddJsonOptions(options =>
     {
@@ -23,6 +26,13 @@ builder.Services.AddControllersWithViews()
         );
     });
 
+// Add Configuration
+IConfigurationSection configurationSection = builder.Configuration.GetSection("Embeddingsearch");
+EmbeddingSearchOptions configuration = configurationSection.Get<EmbeddingSearchOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
+
+builder.Services.Configure<EmbeddingSearchOptions>(configurationSection);
+builder.Services.Configure<ApiKeyOptions>(configurationSection);
+
 // Add Localization
 builder.Services.AddLocalization(options => options.ResourcesPath = "Resources");
 builder.Services.Configure<RequestLocalizationOptions>(options =>
@@ -43,6 +53,31 @@ builder.Services.AddSwaggerGen(c =>
     var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml";
     var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile);
     c.IncludeXmlComments(xmlPath);
+    if (configuration.ApiKeys is not null)
+    {
+        c.AddSecurityDefinition("ApiKey", new OpenApiSecurityScheme
+        {
+            Description = "ApiKey must appear in header",
+            Type = SecuritySchemeType.ApiKey,
+            Name = "X-API-KEY",
+            In = ParameterLocation.Header,
+            Scheme = "ApiKeyScheme"
+        });
+        var key = new OpenApiSecurityScheme()
+        {
+            Reference = new OpenApiReference
+            {
+                Type = ReferenceType.SecurityScheme,
+                Id = "ApiKey"
+            },
+            In = ParameterLocation.Header
+        };
+        var requirement = new OpenApiSecurityRequirement
+        {
+            { key, []}
+        };
+        c.AddSecurityRequirement(requirement);
+    }
 });
 Log.Logger = new LoggerConfiguration()
     .ReadFrom.Configuration(builder.Configuration)
@@ -58,7 +93,12 @@ builder.Services.AddHealthChecks()
 
 builder.Services.AddElmah<XmlFileErrorLog>(Options =>
 {
-    Options.LogPath = builder.Configuration.GetValue<string>("Embeddingsearch:Elmah:LogFolder") ?? "~/logs";
+    Options.OnPermissionCheck = context =>
+        context.User.Claims.Any(claim =>
+            claim.Value.Equals("Admin", StringComparison.OrdinalIgnoreCase)
+            || claim.Value.Equals("Elmah", StringComparison.OrdinalIgnoreCase)
+    );
+    Options.LogPath = configuration.Elmah?.LogPath ?? "~/logs";
 });
 
 builder.Services
@@ -76,35 +116,29 @@ builder.Services.AddAuthorization(options =>
         policy => policy.RequireRole("Admin"));
 });
 
-IConfigurationSection simpleAuthSection = builder.Configuration.GetSection("Embeddingsearch:SimpleAuth");
-if (simpleAuthSection.Exists()) builder.Services.Configure<SimpleAuthOptions>(simpleAuthSection);
+builder.Services.AddResponseCompression(options =>
+{
+    options.EnableForHttps = true;
+    options.Providers.Add<GzipCompressionProvider>();
+    options.Providers.Add<BrotliCompressionProvider>();
+    options.MimeTypes =
+    [
+        "text/plain",
+        "text/css",
+        "application/javascript",
+        "text/javascript",
+        "text/html",
+        "application/xml",
+        "text/xml",
+        "application/json",
+        "image/svg+xml"
+    ];
+});
 
 var app = builder.Build();
 
-List<string>? allowedIps = builder.Configuration.GetSection("Embeddingsearch:Elmah:AllowedHosts")
-    .Get<List<string>>();
-
-app.Use(async (context, next) =>
-{
-    bool requestIsElmah = context.Request.Path.StartsWithSegments("/elmah");
-    bool requestIsSwagger = context.Request.Path.StartsWithSegments("/swagger");
-    
-    if (requestIsElmah || requestIsSwagger)
-    {
-        var remoteIp = context.Connection.RemoteIpAddress?.ToString();
-        bool blockRequest = allowedIps is null
-            || remoteIp is null
-            || !allowedIps.Contains(remoteIp);
-        if (blockRequest)
-        {
-            context.Response.StatusCode = 403;
-            await context.Response.WriteAsync("Forbidden");
-            return;
-        }
-    }
-
-    await next();
-});
+app.UseAuthentication();
+app.UseAuthorization();
 
 app.UseElmah();
 
@@ -120,21 +154,53 @@ app.MapHealthChecks("/healthz/AIProvider", new Microsoft.AspNetCore.Diagnostics.
 });
 
 bool IsDevelopment = app.Environment.IsDevelopment();
-bool useSwagger = app.Configuration.GetValue<bool>("UseSwagger");
-bool? UseMiddleware = app.Configuration.GetValue<bool?>("UseMiddleware");
 
-// Configure the HTTP request pipeline.
-if (IsDevelopment || useSwagger)
+app.Use(async (context, next) =>
 {
-    app.UseSwagger();
-    app.UseSwaggerUI();
-    //app.UseElmahExceptionPage(); // Messes with JSON response for API calls. Leaving this here so I don't accidentally put this in again later on.
-}
-if (UseMiddleware == true && !IsDevelopment)
+    if (context.Request.Path.StartsWithSegments("/swagger"))
+    {
+        if (!context.User.Identity?.IsAuthenticated ?? true)
+        {
+            context.Response.Redirect("/Account/Login");
+            return;
+        }
+
+        if (!context.User.IsInRole("Admin"))
+        {
+            context.Response.StatusCode = StatusCodes.Status403Forbidden;
+            return;
+        }
+    }
+
+    await next();
+});
+
+app.UseSwagger();
+app.UseSwaggerUI(options =>
 {
-    app.UseMiddleware<Shared.ApiKeyMiddleware>();
+    options.EnablePersistAuthorization();
+});
+//app.UseElmahExceptionPage(); // Messes with JSON response for API calls. Leaving this here so I don't accidentally put this in again later on.
+
+if (configuration.ApiKeys is not null)
+{
+    app.UseWhen(context =>
+    {
+        RouteData routeData = context.GetRouteData();
+        string controllerName = routeData.Values["controller"]?.ToString() ?? "StaticFile";
+        if (controllerName == "Account" || controllerName == "Home" || controllerName == "StaticFile")
+        {
+            return false;
+        }
+        return true;
+    }, appBuilder =>
+    {
+        appBuilder.UseMiddleware<Shared.ApiKeyMiddleware>();    
+    });
 }
 
+app.UseResponseCompression();
+
 // Add localization
 var supportedCultures = new[] { "de", "de-DE", "en-US" };
 var localizationOptions = new RequestLocalizationOptions()
@@ -143,10 +209,23 @@ var localizationOptions = new RequestLocalizationOptions()
     .AddSupportedUICultures(supportedCultures);
 app.UseRequestLocalization(localizationOptions);
 
-app.UseAuthentication();
-app.UseAuthorization();
-
 app.MapControllers();
-app.UseStaticFiles();
+app.UseStaticFiles(new StaticFileOptions
+{
+    OnPrepareResponse = ctx =>
+    {
+        string requestPath = ctx.Context.Request.Path.ToString();
+        string[] cachedSuffixes = [".css", ".js", ".png", ".ico", ".woff2"];
+        if (cachedSuffixes.Any(suffix => requestPath.EndsWith(suffix)))
+        {
+            ctx.Context.Response.GetTypedHeaders().CacheControl =
+                new Microsoft.Net.Http.Headers.CacheControlHeaderValue()
+                {
+                    Public = true,
+                    MaxAge = TimeSpan.FromDays(365)
+                };
+        }
+    }
+});
 
 app.Run();

src/Server/Searchdomain.cs

@@ -216,6 +216,11 @@ public class Searchdomain
         return queryEmbeddings;
     }
 
+    public void UpdateModelsInUse()
+    {
+        modelsInUse = GetModels([.. entityCache]);
+    }
+
     private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary<string, float[]> queryEmbeddings)
     {
         List<(string, float)> datapointProbs = [];
@@ -237,16 +242,19 @@ public class Searchdomain
     public static List<string> GetModels(List<Entity> entities)
     {
         List<string> result = [];
-        foreach (Entity entity in entities)
+        lock (entities)
         {
-            foreach (Datapoint datapoint in entity.datapoints)
+            foreach (Entity entity in entities)
             {
-                foreach ((string, float[]) tuple in datapoint.embeddings)
+                foreach (Datapoint datapoint in entity.datapoints)
                 {
-                    string model = tuple.Item1;
-                    if (!result.Contains(model))
+                    foreach ((string, float[]) tuple in datapoint.embeddings)
                     {
-                        result.Add(model);
+                        string model = tuple.Item1;
+                        if (!result.Contains(model))
+                        {
+                            result.Add(model);
+                        }
                     }
                 }
             }

src/Server/Views/Home/Index.cshtml

@@ -18,9 +18,10 @@
 }
 
 <div class="container py-4">
-    <h3 class="mb-4">
+    <h1 class="visually-hidden">Searchdomains</h1>
+    <p class="mb-4 fs-3">
         @(hasName ? T["Hi, {0}!", name] : T["Hi!"])
-    </h3>
+    </p>
 
     <div class="row g-4">
 
@@ -28,7 +29,7 @@
         <div class="col-md-6">
             <div class="card shadow-sm h-100">
                 <div class="card-body">
-                    <h5 class="card-title">@T["Embedding Cache"]</h5>
+                    <h2 class="card-title fs-5">@T["Embedding Cache"]</h2>
 
                     <div class="d-flex justify-content-between">
                         <span>@T["Size"]</span>
@@ -62,7 +63,7 @@
         <div class="col-md-6">
             <div class="card shadow-sm h-100">
                 <div class="card-body">
-                    <h5 class="card-title">@T["Health Checks"]</h5>
+                    <h2 class="card-title fs-5">@T["Health Checks"]</h2>
 
                     <ul class="list-group list-group-flush">
                         <li class="list-group-item d-flex justify-content-between">
@@ -88,7 +89,7 @@
         <div class="col-md-6">
             <div class="card shadow-sm h-100">
                 <div class="card-body">
-                    <h5 class="card-title">@T["Searchdomains"]</h5>
+                    <h2 class="card-title fs-5">@T["Searchdomains"]</h2>
 
                     <div class="d-flex justify-content-between">
                         <span>@T["Count"]</span>
@@ -111,7 +112,7 @@
     </div>
 </div>
 
-<script defer>
+<script>
     var searchdomains = null;
 
     document.addEventListener('DOMContentLoaded', async () => {
@@ -138,41 +139,55 @@
         let healthchecksServer = document.getElementById("healthchecksServer");
         let healthchecksAiProvider = document.getElementById("healthchecksAiProvider");
 
-        listSearchdomains().then(async result => {
-            searchdomains = result.Searchdomains;
-            hideThrobber(searchdomainCount);
-            searchdomainCount.textContent = searchdomains.length;
+        (async() => {
+            listSearchdomains().then(async result => {
+                searchdomains = result.Searchdomains;
+                hideThrobber(searchdomainCount);
+                searchdomainCount.textContent = searchdomains.length;
 
-            var entityCount = 0;
-            var totalUtilization = 0;
-            for (var name in searchdomains){
-                let entityListResult = await listEntities(searchdomains[name]);
-                let entities = entityListResult.Results;
-                entityCount += entities.length;
-                let querycacheUtilizationResult = await getQuerycacheUtilization(searchdomains[name]);
-                let utilization = querycacheUtilizationResult.QueryCacheSizeBytes;
-                totalUtilization += utilization;
-            }
-            hideThrobber(searchdomainEntityCount);
-            hideThrobber(totalQuerycacheUtilization);
-            searchdomainEntityCount.textContent = entityCount;
-            totalQuerycacheUtilization.textContent = NumberOfBytesAsHumanReadable(totalUtilization);
-        });
-        getEmbeddingcacheUtilization().then(result => {
-            let utilization = result.SizeInBytes;
-            let maxElementCount = result.MaxElementCount;
-            let elementCount = result.ElementCount;
-            let embeddingCount = result.EmbeddingsCount;
-            hideThrobber(embeddingcacheSize);
-            embeddingcacheSize.textContent = NumberOfBytesAsHumanReadable(utilization);
-            hideThrobber(embeddingcacheElementCount);
-            embeddingcacheElementCount.textContent = `${elementCount.toLocaleString()} / ${maxElementCount.toLocaleString()}`;
-            hideThrobber(embeddingcacheEmbeddingCount);
-            embeddingcacheEmbeddingCount.textContent = embeddingCount;
-            embeddingcacheElementCountProgressBar.style.width = `${elementCount / maxElementCount * 100}%`;
-        });
-        getHealthCheckStatusAndApply(healthchecksServer, "/healthz/Database");
-        getHealthCheckStatusAndApply(healthchecksAiProvider, "/healthz/AIProvider");
+                const perDomainPromises = searchdomains.map(async domain => {
+                    const [entityListResult, querycacheUtilizationResult] = await Promise.all([
+                        listEntities(domain),
+                        getQuerycacheUtilization(domain)
+                    ]);
+
+                    return {
+                        entityCount: entityListResult.Results.length,
+                        utilization: querycacheUtilizationResult.QueryCacheSizeBytes
+                    };
+                });
+
+                const results = await Promise.all(perDomainPromises);
+
+                let entityCount = 0;
+                let totalUtilization = 0;
+
+                for (const r of results) {
+                    entityCount += r.entityCount;
+                    totalUtilization += r.utilization;
+                }
+
+                hideThrobber(searchdomainEntityCount);
+                hideThrobber(totalQuerycacheUtilization);
+                searchdomainEntityCount.textContent = entityCount;
+                totalQuerycacheUtilization.textContent = NumberOfBytesAsHumanReadable(totalUtilization);
+            });
+            getEmbeddingcacheUtilization().then(result => {
+                let utilization = result.SizeInBytes;
+                let maxElementCount = result.MaxElementCount;
+                let elementCount = result.ElementCount;
+                let embeddingCount = result.EmbeddingsCount;
+                hideThrobber(embeddingcacheSize);
+                embeddingcacheSize.textContent = NumberOfBytesAsHumanReadable(utilization);
+                hideThrobber(embeddingcacheElementCount);
+                embeddingcacheElementCount.textContent = `${elementCount.toLocaleString()} / ${maxElementCount.toLocaleString()}`;
+                hideThrobber(embeddingcacheEmbeddingCount);
+                embeddingcacheEmbeddingCount.textContent = embeddingCount;
+                embeddingcacheElementCountProgressBar.style.width = `${elementCount / maxElementCount * 100}%`;
+            });
+            getHealthCheckStatusAndApply(healthchecksServer, "/healthz/Database");
+            getHealthCheckStatusAndApply(healthchecksAiProvider, "/healthz/AIProvider");
+        })();
     });
 
 

src/Server/Views/Home/Searchdomains.cshtml

@@ -21,7 +21,7 @@
 }
 
 <div class="container-fluid mt-4">
-  <h1 class="visually-hidden">embeddingsearch</h1>
+  <h1 class="visually-hidden">Searchdomains</h1>
   <div class="row">
 
     <!-- Sidebar -->
@@ -804,7 +804,7 @@
             .getElementById('cacheClear')
             .addEventListener('click', () => {
                 const domainKey = getSelectedDomainKey();
-                fetch(`/Searchdomain/SearchCache/Clear?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
+                fetch(`/Searchdomain/QueryCache/Clear?searchdomain=${encodeURIComponent(domains[domainKey])}`, {
                     method: 'POST'
                 }).then(response => {
                     if (response.ok) {
@@ -1031,7 +1031,7 @@
     }
 
     function getSearchdomainCacheUtilization(domainKey) {
-        return fetch(`/Searchdomain/SearchCache/Size?searchdomain=${encodeURIComponent(domains[domainKey])}`)
+        return fetch(`/Searchdomain/QueryCache/Size?searchdomain=${encodeURIComponent(domains[domainKey])}`)
             .then(r => r.json());
     }
 
@@ -1106,10 +1106,10 @@
             }
         });
         cacheUtilizationPromise.then(cacheUtilization => {
-            if (cacheUtilization != null && cacheUtilization.SearchCacheSizeBytes != null)
+            if (cacheUtilization != null && cacheUtilization.QueryCacheSizeBytes != null)
             {
                 document.querySelector('#cacheUtilization').innerText =
-                    `${NumberOfBytesAsHumanReadable(cacheUtilization.SearchCacheSizeBytes)}`;
+                    `${NumberOfBytesAsHumanReadable(cacheUtilization.QueryCacheSizeBytes)}`;
             } else {
                 showToast("@T["Unable to fetch searchdomain cache utilization"]", "danger");
                 console.error('Failed to fetch searchdomain cache utilization');

src/Server/Views/Shared/_Layout.cshtml

@@ -1,15 +1,39 @@
-@using Server.Services
+@using System.Globalization
+@using Server.Services
 @inject LocalizationService T
 
 <!DOCTYPE html>
-<html lang="en">
+<html lang="@CultureInfo.CurrentUICulture.TwoLetterISOLanguageName">
 <head>
     <meta charset="utf-8" />
+    <meta name="description" content="Embeddingsearch server" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>@ViewData["Title"] - embeddingsearch</title>
-    <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.1/font/bootstrap-icons.css">
-    <link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
+    @if (!Context.Request.Query.ContainsKey("renderRaw"))
+    {
+        <link rel="preload" href="~/lib/bootstrap/dist/css/bootstrap.min.css" as="style"/>
+        <link rel="stylesheet" fetchpriority="high"
+        href="~/lib/bootstrap/dist/css/bootstrap.min.css"
+        media="print"
+        onload="this.media='all'">
+    }
+    <style>
+        @Html.Raw(File.ReadAllText(System.IO.Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "css", "site.css")))
+    </style>
+    @if (!Context.Request.Query.ContainsKey("noCriticalCSS"))
+    {
+        <style>
+            @if (Context.Request.Path.Value is not null)
+            {
+                string path = System.IO.Path.Combine("CriticalCSS", Context.Request.Path.Value.Trim('/').Replace("/", ".") + ".css");
+                Console.WriteLine(path);
+                if (File.Exists(path))
+                {
+                    @Html.Raw(File.ReadAllText(path));
+                }
+            }
+        </style>
+    }
     <script>
         window.appTranslations = {
             closeAlert: '@T["Close alert"]'
@@ -61,9 +85,9 @@
             &copy; 2025 - embeddingsearch
         </div>
     </footer>
-    <script src="~/lib/jquery/dist/jquery.min.js"></script>
-    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js"></script>
-    <script src="~/js/site.js" asp-append-version="true"></script>
+    <script src="~/lib/jquery/dist/jquery.min.js" defer></script>
+    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js" defer></script>
+    <script src="~/js/site.js" asp-append-version="true" defer></script>
     @await RenderSectionAsync("Scripts", required: false)
 </body>
 </html>

src/Server/appsettings.Development.json

@@ -18,22 +18,22 @@
       "SQL": "server=localhost;database=embeddingsearch;uid=embeddingsearch;pwd=somepassword!;"
     },
     "Elmah": {
-      "AllowedHosts": [
-        "127.0.0.1",
-        "::1",
-        "172.17.0.1"
-      ]
+      "LogPath": "~/logs"
     },
     "EmbeddingCacheMaxCount": 10000000,
     "AiProviders": {
       "ollama": {
         "handler": "ollama",
-        "baseURL": "http://localhost:11434"
+        "baseURL": "http://localhost:11434",
+        "Allowlist": [".*"],
+        "Denylist": ["qwen3-coder:latest", "qwen3:0.6b", "deepseek-v3.1:671b-cloud", "qwen3-vl", "deepseek-ocr"]
       },
       "localAI": {
         "handler": "openai",
         "baseURL": "http://localhost:8080",
-        "ApiKey": "Some API key here"
+        "ApiKey": "Some API key here",
+        "Allowlist": [".*"],
+        "Denylist": ["cross-encoder", "kitten-tts", "jina-reranker-v1-tiny-en", "whisper-small", "qwen3-vl-2b-instruct"]
       }
     },
     "SimpleAuth": {

src/Server/appsettings.json

@@ -16,14 +16,5 @@
         "Application": "Embeddingsearch.Server"
     }
   },
-  "EmbeddingsearchIndexer": {
-    "Elmah": {
-      "AllowedHosts": [
-        "127.0.0.1",
-        "::1"
-      ],
-      "LogFolder": "./logs"
-    }
-  },
   "AllowedHosts": "*"
 }

src/Server/wwwroot/css/site.css

@@ -49,3 +49,29 @@ body {
 .modal-title {
   font-size: 1.25rem;
 }
+
+/* Bootstrap icons */
+
+@font-face {
+  font-display: block;
+  font-family: "bootstrap-icons";
+  src: url("/fonts/bootstrap-icons.woff2") format("woff2"),
+url("/fonts/bootstrap-icons.woff") format("woff");
+}
+
+.bi::before,
+[class^="bi-"]::before,
+[class*=" bi-"]::before {
+  display: inline-block;
+  font-family: bootstrap-icons !important;
+  font-style: normal;
+  font-weight: normal !important;
+  font-variant: normal;
+  text-transform: none;
+  line-height: 1;
+  vertical-align: -.125em;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+.bi-info-circle-fill::before { content: "\f430"; }

src/Shared/ApiKeyMiddleware.cs

@@ -1,38 +1,41 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Primitives;
+using Shared.Models;
 
 namespace Shared;
 
 public class ApiKeyMiddleware
 {
     private readonly RequestDelegate _next;
-    private readonly IConfiguration _configuration;
+    private readonly ApiKeyOptions _configuration;
 
-    public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration)
+    public ApiKeyMiddleware(RequestDelegate next, IOptions<ApiKeyOptions> configuration)
     {
         _next = next;
-        _configuration = configuration;
+        _configuration = configuration.Value;
     }
 
     public async Task InvokeAsync(HttpContext context)
     {
-        if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
+        if (!(context.User.Identity?.IsAuthenticated ?? false))
         {
-            context.Response.StatusCode = 401;
-            await context.Response.WriteAsync("API Key is missing.");
-            return;
-        }
+            if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
+            {
+                context.Response.StatusCode = 401;
+                await context.Response.WriteAsync("API Key is missing.");
+                return;
+            }
 
-        var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get<List<string>>();
-#pragma warning disable CS8604
-        if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be.
-        {
-            context.Response.StatusCode = 403;
-            await context.Response.WriteAsync("Invalid API Key.");
-            return;
+            string[]? validApiKeys = _configuration.ApiKeys;
+            if (validApiKeys == null || !validApiKeys.ToList().Contains(extractedApiKey))
+            {
+                context.Response.StatusCode = 403;
+                await context.Response.WriteAsync("Invalid API Key.");
+                return;
+            }
         }
-#pragma warning restore CS8604
 
         await _next(context);
     }

src/Shared/Models/OptionModels.cs

@@ -0,0 +1,13 @@
+namespace Shared.Models;
+
+public class ApiKeyOptions
+{
+    public string[]? ApiKeys { get; set; }
+}
+
+public class ServerOptions
+{
+    public required string BaseUri { get; set; }
+    public string? ApiKey { get; set; }
+    public string? Searchdomain { get; set; }
+}