Merge pull request #69 from LD-Reborn/61-add-a-model-allow-denylist

61 add a model allow denylist
2025-12-31 03:58:47 +01:00
parent b5db4bc1e4 7dfe945a48
commit eafc764f73
18 changed files with 269 additions and 207 deletions
--- a/src/Client/Client.cs
+++ b/src/Client/Client.cs
@@ -8,6 +8,8 @@ using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Configuration;
 using System.Reflection.Metadata.Ecma335;
 using Shared.Models;
+using System.Net;
+using Microsoft.Extensions.Options;

 namespace Client;

@@ -24,12 +26,12 @@ public class Client
            this.searchdomain = searchdomain;
        }

-        public Client(IConfiguration configuration)
+        public Client(IOptions<ServerOptions> configuration)
        {
-            string? baseUri = configuration.GetSection("Embeddingsearch").GetValue<string>("BaseUri");
-            string? apiKey = configuration.GetSection("Embeddingsearch").GetValue<string>("ApiKey");
-            string? searchdomain = configuration.GetSection("Embeddingsearch").GetValue<string>("Searchdomain");
-            this.baseUri = baseUri ?? "";
+            string baseUri = configuration.Value.BaseUri;
+            string? apiKey = configuration.Value.ApiKey;
+            string? searchdomain = configuration.Value.Searchdomain;
+            this.baseUri = baseUri;
            this.apiKey = apiKey ?? "";
            this.searchdomain = searchdomain ?? "";
        }
@@ -41,8 +43,8 @@ public class Client

        public async Task<EntityListResults> EntityListAsync(string searchdomain, bool returnEmbeddings = false)
        {
-            var url = $"{baseUri}/Entities?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
-            return await GetUrlAndProcessJson<EntityListResults>(url);
+            var url = $"{baseUri}/Entities?searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
+            return await FetchUrlAndProcessJson<EntityListResults>(HttpMethod.Get, url);
        }

        public async Task<EntityIndexResult> EntityIndexAsync(List<JSONEntity> jsonEntity)
@@ -53,7 +55,7 @@ public class Client
        public async Task<EntityIndexResult> EntityIndexAsync(string jsonEntity)
        {
            var content = new StringContent(jsonEntity, Encoding.UTF8, "application/json");
-            return await PutUrlAndProcessJson<EntityIndexResult>(GetUrl($"{baseUri}", "Entities", apiKey, []), content);
+            return await FetchUrlAndProcessJson<EntityIndexResult>(HttpMethod.Put, GetUrl($"{baseUri}", "Entities", []), content);
        }

        public async Task<EntityDeleteResults> EntityDeleteAsync(string entityName)
@@ -64,12 +66,12 @@ public class Client
        public async Task<EntityDeleteResults> EntityDeleteAsync(string searchdomain, string entityName)
        {
            var url = $"{baseUri}/Entity?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&entity={HttpUtility.UrlEncode(entityName)}";
-            return await DeleteUrlAndProcessJson<EntityDeleteResults>(url);
+            return await FetchUrlAndProcessJson<EntityDeleteResults>(HttpMethod.Delete, url);
        }

        public async Task<SearchdomainListResults> SearchdomainListAsync()
        {
-            return await GetUrlAndProcessJson<SearchdomainListResults>(GetUrl($"{baseUri}", "Searchdomains", apiKey, []));
+            return await FetchUrlAndProcessJson<SearchdomainListResults>(HttpMethod.Get, GetUrl($"{baseUri}", "Searchdomains", []));
        }

        public async Task<SearchdomainCreateResults> SearchdomainCreateAsync()
@@ -79,7 +81,7 @@ public class Client

        public async Task<SearchdomainCreateResults> SearchdomainCreateAsync(string searchdomain, SearchdomainSettings searchdomainSettings = new())
        {
-            return await PostUrlAndProcessJson<SearchdomainCreateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainCreateResults>(HttpMethod.Post, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
            {
                {"searchdomain", searchdomain}
            }), new StringContent(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json"));
@@ -92,7 +94,7 @@ public class Client

        public async Task<SearchdomainDeleteResults> SearchdomainDeleteAsync(string searchdomain)
        {
-            return await DeleteUrlAndProcessJson<SearchdomainDeleteResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainDeleteResults>(HttpMethod.Delete, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
            {
                {"searchdomain", searchdomain}
            }));
@@ -112,7 +114,7 @@ public class Client

        public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string searchdomain, string newName, string settings = "{}")
        {
-            return await PutUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
            {
                {"searchdomain", searchdomain},
                {"newName", newName}
@@ -125,7 +127,7 @@ public class Client
            {
                {"searchdomain", searchdomain}
            };
-            return await GetUrlAndProcessJson<SearchdomainSearchesResults>(GetUrl($"{baseUri}/Searchdomain", "Queries", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainSearchesResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Queries", parameters));
        }

        public async Task<EntityQueryResults> SearchdomainQueryAsync(string query)
@@ -143,7 +145,7 @@ public class Client
            if (topN is not null) parameters.Add("topN", ((int)topN).ToString());
            if (returnAttributes) parameters.Add("returnAttributes", returnAttributes.ToString());

-            return await PostUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters), null);
+            return await FetchUrlAndProcessJson<EntityQueryResults>(HttpMethod.Post, GetUrl($"{baseUri}/Searchdomain", "Query", parameters), null);
        }

        public async Task<SearchdomainDeleteSearchResult> SearchdomainDeleteQueryAsync(string searchdomain, string query)
@@ -153,7 +155,7 @@ public class Client
                {"searchdomain", searchdomain},
                {"query", query}
            };
-            return await DeleteUrlAndProcessJson<SearchdomainDeleteSearchResult>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainDeleteSearchResult>(HttpMethod.Delete, GetUrl($"{baseUri}/Searchdomain", "Query", parameters));
        }

        public async Task<SearchdomainUpdateSearchResult> SearchdomainUpdateQueryAsync(string searchdomain, string query, List<ResultItem> results)
@@ -163,8 +165,9 @@ public class Client
                {"searchdomain", searchdomain},
                {"query", query}
            };
-            return await PatchUrlAndProcessJson<SearchdomainUpdateSearchResult>(
-                GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters),
+            return await FetchUrlAndProcessJson<SearchdomainUpdateSearchResult>(
+                HttpMethod.Patch,
+                GetUrl($"{baseUri}/Searchdomain", "Query", parameters),
                new StringContent(JsonSerializer.Serialize(results), Encoding.UTF8, "application/json"));
        }

@@ -174,7 +177,7 @@ public class Client
            {
                {"searchdomain", searchdomain}
            };
-            return await GetUrlAndProcessJson<SearchdomainSettingsResults>(GetUrl($"{baseUri}/Searchdomain", "Settings", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainSettingsResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters));
        }

        public async Task<SearchdomainUpdateResults> SearchdomainUpdateSettingsAsync(string searchdomain, SearchdomainSettings searchdomainSettings)
@@ -184,7 +187,7 @@ public class Client
                {"searchdomain", searchdomain}
            };
            StringContent content = new(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json");
-            return await PutUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}/Searchdomain", "Settings", apiKey, parameters), content);
+            return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters), content);
        }

        public async Task<SearchdomainSearchCacheSizeResults> SearchdomainGetQueryCacheSizeAsync(string searchdomain)
@@ -193,7 +196,7 @@ public class Client
            {
                {"searchdomain", searchdomain}
            };
-            return await GetUrlAndProcessJson<SearchdomainSearchCacheSizeResults>(GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainSearchCacheSizeResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", parameters));
        }

        public async Task<SearchdomainInvalidateCacheResults> SearchdomainClearQueryCache(string searchdomain)
@@ -202,7 +205,7 @@ public class Client
            {
                {"searchdomain", searchdomain}
            };
-            return await PostUrlAndProcessJson<SearchdomainInvalidateCacheResults>(GetUrl($"{baseUri}/Searchdomain/QueryCache", "Clear", apiKey, parameters), null);
+            return await FetchUrlAndProcessJson<SearchdomainInvalidateCacheResults>(HttpMethod.Post, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Clear", parameters), null);
        }

        public async Task<SearchdomainGetDatabaseSizeResult> SearchdomainGetDatabaseSizeAsync(string searchdomain)
@@ -211,74 +214,40 @@ public class Client
            {
                {"searchdomain", searchdomain}
            };
-            return await GetUrlAndProcessJson<SearchdomainGetDatabaseSizeResult>(GetUrl($"{baseUri}/Searchdomain/Database", "Size", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainGetDatabaseSizeResult>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/Database", "Size", parameters));
        }

        public async Task<ServerGetModelsResult> ServerGetModelsAsync()
        {
-            return await GetUrlAndProcessJson<ServerGetModelsResult>(GetUrl($"{baseUri}/Server", "Models", apiKey, []));
+            return await FetchUrlAndProcessJson<ServerGetModelsResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server", "Models", []));
        }

        public async Task<ServerGetEmbeddingCacheSizeResult> ServerGetEmbeddingCacheSizeAsync()
        {
-            return await GetUrlAndProcessJson<ServerGetEmbeddingCacheSizeResult>(GetUrl($"{baseUri}/Server/EmbeddingCache", "Size", apiKey, []));
+            return await FetchUrlAndProcessJson<ServerGetEmbeddingCacheSizeResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server/EmbeddingCache", "Size", []));
        }

-        private static async Task<T> GetUrlAndProcessJson<T>(string url)
+        private async Task<T> FetchUrlAndProcessJson<T>(HttpMethod httpMethod, string url, HttpContent? content = null)
        {
+            HttpRequestMessage requestMessage = new(httpMethod, url)
+            {
+                Content = content,
+            };
+            requestMessage.Headers.Add("X-API-KEY", apiKey);
            using var client = new HttpClient();
-            var response = await client.GetAsync(url);
+            var response = await client.SendAsync(requestMessage);
            string responseContent = await response.Content.ReadAsStringAsync();
+            if (response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Unauthorized) throw new UnauthorizedAccessException(responseContent); // TODO implement distinct exceptions
+            if (response.StatusCode == HttpStatusCode.InternalServerError) throw new Exception($"Request was unsuccessful due to an internal server error: {responseContent}"); // TODO implement proper InternalServerErrorException
            var result = JsonSerializer.Deserialize<T>(responseContent)
                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
            return result;
        }

-        private static async Task<T> PostUrlAndProcessJson<T>(string url, HttpContent? content)
-        {
-            using var client = new HttpClient();
-            var response = await client.PostAsync(url, content);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        private static async Task<T> PutUrlAndProcessJson<T>(string url, HttpContent content)
-        {
-            using var client = new HttpClient();
-            var response = await client.PutAsync(url, content);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        private static async Task<T> PatchUrlAndProcessJson<T>(string url, HttpContent content)
-        {
-            using var client = new HttpClient();
-            var response = await client.PatchAsync(url, content);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        private static async Task<T> DeleteUrlAndProcessJson<T>(string url)
-        {
-            using var client = new HttpClient();
-            var response = await client.DeleteAsync(url);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        public static string GetUrl(string baseUri, string endpoint, string apiKey, Dictionary<string, string> parameters)
+        public static string GetUrl(string baseUri, string endpoint, Dictionary<string, string> parameters)
        {
            var uriBuilder = new UriBuilder($"{baseUri}/{endpoint}");
            var query = HttpUtility.ParseQueryString(uriBuilder.Query);
-            if (apiKey.Length > 0) query["apiKey"] = apiKey;
            foreach (var param in parameters)
            {
                query[param.Key] = param.Value;
--- a/src/Indexer/Indexer.csproj
+++ b/src/Indexer/Indexer.csproj
@@ -15,6 +15,7 @@
    <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
    <PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
+    <PackageReference Include="System.Configuration.ConfigurationManager" Version="9.0.3" />
    <PackageReference Include="Python" Version="3.13.3" />
    <PackageReference Include="Pythonnet" Version="3.0.5" />
  </ItemGroup>
--- a/src/Indexer/Models/OptionModels.cs
+++ b/src/Indexer/Models/OptionModels.cs
@@ -0,0 +1,9 @@
+using Shared.Models;
+namespace Indexer.Models;
+
+public class IndexerOptions : ApiKeyOptions
+{
+    public required WorkerConfig[] Workers { get; set; }
+    public required ServerOptions Server { get; set;}
+    public required string PythonRuntime { get; set; } = "libpython3.13.so";
+}
--- a/src/Indexer/Models/ScriptModels.cs
+++ b/src/Indexer/Models/ScriptModels.cs
@@ -15,11 +15,11 @@ public class ScriptToolSet
    public Client.Client Client;
    public LoggerWrapper Logger;
    public ICallbackInfos? CallbackInfos;
-    public IConfiguration Configuration;
+    public IndexerOptions Configuration;
    public CancellationToken CancellationToken;
    public string Name;

-    public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IConfiguration configuration, CancellationToken cancellationToken, string name)
+    public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IndexerOptions configuration, CancellationToken cancellationToken, string name)
    {
        Configuration = configuration;
        Name = name;
--- a/src/Indexer/Program.cs
+++ b/src/Indexer/Program.cs
@@ -6,6 +6,8 @@ using ElmahCore.Mvc;
 using ElmahCore.Mvc.Logger;
 using Serilog;
 using Quartz;
+using System.Configuration;
+using Shared.Models;

 var builder = WebApplication.CreateBuilder(args);

@@ -21,6 +23,12 @@ Log.Logger = new LoggerConfiguration()
 builder.Logging.AddSerilog();
 builder.Services.AddHttpContextAccessor();
 builder.Services.AddSingleton<IConfigurationRoot>(builder.Configuration);
+
+IConfigurationSection configurationSection = builder.Configuration.GetSection("Indexer");
+IndexerOptions configuration = configurationSection.Get<IndexerOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
+builder.Services.Configure<IndexerOptions>(configurationSection);
+builder.Services.Configure<ServerOptions>(configurationSection.GetSection("Server"));
+builder.Services.Configure<ApiKeyOptions>(configurationSection);
 builder.Services.AddSingleton<Client.Client>();
 builder.Services.AddSingleton<WorkerManager>();
 builder.Services.AddHostedService<IndexerService>();
--- a/src/Indexer/ScriptContainers/PythonScriptContainer.cs
+++ b/src/Indexer/ScriptContainers/PythonScriptContainer.cs
@@ -15,11 +15,8 @@ public class PythonScriptable : IScriptContainer
    public ILogger _logger { get; set; }
    public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
    {
-        string? runtime = toolSet.Configuration.GetValue<string>("EmbeddingsearchIndexer:PythonRuntime");
-        if (runtime is not null)
-        {
+        string runtime = toolSet.Configuration.PythonRuntime;
        Runtime.PythonDLL ??= runtime;
-        }
        _logger = logger;
        SourceLoaded = false;
        if (!PythonEngine.IsInitialized)
--- a/src/Indexer/WorkerManager.cs
+++ b/src/Indexer/WorkerManager.cs
@@ -1,21 +1,22 @@
 using Indexer.Exceptions;
 using Indexer.Models;
 using Indexer.ScriptContainers;
+using Microsoft.Extensions.Options;

 public class WorkerManager
 {
    public Dictionary<string, Worker> Workers;
    public List<Type> types;
    private readonly ILogger<WorkerManager> _logger;
-    private readonly IConfiguration _configuration;
+    private readonly IndexerOptions _configuration;
    private readonly Client.Client client;

-    public WorkerManager(ILogger<WorkerManager> logger, IConfiguration configuration, Client.Client client)
+    public WorkerManager(ILogger<WorkerManager> logger, IOptions<IndexerOptions> configuration, Client.Client client)
    {
        Workers = [];
        types = [typeof(PythonScriptable), typeof(CSharpScriptable)];
        _logger = logger;
-        _configuration = configuration;
+        _configuration = configuration.Value;
        this.client = client;
    }

@@ -23,28 +24,13 @@ public class WorkerManager
    {
        _logger.LogInformation("Initializing workers");
        // Load and configure all workers
-        var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer");
-        if (!sectionMain.Exists())
-        {
-            _logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
-            throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
-        }

-        WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
-        if (sectionWorker is not null)
-        {
-            foreach (WorkerConfig workerConfig in sectionWorker.Worker)
+        foreach (WorkerConfig workerConfig in _configuration.Workers)
        {
            CancellationTokenSource cancellationTokenSource = new();
            ScriptToolSet toolSet = new(workerConfig.Script, client, _logger, _configuration, cancellationTokenSource.Token, workerConfig.Name);
            InitializeWorker(toolSet, workerConfig, cancellationTokenSource);
        }
-        }
-        else
-        {
-            _logger.LogCritical("Unable to load section \"Worker\"");
-            throw new IndexerConfigurationException("Unable to load section \"Worker\"");
-        }
        _logger.LogInformation("Initialized workers");
    }

--- a/src/Server/AIProvider.cs
+++ b/src/Server/AIProvider.cs
@@ -1,24 +1,25 @@

 using System.Text;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Server.Exceptions;
+using Server.Models;

 namespace Server;

 public class AIProvider
 {
    private readonly ILogger<AIProvider> _logger;
-    private readonly IConfiguration _configuration;
-    public AIProvidersConfiguration aIProvidersConfiguration;
+    private readonly EmbeddingSearchOptions _configuration;
+    public Dictionary<string, AiProvider> aIProvidersConfiguration;

-    public AIProvider(ILogger<AIProvider> logger, IConfiguration configuration)
+    public AIProvider(ILogger<AIProvider> logger, IOptions<EmbeddingSearchOptions> configuration)
    {
        _logger = logger;
-        _configuration = configuration;
-        AIProvidersConfiguration? retrievedAiProvidersConfiguration = _configuration
-            .GetSection("Embeddingsearch")
-            .Get<AIProvidersConfiguration>();
+        _configuration = configuration.Value;
+        Dictionary<string, AiProvider>? retrievedAiProvidersConfiguration = _configuration.AiProviders;
        if (retrievedAiProvidersConfiguration is null)
        {
            _logger.LogCritical("Unable to build AIProvidersConfiguration. Please check your configuration.");
@@ -35,8 +36,8 @@ public class AIProvider
        Uri uri = new(modelUri);
        string provider = uri.Scheme;
        string model = uri.AbsolutePath;
-        AIProviderConfiguration? aIProvider = aIProvidersConfiguration.AiProviders
-            .FirstOrDefault(x => String.Equals(x.Key.ToLower(), provider.ToLower()))
+        AiProvider? aIProvider = aIProvidersConfiguration
+            .FirstOrDefault(x => string.Equals(x.Key.ToLower(), provider.ToLower()))
            .Value;
        if (aIProvider is null)
        {
@@ -119,12 +120,12 @@ public class AIProvider

    public string[] GetModels()
    {
-        var aIProviders = aIProvidersConfiguration.AiProviders;
+        var aIProviders = aIProvidersConfiguration;
        List<string> results = [];
-        foreach (KeyValuePair<string, AIProviderConfiguration> aIProviderKV in aIProviders)
+        foreach (KeyValuePair<string, AiProvider> aIProviderKV in aIProviders)
        {
            string aIProviderName = aIProviderKV.Key;
-            AIProviderConfiguration aIProvider = aIProviderKV.Value;
+            AiProvider aIProvider = aIProviderKV.Value;

            using var httpClient = new HttpClient();

@@ -178,9 +179,14 @@ public class AIProvider
                foreach (string? result in aIProviderResult)
                {
                    if (result is null) continue;
+                    bool isInAllowList = ElementMatchesAnyRegexInList(result, aIProvider.Allowlist);
+                    bool isInDenyList =  ElementMatchesAnyRegexInList(result, aIProvider.Denylist);
+                    if (isInAllowList && !isInDenyList)
+                    {
                        results.Add(aIProviderName + ":" + result);
                    }
                }
+            }
            catch (Exception ex)
            {
                _logger.LogError("Unable to parse the response to valid models. {ex.Message}", [ex.Message]);
@@ -189,6 +195,11 @@ public class AIProvider
        }
        return [.. results];
    }
+
+    private static bool ElementMatchesAnyRegexInList(string element, string[] list)
+    {
+        return list?.Any(pattern => pattern != null && Regex.IsMatch(element, pattern)) ?? false;
+    }
 }

 public class AIProvidersConfiguration
--- a/src/Server/Controllers/AccountController.cs
+++ b/src/Server/Controllers/AccountController.cs
@@ -12,9 +12,9 @@ public class AccountController : Controller
 {
    private readonly SimpleAuthOptions _options;

-    public AccountController(IOptions<SimpleAuthOptions> options)
+    public AccountController(IOptions<EmbeddingSearchOptions> options)
    {
-        _options = options.Value;
+        _options = options.Value.SimpleAuth;
    }

    [HttpGet("Login")]
--- a/src/Server/Helper/SearchdomainHelper.cs
+++ b/src/Server/Helper/SearchdomainHelper.cs
@@ -218,6 +218,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
            {
                searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(preexistingEntity);
            }
+            searchdomain.UpdateModelsInUse();
            return preexistingEntity;
        }
        else
@@ -243,6 +244,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
            };
            entityCache.Add(entity);
            searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(entity);
+            searchdomain.UpdateModelsInUse();
            return entity;
        }
    }
--- a/src/Server/Models/Auth.cs
+++ b/src/Server/Models/Auth.cs
@@ -1,13 +0,0 @@
-namespace Server.Models;
-
-public class SimpleAuthOptions
-{
-    public List<SimpleUser> Users { get; set; } = new();
-}
-
-public class SimpleUser
-{
-    public string Username { get; set; } = "";
-    public string Password { get; set; } = "";
-    public string[] Roles { get; set; } = Array.Empty<string>();
-}
--- a/src/Server/Models/ConfigModels.cs
+++ b/src/Server/Models/ConfigModels.cs
@@ -0,0 +1,36 @@
+using System.Configuration;
+using ElmahCore;
+using Shared.Models;
+
+namespace Server.Models;
+
+public class EmbeddingSearchOptions : ApiKeyOptions
+{
+    public required ConnectionStringsSection ConnectionStrings { get; set; }
+    public ElmahOptions? Elmah { get; set; }
+    public required long EmbeddingCacheMaxCount { get; set; }
+    public required Dictionary<string, AiProvider> AiProviders { get; set; }
+    public required SimpleAuthOptions SimpleAuth { get; set; }
+    public required bool UseHttpsRedirection { get; set; }
+}
+
+public class AiProvider
+{
+    public required string Handler { get; set; }
+    public required string BaseURL { get; set; }
+    public string? ApiKey { get; set; }
+    public required string[] Allowlist { get; set; }
+    public required string[] Denylist { get; set; }
+}
+
+public class SimpleAuthOptions
+{
+    public List<SimpleUser> Users { get; set; } = [];
+}
+
+public class SimpleUser
+{
+    public string Username { get; set; } = "";
+    public string Password { get; set; } = "";
+    public string[] Roles { get; set; } = [];
+}
--- a/src/Server/Program.cs
+++ b/src/Server/Program.cs
@@ -10,11 +10,13 @@ using Server.Models;
 using Server.Services;
 using System.Text.Json.Serialization;
 using System.Reflection;
+using System.Configuration;
+using Microsoft.OpenApi.Models;
+using Shared.Models;

 var builder = WebApplication.CreateBuilder(args);

-// Add services to the container.
-
+// Add Controllers with views & string conversion for enums
 builder.Services.AddControllersWithViews()
    .AddJsonOptions(options =>
    {
@@ -23,6 +25,13 @@ builder.Services.AddControllersWithViews()
        );
    });

+// Add Configuration
+IConfigurationSection configurationSection = builder.Configuration.GetSection("Embeddingsearch");
+EmbeddingSearchOptions configuration = configurationSection.Get<EmbeddingSearchOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
+
+builder.Services.Configure<EmbeddingSearchOptions>(configurationSection);
+builder.Services.Configure<ApiKeyOptions>(configurationSection);
+
 // Add Localization
 builder.Services.AddLocalization(options => options.ResourcesPath = "Resources");
 builder.Services.Configure<RequestLocalizationOptions>(options =>
@@ -43,6 +52,31 @@ builder.Services.AddSwaggerGen(c =>
    var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml";
    var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile);
    c.IncludeXmlComments(xmlPath);
+    if (configuration.ApiKeys is not null)
+    {
+        c.AddSecurityDefinition("ApiKey", new OpenApiSecurityScheme
+        {
+            Description = "ApiKey must appear in header",
+            Type = SecuritySchemeType.ApiKey,
+            Name = "X-API-KEY",
+            In = ParameterLocation.Header,
+            Scheme = "ApiKeyScheme"
+        });
+        var key = new OpenApiSecurityScheme()
+        {
+            Reference = new OpenApiReference
+            {
+                Type = ReferenceType.SecurityScheme,
+                Id = "ApiKey"
+            },
+            In = ParameterLocation.Header
+        };
+        var requirement = new OpenApiSecurityRequirement
+        {
+            { key, []}
+        };
+        c.AddSecurityRequirement(requirement);
+    }
 });
 Log.Logger = new LoggerConfiguration()
    .ReadFrom.Configuration(builder.Configuration)
@@ -58,7 +92,12 @@ builder.Services.AddHealthChecks()

 builder.Services.AddElmah<XmlFileErrorLog>(Options =>
 {
-    Options.LogPath = builder.Configuration.GetValue<string>("Embeddingsearch:Elmah:LogFolder") ?? "~/logs";
+    Options.OnPermissionCheck = context =>
+        context.User.Claims.Any(claim =>
+            claim.Value.Equals("Admin", StringComparison.OrdinalIgnoreCase)
+            || claim.Value.Equals("Elmah", StringComparison.OrdinalIgnoreCase)
+    );
+    Options.LogPath = configuration.Elmah?.LogPath ?? "~/logs";
 });

 builder.Services
@@ -76,35 +115,11 @@ builder.Services.AddAuthorization(options =>
        policy => policy.RequireRole("Admin"));
 });

-IConfigurationSection simpleAuthSection = builder.Configuration.GetSection("Embeddingsearch:SimpleAuth");
-if (simpleAuthSection.Exists()) builder.Services.Configure<SimpleAuthOptions>(simpleAuthSection);

 var app = builder.Build();

-List<string>? allowedIps = builder.Configuration.GetSection("Embeddingsearch:Elmah:AllowedHosts")
-    .Get<List<string>>();
-
-app.Use(async (context, next) =>
-{
-    bool requestIsElmah = context.Request.Path.StartsWithSegments("/elmah");
-    bool requestIsSwagger = context.Request.Path.StartsWithSegments("/swagger");
-    
-    if (requestIsElmah || requestIsSwagger)
-    {
-        var remoteIp = context.Connection.RemoteIpAddress?.ToString();
-        bool blockRequest = allowedIps is null
-            || remoteIp is null
-            || !allowedIps.Contains(remoteIp);
-        if (blockRequest)
-        {
-            context.Response.StatusCode = 403;
-            await context.Response.WriteAsync("Forbidden");
-            return;
-        }
-    }
-
-    await next();
-});
+app.UseAuthentication();
+app.UseAuthorization();

 app.UseElmah();

@@ -120,19 +135,49 @@ app.MapHealthChecks("/healthz/AIProvider", new Microsoft.AspNetCore.Diagnostics.
 });

 bool IsDevelopment = app.Environment.IsDevelopment();
-bool useSwagger = app.Configuration.GetValue<bool>("UseSwagger");
-bool? UseMiddleware = app.Configuration.GetValue<bool?>("UseMiddleware");

-// Configure the HTTP request pipeline.
-if (IsDevelopment || useSwagger)
+app.Use(async (context, next) =>
 {
-    app.UseSwagger();
-    app.UseSwaggerUI();
-    //app.UseElmahExceptionPage(); // Messes with JSON response for API calls. Leaving this here so I don't accidentally put this in again later on.
+    if (context.Request.Path.StartsWithSegments("/swagger"))
+    {
+        if (!context.User.Identity?.IsAuthenticated ?? true)
+        {
+            context.Response.Redirect("/Account/Login");
+            return;
        }
-if (UseMiddleware == true && !IsDevelopment)
+
+        if (!context.User.IsInRole("Admin"))
        {
-    app.UseMiddleware<Shared.ApiKeyMiddleware>();
+            context.Response.StatusCode = StatusCodes.Status403Forbidden;
+            return;
+        }
+    }
+
+    await next();
+});
+
+app.UseSwagger();
+app.UseSwaggerUI(options =>
+{
+    options.EnablePersistAuthorization();
+});
+//app.UseElmahExceptionPage(); // Messes with JSON response for API calls. Leaving this here so I don't accidentally put this in again later on.
+
+if (configuration.ApiKeys is not null)
+{
+    app.UseWhen(context =>
+    {
+        RouteData routeData = context.GetRouteData();
+        string controllerName = routeData.Values["controller"]?.ToString() ?? "StaticFile";
+        if (controllerName == "Account" || controllerName == "Home" || controllerName == "StaticFile")
+        {
+            return false;
+        }
+        return true;
+    }, appBuilder =>
+    {
+        appBuilder.UseMiddleware<Shared.ApiKeyMiddleware>();    
+    });
 }

 // Add localization
@@ -143,9 +188,6 @@ var localizationOptions = new RequestLocalizationOptions()
    .AddSupportedUICultures(supportedCultures);
 app.UseRequestLocalization(localizationOptions);

-app.UseAuthentication();
-app.UseAuthorization();
-
 app.MapControllers();
 app.UseStaticFiles();

--- a/src/Server/Searchdomain.cs
+++ b/src/Server/Searchdomain.cs
@@ -216,6 +216,11 @@ public class Searchdomain
        return queryEmbeddings;
    }

+    public void UpdateModelsInUse()
+    {
+        modelsInUse = GetModels([.. entityCache]);
+    }
+
    private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary<string, float[]> queryEmbeddings)
    {
        List<(string, float)> datapointProbs = [];
@@ -237,6 +242,8 @@ public class Searchdomain
    public static List<string> GetModels(List<Entity> entities)
    {
        List<string> result = [];
+        lock (entities)
+        {
            foreach (Entity entity in entities)
            {
                foreach (Datapoint datapoint in entity.datapoints)
@@ -251,6 +258,7 @@ public class Searchdomain
                    }
                }
            }
+        }
        return result;
    }

--- a/src/Server/appsettings.Development.json
+++ b/src/Server/appsettings.Development.json
@@ -18,22 +18,21 @@
      "SQL": "server=localhost;database=embeddingsearch;uid=embeddingsearch;pwd=somepassword!;"
    },
    "Elmah": {
-      "AllowedHosts": [
-        "127.0.0.1",
-        "::1",
-        "172.17.0.1"
-      ]
+      "LogPath": "~/logs"
    },
    "EmbeddingCacheMaxCount": 10000000,
    "AiProviders": {
      "ollama": {
        "handler": "ollama",
-        "baseURL": "http://localhost:11434"
+        "baseURL": "http://192.168.0.101:11434",
+        "Allowlist": ["*"],
+        "Denylist": ["qwen3-coder:latest", "qwen3:0.6b", "deepseek-v3.1:671b-cloud", "qwen3-vl", "deepseek-ocr"]
      },
      "localAI": {
        "handler": "openai",
-        "baseURL": "http://localhost:8080",
-        "ApiKey": "Some API key here"
+        "ApiKey": "Some API key here",
+        "Allowlist": ["*"],
+        "Denylist": ["cross-encoder", "kitten-tts", "jina-reranker-v1-tiny-en", "whisper-small", "qwen3-vl-2b-instruct"]
      }
    },
    "SimpleAuth": {
--- a/src/Server/appsettings.json
+++ b/src/Server/appsettings.json
@@ -16,14 +16,5 @@
        "Application": "Embeddingsearch.Server"
    }
  },
-  "EmbeddingsearchIndexer": {
-    "Elmah": {
-      "AllowedHosts": [
-        "127.0.0.1",
-        "::1"
-      ],
-      "LogFolder": "./logs"
-    }
-  },
  "AllowedHosts": "*"
 }
--- a/src/Shared/ApiKeyMiddleware.cs
+++ b/src/Shared/ApiKeyMiddleware.cs
@@ -1,21 +1,25 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Primitives;
+using Shared.Models;

 namespace Shared;

 public class ApiKeyMiddleware
 {
    private readonly RequestDelegate _next;
-    private readonly IConfiguration _configuration;
+    private readonly ApiKeyOptions _configuration;

-    public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration)
+    public ApiKeyMiddleware(RequestDelegate next, IOptions<ApiKeyOptions> configuration)
    {
        _next = next;
-        _configuration = configuration;
+        _configuration = configuration.Value;
    }

    public async Task InvokeAsync(HttpContext context)
+    {
+        if (!(context.User.Identity?.IsAuthenticated ?? false))
        {
            if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
            {
@@ -24,15 +28,14 @@ public class ApiKeyMiddleware
                return;
            }

-        var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get<List<string>>();
-#pragma warning disable CS8604
-        if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be.
+            string[]? validApiKeys = _configuration.ApiKeys;
+            if (validApiKeys == null || !validApiKeys.ToList().Contains(extractedApiKey))
            {
                context.Response.StatusCode = 403;
                await context.Response.WriteAsync("Invalid API Key.");
                return;
            }
-#pragma warning restore CS8604
+        }

        await _next(context);
    }
--- a/src/Shared/Models/OptionModels.cs
+++ b/src/Shared/Models/OptionModels.cs
@@ -0,0 +1,13 @@
+namespace Shared.Models;
+
+public class ApiKeyOptions
+{
+    public string[]? ApiKeys { get; set; }
+}
+
+public class ServerOptions
+{
+    public required string BaseUri { get; set; }
+    public string? ApiKey { get; set; }
+    public string? Searchdomain { get; set; }
+}