Merge pull request #70 from LD-Reborn/61-add-a-model-allow-denylist

Added missing configuration file changes

src/Client/Client.cs

@@ -8,6 +8,8 @@ using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Configuration;
 using System.Reflection.Metadata.Ecma335;
 using Shared.Models;
+using System.Net;
+using Microsoft.Extensions.Options;
 
 namespace Client;
 
@@ -24,12 +26,12 @@ public class Client
             this.searchdomain = searchdomain;
         }
 
-        public Client(IConfiguration configuration)
+        public Client(IOptions<ServerOptions> configuration)
         {
-            string? baseUri = configuration.GetSection("Embeddingsearch").GetValue<string>("BaseUri");
+            string baseUri = configuration.Value.BaseUri;
-            string? apiKey = configuration.GetSection("Embeddingsearch").GetValue<string>("ApiKey");
+            string? apiKey = configuration.Value.ApiKey;
-            string? searchdomain = configuration.GetSection("Embeddingsearch").GetValue<string>("Searchdomain");
+            string? searchdomain = configuration.Value.Searchdomain;
-            this.baseUri = baseUri ?? "";
+            this.baseUri = baseUri;
             this.apiKey = apiKey ?? "";
             this.searchdomain = searchdomain ?? "";
         }
@@ -41,8 +43,8 @@ public class Client
 
         public async Task<EntityListResults> EntityListAsync(string searchdomain, bool returnEmbeddings = false)
         {
-            var url = $"{baseUri}/Entities?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
+            var url = $"{baseUri}/Entities?searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
-            return await GetUrlAndProcessJson<EntityListResults>(url);
+            return await FetchUrlAndProcessJson<EntityListResults>(HttpMethod.Get, url);
         }
 
         public async Task<EntityIndexResult> EntityIndexAsync(List<JSONEntity> jsonEntity)
@@ -53,7 +55,7 @@ public class Client
         public async Task<EntityIndexResult> EntityIndexAsync(string jsonEntity)
         {
             var content = new StringContent(jsonEntity, Encoding.UTF8, "application/json");
-            return await PutUrlAndProcessJson<EntityIndexResult>(GetUrl($"{baseUri}", "Entities", apiKey, []), content);
+            return await FetchUrlAndProcessJson<EntityIndexResult>(HttpMethod.Put, GetUrl($"{baseUri}", "Entities", []), content);
         }
 
         public async Task<EntityDeleteResults> EntityDeleteAsync(string entityName)
@@ -64,12 +66,12 @@ public class Client
         public async Task<EntityDeleteResults> EntityDeleteAsync(string searchdomain, string entityName)
         {
             var url = $"{baseUri}/Entity?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&entity={HttpUtility.UrlEncode(entityName)}";
-            return await DeleteUrlAndProcessJson<EntityDeleteResults>(url);
+            return await FetchUrlAndProcessJson<EntityDeleteResults>(HttpMethod.Delete, url);
         }
 
         public async Task<SearchdomainListResults> SearchdomainListAsync()
         {
-            return await GetUrlAndProcessJson<SearchdomainListResults>(GetUrl($"{baseUri}", "Searchdomains", apiKey, []));
+            return await FetchUrlAndProcessJson<SearchdomainListResults>(HttpMethod.Get, GetUrl($"{baseUri}", "Searchdomains", []));
         }
 
         public async Task<SearchdomainCreateResults> SearchdomainCreateAsync()
@@ -79,7 +81,7 @@ public class Client
 
         public async Task<SearchdomainCreateResults> SearchdomainCreateAsync(string searchdomain, SearchdomainSettings searchdomainSettings = new())
         {
-            return await PostUrlAndProcessJson<SearchdomainCreateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainCreateResults>(HttpMethod.Post, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
             {
                 {"searchdomain", searchdomain}
             }), new StringContent(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json"));
@@ -92,7 +94,7 @@ public class Client
 
         public async Task<SearchdomainDeleteResults> SearchdomainDeleteAsync(string searchdomain)
         {
-            return await DeleteUrlAndProcessJson<SearchdomainDeleteResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainDeleteResults>(HttpMethod.Delete, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
             {
                 {"searchdomain", searchdomain}
             }));
@@ -112,7 +114,7 @@ public class Client
 
         public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string searchdomain, string newName, string settings = "{}")
         {
-            return await PutUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
+            return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
             {
                 {"searchdomain", searchdomain},
                 {"newName", newName}
@@ -125,7 +127,7 @@ public class Client
             {
                 {"searchdomain", searchdomain}
             };
-            return await GetUrlAndProcessJson<SearchdomainSearchesResults>(GetUrl($"{baseUri}/Searchdomain", "Queries", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainSearchesResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Queries", parameters));
         }
 
         public async Task<EntityQueryResults> SearchdomainQueryAsync(string query)
@@ -143,7 +145,7 @@ public class Client
             if (topN is not null) parameters.Add("topN", ((int)topN).ToString());
             if (returnAttributes) parameters.Add("returnAttributes", returnAttributes.ToString());
 
-            return await PostUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters), null);
+            return await FetchUrlAndProcessJson<EntityQueryResults>(HttpMethod.Post, GetUrl($"{baseUri}/Searchdomain", "Query", parameters), null);
         }
 
         public async Task<SearchdomainDeleteSearchResult> SearchdomainDeleteQueryAsync(string searchdomain, string query)
@@ -153,7 +155,7 @@ public class Client
                 {"searchdomain", searchdomain},
                 {"query", query}
             };
-            return await DeleteUrlAndProcessJson<SearchdomainDeleteSearchResult>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainDeleteSearchResult>(HttpMethod.Delete, GetUrl($"{baseUri}/Searchdomain", "Query", parameters));
         }
 
         public async Task<SearchdomainUpdateSearchResult> SearchdomainUpdateQueryAsync(string searchdomain, string query, List<ResultItem> results)
@@ -163,8 +165,9 @@ public class Client
                 {"searchdomain", searchdomain},
                 {"query", query}
             };
-            return await PatchUrlAndProcessJson<SearchdomainUpdateSearchResult>(
+            return await FetchUrlAndProcessJson<SearchdomainUpdateSearchResult>(
-                GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters),
+                HttpMethod.Patch,
+                GetUrl($"{baseUri}/Searchdomain", "Query", parameters),
                 new StringContent(JsonSerializer.Serialize(results), Encoding.UTF8, "application/json"));
         }
 
@@ -174,7 +177,7 @@ public class Client
             {
                 {"searchdomain", searchdomain}
             };
-            return await GetUrlAndProcessJson<SearchdomainSettingsResults>(GetUrl($"{baseUri}/Searchdomain", "Settings", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainSettingsResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters));
         }
 
         public async Task<SearchdomainUpdateResults> SearchdomainUpdateSettingsAsync(string searchdomain, SearchdomainSettings searchdomainSettings)
@@ -184,7 +187,7 @@ public class Client
                 {"searchdomain", searchdomain}
             };
             StringContent content = new(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json");
-            return await PutUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}/Searchdomain", "Settings", apiKey, parameters), content);
+            return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters), content);
         }
 
         public async Task<SearchdomainSearchCacheSizeResults> SearchdomainGetQueryCacheSizeAsync(string searchdomain)
@@ -193,7 +196,7 @@ public class Client
             {
                 {"searchdomain", searchdomain}
             };
-            return await GetUrlAndProcessJson<SearchdomainSearchCacheSizeResults>(GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainSearchCacheSizeResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", parameters));
         }
 
         public async Task<SearchdomainInvalidateCacheResults> SearchdomainClearQueryCache(string searchdomain)
@@ -202,7 +205,7 @@ public class Client
             {
                 {"searchdomain", searchdomain}
             };
-            return await PostUrlAndProcessJson<SearchdomainInvalidateCacheResults>(GetUrl($"{baseUri}/Searchdomain/QueryCache", "Clear", apiKey, parameters), null);
+            return await FetchUrlAndProcessJson<SearchdomainInvalidateCacheResults>(HttpMethod.Post, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Clear", parameters), null);
         }
 
         public async Task<SearchdomainGetDatabaseSizeResult> SearchdomainGetDatabaseSizeAsync(string searchdomain)
@@ -211,74 +214,40 @@ public class Client
             {
                 {"searchdomain", searchdomain}
             };
-            return await GetUrlAndProcessJson<SearchdomainGetDatabaseSizeResult>(GetUrl($"{baseUri}/Searchdomain/Database", "Size", apiKey, parameters));
+            return await FetchUrlAndProcessJson<SearchdomainGetDatabaseSizeResult>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/Database", "Size", parameters));
         }
 
         public async Task<ServerGetModelsResult> ServerGetModelsAsync()
         {
-            return await GetUrlAndProcessJson<ServerGetModelsResult>(GetUrl($"{baseUri}/Server", "Models", apiKey, []));
+            return await FetchUrlAndProcessJson<ServerGetModelsResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server", "Models", []));
         }
 
         public async Task<ServerGetEmbeddingCacheSizeResult> ServerGetEmbeddingCacheSizeAsync()
         {
-            return await GetUrlAndProcessJson<ServerGetEmbeddingCacheSizeResult>(GetUrl($"{baseUri}/Server/EmbeddingCache", "Size", apiKey, []));
+            return await FetchUrlAndProcessJson<ServerGetEmbeddingCacheSizeResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server/EmbeddingCache", "Size", []));
         }
 
-        private static async Task<T> GetUrlAndProcessJson<T>(string url)
+        private async Task<T> FetchUrlAndProcessJson<T>(HttpMethod httpMethod, string url, HttpContent? content = null)
         {
+            HttpRequestMessage requestMessage = new(httpMethod, url)
+            {
+                Content = content,
+            };
+            requestMessage.Headers.Add("X-API-KEY", apiKey);
             using var client = new HttpClient();
-            var response = await client.GetAsync(url);
+            var response = await client.SendAsync(requestMessage);
             string responseContent = await response.Content.ReadAsStringAsync();
+            if (response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Unauthorized) throw new UnauthorizedAccessException(responseContent); // TODO implement distinct exceptions
+            if (response.StatusCode == HttpStatusCode.InternalServerError) throw new Exception($"Request was unsuccessful due to an internal server error: {responseContent}"); // TODO implement proper InternalServerErrorException
             var result = JsonSerializer.Deserialize<T>(responseContent)
                 ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
             return result;
         }
 
-        private static async Task<T> PostUrlAndProcessJson<T>(string url, HttpContent? content)
+        public static string GetUrl(string baseUri, string endpoint, Dictionary<string, string> parameters)
-        {
-            using var client = new HttpClient();
-            var response = await client.PostAsync(url, content);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        private static async Task<T> PutUrlAndProcessJson<T>(string url, HttpContent content)
-        {
-            using var client = new HttpClient();
-            var response = await client.PutAsync(url, content);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        private static async Task<T> PatchUrlAndProcessJson<T>(string url, HttpContent content)
-        {
-            using var client = new HttpClient();
-            var response = await client.PatchAsync(url, content);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        private static async Task<T> DeleteUrlAndProcessJson<T>(string url)
-        {
-            using var client = new HttpClient();
-            var response = await client.DeleteAsync(url);
-            string responseContent = await response.Content.ReadAsStringAsync();
-            var result = JsonSerializer.Deserialize<T>(responseContent)
-                ?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
-            return result;
-        }
-
-        public static string GetUrl(string baseUri, string endpoint, string apiKey, Dictionary<string, string> parameters)
         {
             var uriBuilder = new UriBuilder($"{baseUri}/{endpoint}");
             var query = HttpUtility.ParseQueryString(uriBuilder.Query);
-            if (apiKey.Length > 0) query["apiKey"] = apiKey;
             foreach (var param in parameters)
             {
                 query[param.Key] = param.Value;

src/Indexer/Indexer.csproj

@@ -15,6 +15,7 @@
     <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
     <PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
+    <PackageReference Include="System.Configuration.ConfigurationManager" Version="9.0.3" />
     <PackageReference Include="Python" Version="3.13.3" />
     <PackageReference Include="Pythonnet" Version="3.0.5" />
   </ItemGroup>

src/Indexer/Models/OptionModels.cs

@@ -0,0 +1,9 @@
+using Shared.Models;
+namespace Indexer.Models;
+
+public class IndexerOptions : ApiKeyOptions
+{
+    public required WorkerConfig[] Workers { get; set; }
+    public required ServerOptions Server { get; set;}
+    public required string PythonRuntime { get; set; } = "libpython3.13.so";
+}

src/Indexer/Models/ScriptModels.cs

@@ -15,11 +15,11 @@ public class ScriptToolSet
     public Client.Client Client;
     public LoggerWrapper Logger;
     public ICallbackInfos? CallbackInfos;
-    public IConfiguration Configuration;
+    public IndexerOptions Configuration;
     public CancellationToken CancellationToken;
     public string Name;
 
-    public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IConfiguration configuration, CancellationToken cancellationToken, string name)
+    public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IndexerOptions configuration, CancellationToken cancellationToken, string name)
     {
         Configuration = configuration;
         Name = name;

src/Indexer/Program.cs

@@ -6,6 +6,8 @@ using ElmahCore.Mvc;
 using ElmahCore.Mvc.Logger;
 using Serilog;
 using Quartz;
+using System.Configuration;
+using Shared.Models;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -21,6 +23,12 @@ Log.Logger = new LoggerConfiguration()
 builder.Logging.AddSerilog();
 builder.Services.AddHttpContextAccessor();
 builder.Services.AddSingleton<IConfigurationRoot>(builder.Configuration);
+
+IConfigurationSection configurationSection = builder.Configuration.GetSection("Indexer");
+IndexerOptions configuration = configurationSection.Get<IndexerOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
+builder.Services.Configure<IndexerOptions>(configurationSection);
+builder.Services.Configure<ServerOptions>(configurationSection.GetSection("Server"));
+builder.Services.Configure<ApiKeyOptions>(configurationSection);
 builder.Services.AddSingleton<Client.Client>();
 builder.Services.AddSingleton<WorkerManager>();
 builder.Services.AddHostedService<IndexerService>();

src/Indexer/ScriptContainers/PythonScriptContainer.cs

@@ -15,11 +15,8 @@ public class PythonScriptable : IScriptContainer
     public ILogger _logger { get; set; }
     public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
     {
-        string? runtime = toolSet.Configuration.GetValue<string>("EmbeddingsearchIndexer:PythonRuntime");
+        string runtime = toolSet.Configuration.PythonRuntime;
-        if (runtime is not null)
-        {
         Runtime.PythonDLL ??= runtime;
-        }
         _logger = logger;
         SourceLoaded = false;
         if (!PythonEngine.IsInitialized)

src/Indexer/WorkerManager.cs

@@ -1,21 +1,22 @@
 using Indexer.Exceptions;
 using Indexer.Models;
 using Indexer.ScriptContainers;
+using Microsoft.Extensions.Options;
 
 public class WorkerManager
 {
     public Dictionary<string, Worker> Workers;
     public List<Type> types;
     private readonly ILogger<WorkerManager> _logger;
-    private readonly IConfiguration _configuration;
+    private readonly IndexerOptions _configuration;
     private readonly Client.Client client;
 
-    public WorkerManager(ILogger<WorkerManager> logger, IConfiguration configuration, Client.Client client)
+    public WorkerManager(ILogger<WorkerManager> logger, IOptions<IndexerOptions> configuration, Client.Client client)
     {
         Workers = [];
         types = [typeof(PythonScriptable), typeof(CSharpScriptable)];
         _logger = logger;
-        _configuration = configuration;
+        _configuration = configuration.Value;
         this.client = client;
     }
 
@@ -23,28 +24,13 @@ public class WorkerManager
     {
         _logger.LogInformation("Initializing workers");
         // Load and configure all workers
-        var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer");
-        if (!sectionMain.Exists())
-        {
-            _logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
-            throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
-        }
 
-        WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
+        foreach (WorkerConfig workerConfig in _configuration.Workers)
-        if (sectionWorker is not null)
-        {
-            foreach (WorkerConfig workerConfig in sectionWorker.Worker)
         {
             CancellationTokenSource cancellationTokenSource = new();
             ScriptToolSet toolSet = new(workerConfig.Script, client, _logger, _configuration, cancellationTokenSource.Token, workerConfig.Name);
             InitializeWorker(toolSet, workerConfig, cancellationTokenSource);
         }
-        }
-        else
-        {
-            _logger.LogCritical("Unable to load section \"Worker\"");
-            throw new IndexerConfigurationException("Unable to load section \"Worker\"");
-        }
         _logger.LogInformation("Initialized workers");
     }
 

src/Indexer/appsettings.Development.json

@@ -5,46 +5,23 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
-  "Embeddingsearch": {
+  "Indexer": {
-    "BaseUri": "http://localhost:5146"
+    "Workers": [
-  },
-  "EmbeddingsearchIndexer": {
-    "Worker":
-    [
       {
         "Name": "pythonExample",
         "Script": "Scripts/example.py",
-        "Calls": [
-          {
-            "Name": "intervalCall",
-            "Type": "interval",
-            "Interval": 30000
-          }
-        ]
-      },
-      {
-        "Name": "csharpExample",
-        "Script": "Scripts/example.csx",
         "Calls": [
           {
             "Name": "runonceCall",
             "Type": "runonce"
-          },
-          {
-            "Name": "scheduleCall",
-            "Type": "schedule",
-            "Schedule": "0 0/5 * * * ?"
-          },
-          {
-            "Name": "fileupdateCall",
-            "Type": "fileupdate",
-            "Path": "./Scripts/example_content",
-            "Events": ["Created", "Changed", "Deleted", "Renamed"],
-            "Filters": ["*.md", "*.txt"],
-            "IncludeSubdirectories": true
           }
         ]
       }
-    ]
+    ],
+    "ApiKeys": ["xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"],
+    "Server": {
+      "BaseUri": "http://localhost:5146",
+      "ApiKey": "yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy"
+    }
   }
 }

src/Indexer/appsettings.json

@@ -24,7 +24,7 @@
       ],
       "LogFolder": "./logs"
     },
-    "PythonRuntime": "libpython3.12.so"
+    "PythonRuntime": "libpython3.13.so"
   },
   "AllowedHosts": "*"
 }

src/Server/AIProvider.cs

@@ -1,24 +1,25 @@
 
 using System.Text;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Server.Exceptions;
+using Server.Models;
 
 namespace Server;
 
 public class AIProvider
 {
     private readonly ILogger<AIProvider> _logger;
-    private readonly IConfiguration _configuration;
+    private readonly EmbeddingSearchOptions _configuration;
-    public AIProvidersConfiguration aIProvidersConfiguration;
+    public Dictionary<string, AiProvider> aIProvidersConfiguration;
 
-    public AIProvider(ILogger<AIProvider> logger, IConfiguration configuration)
+    public AIProvider(ILogger<AIProvider> logger, IOptions<EmbeddingSearchOptions> configuration)
     {
         _logger = logger;
-        _configuration = configuration;
+        _configuration = configuration.Value;
-        AIProvidersConfiguration? retrievedAiProvidersConfiguration = _configuration
+        Dictionary<string, AiProvider>? retrievedAiProvidersConfiguration = _configuration.AiProviders;
-            .GetSection("Embeddingsearch")
-            .Get<AIProvidersConfiguration>();
         if (retrievedAiProvidersConfiguration is null)
         {
             _logger.LogCritical("Unable to build AIProvidersConfiguration. Please check your configuration.");
@@ -35,8 +36,8 @@ public class AIProvider
         Uri uri = new(modelUri);
         string provider = uri.Scheme;
         string model = uri.AbsolutePath;
-        AIProviderConfiguration? aIProvider = aIProvidersConfiguration.AiProviders
+        AiProvider? aIProvider = aIProvidersConfiguration
-            .FirstOrDefault(x => String.Equals(x.Key.ToLower(), provider.ToLower()))
+            .FirstOrDefault(x => string.Equals(x.Key.ToLower(), provider.ToLower()))
             .Value;
         if (aIProvider is null)
         {
@@ -119,12 +120,12 @@ public class AIProvider
 
     public string[] GetModels()
     {
-        var aIProviders = aIProvidersConfiguration.AiProviders;
+        var aIProviders = aIProvidersConfiguration;
         List<string> results = [];
-        foreach (KeyValuePair<string, AIProviderConfiguration> aIProviderKV in aIProviders)
+        foreach (KeyValuePair<string, AiProvider> aIProviderKV in aIProviders)
         {
             string aIProviderName = aIProviderKV.Key;
-            AIProviderConfiguration aIProvider = aIProviderKV.Value;
+            AiProvider aIProvider = aIProviderKV.Value;
 
             using var httpClient = new HttpClient();
 
@@ -178,9 +179,14 @@ public class AIProvider
                 foreach (string? result in aIProviderResult)
                 {
                     if (result is null) continue;
+                    bool isInAllowList = ElementMatchesAnyRegexInList(result, aIProvider.Allowlist);
+                    bool isInDenyList =  ElementMatchesAnyRegexInList(result, aIProvider.Denylist);
+                    if (isInAllowList && !isInDenyList)
+                    {
                         results.Add(aIProviderName + ":" + result);
                     }
                 }
+            }
             catch (Exception ex)
             {
                 _logger.LogError("Unable to parse the response to valid models. {ex.Message}", [ex.Message]);
@@ -189,6 +195,11 @@ public class AIProvider
         }
         return [.. results];
     }
+
+    private static bool ElementMatchesAnyRegexInList(string element, string[] list)
+    {
+        return list?.Any(pattern => pattern != null && Regex.IsMatch(element, pattern)) ?? false;
+    }
 }
 
 public class AIProvidersConfiguration

src/Server/Controllers/AccountController.cs

@@ -12,9 +12,9 @@ public class AccountController : Controller
 {
     private readonly SimpleAuthOptions _options;
 
-    public AccountController(IOptions<SimpleAuthOptions> options)
+    public AccountController(IOptions<EmbeddingSearchOptions> options)
     {
-        _options = options.Value;
+        _options = options.Value.SimpleAuth;
     }
 
     [HttpGet("Login")]

src/Server/Helper/SearchdomainHelper.cs

@@ -218,6 +218,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             {
                 searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(preexistingEntity);
             }
+            searchdomain.UpdateModelsInUse();
             return preexistingEntity;
         }
         else
@@ -243,6 +244,7 @@ public class SearchdomainHelper(ILogger<SearchdomainHelper> logger, DatabaseHelp
             };
             entityCache.Add(entity);
             searchdomain.ReconciliateOrInvalidateCacheForNewOrUpdatedEntity(entity);
+            searchdomain.UpdateModelsInUse();
             return entity;
         }
     }

src/Server/Models/Auth.cs

@@ -1,13 +0,0 @@
-namespace Server.Models;
-
-public class SimpleAuthOptions
-{
-    public List<SimpleUser> Users { get; set; } = new();
-}
-
-public class SimpleUser
-{
-    public string Username { get; set; } = "";
-    public string Password { get; set; } = "";
-    public string[] Roles { get; set; } = Array.Empty<string>();
-}

src/Server/Models/ConfigModels.cs

@@ -0,0 +1,36 @@
+using System.Configuration;
+using ElmahCore;
+using Shared.Models;
+
+namespace Server.Models;
+
+public class EmbeddingSearchOptions : ApiKeyOptions
+{
+    public required ConnectionStringsSection ConnectionStrings { get; set; }
+    public ElmahOptions? Elmah { get; set; }
+    public required long EmbeddingCacheMaxCount { get; set; }
+    public required Dictionary<string, AiProvider> AiProviders { get; set; }
+    public required SimpleAuthOptions SimpleAuth { get; set; }
+    public required bool UseHttpsRedirection { get; set; }
+}
+
+public class AiProvider
+{
+    public required string Handler { get; set; }
+    public required string BaseURL { get; set; }
+    public string? ApiKey { get; set; }
+    public required string[] Allowlist { get; set; }
+    public required string[] Denylist { get; set; }
+}
+
+public class SimpleAuthOptions
+{
+    public List<SimpleUser> Users { get; set; } = [];
+}
+
+public class SimpleUser
+{
+    public string Username { get; set; } = "";
+    public string Password { get; set; } = "";
+    public string[] Roles { get; set; } = [];
+}

src/Server/Program.cs

@@ -10,11 +10,13 @@ using Server.Models;
 using Server.Services;
 using System.Text.Json.Serialization;
 using System.Reflection;
+using System.Configuration;
+using Microsoft.OpenApi.Models;
+using Shared.Models;
 
 var builder = WebApplication.CreateBuilder(args);
 
-// Add services to the container.
+// Add Controllers with views & string conversion for enums
-
 builder.Services.AddControllersWithViews()
     .AddJsonOptions(options =>
     {
@@ -23,6 +25,13 @@ builder.Services.AddControllersWithViews()
         );
     });
 
+// Add Configuration
+IConfigurationSection configurationSection = builder.Configuration.GetSection("Embeddingsearch");
+EmbeddingSearchOptions configuration = configurationSection.Get<EmbeddingSearchOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
+
+builder.Services.Configure<EmbeddingSearchOptions>(configurationSection);
+builder.Services.Configure<ApiKeyOptions>(configurationSection);
+
 // Add Localization
 builder.Services.AddLocalization(options => options.ResourcesPath = "Resources");
 builder.Services.Configure<RequestLocalizationOptions>(options =>
@@ -43,6 +52,31 @@ builder.Services.AddSwaggerGen(c =>
     var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml";
     var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile);
     c.IncludeXmlComments(xmlPath);
+    if (configuration.ApiKeys is not null)
+    {
+        c.AddSecurityDefinition("ApiKey", new OpenApiSecurityScheme
+        {
+            Description = "ApiKey must appear in header",
+            Type = SecuritySchemeType.ApiKey,
+            Name = "X-API-KEY",
+            In = ParameterLocation.Header,
+            Scheme = "ApiKeyScheme"
+        });
+        var key = new OpenApiSecurityScheme()
+        {
+            Reference = new OpenApiReference
+            {
+                Type = ReferenceType.SecurityScheme,
+                Id = "ApiKey"
+            },
+            In = ParameterLocation.Header
+        };
+        var requirement = new OpenApiSecurityRequirement
+        {
+            { key, []}
+        };
+        c.AddSecurityRequirement(requirement);
+    }
 });
 Log.Logger = new LoggerConfiguration()
     .ReadFrom.Configuration(builder.Configuration)
@@ -58,7 +92,12 @@ builder.Services.AddHealthChecks()
 
 builder.Services.AddElmah<XmlFileErrorLog>(Options =>
 {
-    Options.LogPath = builder.Configuration.GetValue<string>("Embeddingsearch:Elmah:LogFolder") ?? "~/logs";
+    Options.OnPermissionCheck = context =>
+        context.User.Claims.Any(claim =>
+            claim.Value.Equals("Admin", StringComparison.OrdinalIgnoreCase)
+            || claim.Value.Equals("Elmah", StringComparison.OrdinalIgnoreCase)
+    );
+    Options.LogPath = configuration.Elmah?.LogPath ?? "~/logs";
 });
 
 builder.Services
@@ -76,35 +115,11 @@ builder.Services.AddAuthorization(options =>
         policy => policy.RequireRole("Admin"));
 });
 
-IConfigurationSection simpleAuthSection = builder.Configuration.GetSection("Embeddingsearch:SimpleAuth");
-if (simpleAuthSection.Exists()) builder.Services.Configure<SimpleAuthOptions>(simpleAuthSection);
 
 var app = builder.Build();
 
-List<string>? allowedIps = builder.Configuration.GetSection("Embeddingsearch:Elmah:AllowedHosts")
+app.UseAuthentication();
-    .Get<List<string>>();
+app.UseAuthorization();
-
-app.Use(async (context, next) =>
-{
-    bool requestIsElmah = context.Request.Path.StartsWithSegments("/elmah");
-    bool requestIsSwagger = context.Request.Path.StartsWithSegments("/swagger");
-    
-    if (requestIsElmah || requestIsSwagger)
-    {
-        var remoteIp = context.Connection.RemoteIpAddress?.ToString();
-        bool blockRequest = allowedIps is null
-            || remoteIp is null
-            || !allowedIps.Contains(remoteIp);
-        if (blockRequest)
-        {
-            context.Response.StatusCode = 403;
-            await context.Response.WriteAsync("Forbidden");
-            return;
-        }
-    }
-
-    await next();
-});
 
 app.UseElmah();
 
@@ -120,19 +135,49 @@ app.MapHealthChecks("/healthz/AIProvider", new Microsoft.AspNetCore.Diagnostics.
 });
 
 bool IsDevelopment = app.Environment.IsDevelopment();
-bool useSwagger = app.Configuration.GetValue<bool>("UseSwagger");
-bool? UseMiddleware = app.Configuration.GetValue<bool?>("UseMiddleware");
 
-// Configure the HTTP request pipeline.
+app.Use(async (context, next) =>
-if (IsDevelopment || useSwagger)
 {
-    app.UseSwagger();
+    if (context.Request.Path.StartsWithSegments("/swagger"))
-    app.UseSwaggerUI();
+    {
-    //app.UseElmahExceptionPage(); // Messes with JSON response for API calls. Leaving this here so I don't accidentally put this in again later on.
+        if (!context.User.Identity?.IsAuthenticated ?? true)
-}
+        {
-if (UseMiddleware == true && !IsDevelopment)
+            context.Response.Redirect("/Account/Login");
+            return;
+        }
+
+        if (!context.User.IsInRole("Admin"))
+        {
+            context.Response.StatusCode = StatusCodes.Status403Forbidden;
+            return;
+        }
+    }
+
+    await next();
+});
+
+app.UseSwagger();
+app.UseSwaggerUI(options =>
 {
-    app.UseMiddleware<Shared.ApiKeyMiddleware>();
+    options.EnablePersistAuthorization();
+});
+//app.UseElmahExceptionPage(); // Messes with JSON response for API calls. Leaving this here so I don't accidentally put this in again later on.
+
+if (configuration.ApiKeys is not null)
+{
+    app.UseWhen(context =>
+    {
+        RouteData routeData = context.GetRouteData();
+        string controllerName = routeData.Values["controller"]?.ToString() ?? "StaticFile";
+        if (controllerName == "Account" || controllerName == "Home" || controllerName == "StaticFile")
+        {
+            return false;
+        }
+        return true;
+    }, appBuilder =>
+    {
+        appBuilder.UseMiddleware<Shared.ApiKeyMiddleware>();    
+    });
 }
 
 // Add localization
@@ -143,9 +188,6 @@ var localizationOptions = new RequestLocalizationOptions()
     .AddSupportedUICultures(supportedCultures);
 app.UseRequestLocalization(localizationOptions);
 
-app.UseAuthentication();
-app.UseAuthorization();
-
 app.MapControllers();
 app.UseStaticFiles();
 

src/Server/Searchdomain.cs

@@ -216,6 +216,11 @@ public class Searchdomain
         return queryEmbeddings;
     }
 
+    public void UpdateModelsInUse()
+    {
+        modelsInUse = GetModels([.. entityCache]);
+    }
+
     private static float EvaluateEntityAgainstQueryEmbeddings(Entity entity, Dictionary<string, float[]> queryEmbeddings)
     {
         List<(string, float)> datapointProbs = [];
@@ -237,6 +242,8 @@ public class Searchdomain
     public static List<string> GetModels(List<Entity> entities)
     {
         List<string> result = [];
+        lock (entities)
+        {
             foreach (Entity entity in entities)
             {
                 foreach (Datapoint datapoint in entity.datapoints)
@@ -251,6 +258,7 @@ public class Searchdomain
                     }
                 }
             }
+        }
         return result;
     }
 

src/Server/appsettings.Development.json

@@ -18,22 +18,22 @@
       "SQL": "server=localhost;database=embeddingsearch;uid=embeddingsearch;pwd=somepassword!;"
     },
     "Elmah": {
-      "AllowedHosts": [
+      "LogPath": "~/logs"
-        "127.0.0.1",
-        "::1",
-        "172.17.0.1"
-      ]
     },
     "EmbeddingCacheMaxCount": 10000000,
     "AiProviders": {
       "ollama": {
         "handler": "ollama",
-        "baseURL": "http://localhost:11434"
+        "baseURL": "http://localhost:11434",
+        "Allowlist": [".*"],
+        "Denylist": ["qwen3-coder:latest", "qwen3:0.6b", "deepseek-v3.1:671b-cloud", "qwen3-vl", "deepseek-ocr"]
       },
       "localAI": {
         "handler": "openai",
         "baseURL": "http://localhost:8080",
-        "ApiKey": "Some API key here"
+        "ApiKey": "Some API key here",
+        "Allowlist": [".*"],
+        "Denylist": ["cross-encoder", "kitten-tts", "jina-reranker-v1-tiny-en", "whisper-small", "qwen3-vl-2b-instruct"]
       }
     },
     "SimpleAuth": {

src/Server/appsettings.json

@@ -16,14 +16,5 @@
         "Application": "Embeddingsearch.Server"
     }
   },
-  "EmbeddingsearchIndexer": {
-    "Elmah": {
-      "AllowedHosts": [
-        "127.0.0.1",
-        "::1"
-      ],
-      "LogFolder": "./logs"
-    }
-  },
   "AllowedHosts": "*"
 }

src/Shared/ApiKeyMiddleware.cs

@@ -1,21 +1,25 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Primitives;
+using Shared.Models;
 
 namespace Shared;
 
 public class ApiKeyMiddleware
 {
     private readonly RequestDelegate _next;
-    private readonly IConfiguration _configuration;
+    private readonly ApiKeyOptions _configuration;
 
-    public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration)
+    public ApiKeyMiddleware(RequestDelegate next, IOptions<ApiKeyOptions> configuration)
     {
         _next = next;
-        _configuration = configuration;
+        _configuration = configuration.Value;
     }
 
     public async Task InvokeAsync(HttpContext context)
+    {
+        if (!(context.User.Identity?.IsAuthenticated ?? false))
         {
             if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
             {
@@ -24,15 +28,14 @@ public class ApiKeyMiddleware
                 return;
             }
 
-        var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get<List<string>>();
+            string[]? validApiKeys = _configuration.ApiKeys;
-#pragma warning disable CS8604
+            if (validApiKeys == null || !validApiKeys.ToList().Contains(extractedApiKey))
-        if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be.
             {
                 context.Response.StatusCode = 403;
                 await context.Response.WriteAsync("Invalid API Key.");
                 return;
             }
-#pragma warning restore CS8604
+        }
 
         await _next(context);
     }

src/Shared/Models/OptionModels.cs

@@ -0,0 +1,13 @@
+namespace Shared.Models;
+
+public class ApiKeyOptions
+{
+    public string[]? ApiKeys { get; set; }
+}
+
+public class ServerOptions
+{
+    public required string BaseUri { get; set; }
+    public string? ApiKey { get; set; }
+    public string? Searchdomain { get; set; }
+}