Added allowlist and denylist, fixed patchy configuration with proper options models, fixed api middleware authorization issues

src/Client/Client.cs

@@ -9,6 +9,7 @@ using Microsoft.Extensions.Configuration;
 using System.Reflection.Metadata.Ecma335;
 using Shared.Models;
 using System.Net;
+using Microsoft.Extensions.Options;
 
 namespace Client;
 
@@ -25,12 +26,12 @@ public class Client
             this.searchdomain = searchdomain;
         }
 
-        public Client(IConfiguration configuration)
+        public Client(IOptions<ServerOptions> configuration)
         {
-            string? baseUri = configuration.GetSection("Embeddingsearch").GetValue<string>("BaseUri");
+            string baseUri = configuration.Value.BaseUri;
-            string? apiKey = configuration.GetSection("Embeddingsearch").GetValue<string>("ApiKey");
+            string? apiKey = configuration.Value.ApiKey;
-            string? searchdomain = configuration.GetSection("Embeddingsearch").GetValue<string>("Searchdomain");
+            string? searchdomain = configuration.Value.Searchdomain;
-            this.baseUri = baseUri ?? "";
+            this.baseUri = baseUri;
             this.apiKey = apiKey ?? "";
             this.searchdomain = searchdomain ?? "";
         }

src/Indexer/Indexer.csproj

@@ -15,6 +15,7 @@
     <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
     <PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
+    <PackageReference Include="System.Configuration.ConfigurationManager" Version="9.0.3" />
     <PackageReference Include="Python" Version="3.13.3" />
     <PackageReference Include="Pythonnet" Version="3.0.5" />
   </ItemGroup>

src/Indexer/Models/OptionModels.cs

@@ -0,0 +1,9 @@
+using Shared.Models;
+namespace Indexer.Models;
+
+public class IndexerOptions : ApiKeyOptions
+{
+    public required WorkerConfig[] Workers { get; set; }
+    public required ServerOptions Server { get; set;}
+    public required string PythonRuntime { get; set; } = "libpython3.13.so";
+}

src/Indexer/Models/ScriptModels.cs

@@ -15,11 +15,11 @@ public class ScriptToolSet
     public Client.Client Client;
     public LoggerWrapper Logger;
     public ICallbackInfos? CallbackInfos;
-    public IConfiguration Configuration;
+    public IndexerOptions Configuration;
     public CancellationToken CancellationToken;
     public string Name;
 
-    public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IConfiguration configuration, CancellationToken cancellationToken, string name)
+    public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IndexerOptions configuration, CancellationToken cancellationToken, string name)
     {
         Configuration = configuration;
         Name = name;

src/Indexer/Program.cs

@@ -6,6 +6,8 @@ using ElmahCore.Mvc;
 using ElmahCore.Mvc.Logger;
 using Serilog;
 using Quartz;
+using System.Configuration;
+using Shared.Models;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -21,6 +23,12 @@ Log.Logger = new LoggerConfiguration()
 builder.Logging.AddSerilog();
 builder.Services.AddHttpContextAccessor();
 builder.Services.AddSingleton<IConfigurationRoot>(builder.Configuration);
+
+IConfigurationSection configurationSection = builder.Configuration.GetSection("Indexer");
+IndexerOptions configuration = configurationSection.Get<IndexerOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
+builder.Services.Configure<IndexerOptions>(configurationSection);
+builder.Services.Configure<ServerOptions>(configurationSection.GetSection("Server"));
+builder.Services.Configure<ApiKeyOptions>(configurationSection);
 builder.Services.AddSingleton<Client.Client>();
 builder.Services.AddSingleton<WorkerManager>();
 builder.Services.AddHostedService<IndexerService>();

src/Indexer/ScriptContainers/PythonScriptContainer.cs

@@ -15,11 +15,8 @@ public class PythonScriptable : IScriptContainer
     public ILogger _logger { get; set; }
     public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
     {
-        string? runtime = toolSet.Configuration.GetValue<string>("EmbeddingsearchIndexer:PythonRuntime");
+        string runtime = toolSet.Configuration.PythonRuntime;
-        if (runtime is not null)
-        {
         Runtime.PythonDLL ??= runtime;
-        }
         _logger = logger;
         SourceLoaded = false;
         if (!PythonEngine.IsInitialized)

src/Indexer/WorkerManager.cs

@@ -1,21 +1,22 @@
 using Indexer.Exceptions;
 using Indexer.Models;
 using Indexer.ScriptContainers;
+using Microsoft.Extensions.Options;
 
 public class WorkerManager
 {
     public Dictionary<string, Worker> Workers;
     public List<Type> types;
     private readonly ILogger<WorkerManager> _logger;
-    private readonly IConfiguration _configuration;
+    private readonly IndexerOptions _configuration;
     private readonly Client.Client client;
 
-    public WorkerManager(ILogger<WorkerManager> logger, IConfiguration configuration, Client.Client client)
+    public WorkerManager(ILogger<WorkerManager> logger, IOptions<IndexerOptions> configuration, Client.Client client)
     {
         Workers = [];
         types = [typeof(PythonScriptable), typeof(CSharpScriptable)];
         _logger = logger;
-        _configuration = configuration;
+        _configuration = configuration.Value;
         this.client = client;
     }
 
@@ -23,28 +24,13 @@ public class WorkerManager
     {
         _logger.LogInformation("Initializing workers");
         // Load and configure all workers
-        var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer");
-        if (!sectionMain.Exists())
-        {
-            _logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
-            throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
-        }
 
-        WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
+        foreach (WorkerConfig workerConfig in _configuration.Workers)
-        if (sectionWorker is not null)
-        {
-            foreach (WorkerConfig workerConfig in sectionWorker.Worker)
         {
             CancellationTokenSource cancellationTokenSource = new();
             ScriptToolSet toolSet = new(workerConfig.Script, client, _logger, _configuration, cancellationTokenSource.Token, workerConfig.Name);
             InitializeWorker(toolSet, workerConfig, cancellationTokenSource);
         }
-        }
-        else
-        {
-            _logger.LogCritical("Unable to load section \"Worker\"");
-            throw new IndexerConfigurationException("Unable to load section \"Worker\"");
-        }
         _logger.LogInformation("Initialized workers");
     }
 

src/Server/AIProvider.cs

@@ -1,24 +1,25 @@
 
 using System.Text;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Server.Exceptions;
+using Server.Models;
 
 namespace Server;
 
 public class AIProvider
 {
     private readonly ILogger<AIProvider> _logger;
-    private readonly IConfiguration _configuration;
+    private readonly EmbeddingSearchOptions _configuration;
-    public AIProvidersConfiguration aIProvidersConfiguration;
+    public Dictionary<string, AiProvider> aIProvidersConfiguration;
 
-    public AIProvider(ILogger<AIProvider> logger, IConfiguration configuration)
+    public AIProvider(ILogger<AIProvider> logger, IOptions<EmbeddingSearchOptions> configuration)
     {
         _logger = logger;
-        _configuration = configuration;
+        _configuration = configuration.Value;
-        AIProvidersConfiguration? retrievedAiProvidersConfiguration = _configuration
+        Dictionary<string, AiProvider>? retrievedAiProvidersConfiguration = _configuration.AiProviders;
-            .GetSection("Embeddingsearch")
-            .Get<AIProvidersConfiguration>();
         if (retrievedAiProvidersConfiguration is null)
         {
             _logger.LogCritical("Unable to build AIProvidersConfiguration. Please check your configuration.");
@@ -35,8 +36,8 @@ public class AIProvider
         Uri uri = new(modelUri);
         string provider = uri.Scheme;
         string model = uri.AbsolutePath;
-        AIProviderConfiguration? aIProvider = aIProvidersConfiguration.AiProviders
+        AiProvider? aIProvider = aIProvidersConfiguration
-            .FirstOrDefault(x => String.Equals(x.Key.ToLower(), provider.ToLower()))
+            .FirstOrDefault(x => string.Equals(x.Key.ToLower(), provider.ToLower()))
             .Value;
         if (aIProvider is null)
         {
@@ -119,12 +120,12 @@ public class AIProvider
 
     public string[] GetModels()
     {
-        var aIProviders = aIProvidersConfiguration.AiProviders;
+        var aIProviders = aIProvidersConfiguration;
         List<string> results = [];
-        foreach (KeyValuePair<string, AIProviderConfiguration> aIProviderKV in aIProviders)
+        foreach (KeyValuePair<string, AiProvider> aIProviderKV in aIProviders)
         {
             string aIProviderName = aIProviderKV.Key;
-            AIProviderConfiguration aIProvider = aIProviderKV.Value;
+            AiProvider aIProvider = aIProviderKV.Value;
 
             using var httpClient = new HttpClient();
 
@@ -178,9 +179,14 @@ public class AIProvider
                 foreach (string? result in aIProviderResult)
                 {
                     if (result is null) continue;
+                    bool isInAllowList = ElementMatchesAnyRegexInList(result, aIProvider.Allowlist);
+                    bool isInDenyList =  ElementMatchesAnyRegexInList(result, aIProvider.Denylist);
+                    if (isInAllowList && !isInDenyList)
+                    {
                         results.Add(aIProviderName + ":" + result);
                     }
                 }
+            }
             catch (Exception ex)
             {
                 _logger.LogError("Unable to parse the response to valid models. {ex.Message}", [ex.Message]);
@@ -189,6 +195,11 @@ public class AIProvider
         }
         return [.. results];
     }
+
+    private static bool ElementMatchesAnyRegexInList(string element, string[] list)
+    {
+        return list?.Any(pattern => pattern != null && Regex.IsMatch(element, pattern)) ?? false;
+    }
 }
 
 public class AIProvidersConfiguration

src/Server/Controllers/AccountController.cs

@@ -12,9 +12,9 @@ public class AccountController : Controller
 {
     private readonly SimpleAuthOptions _options;
 
-    public AccountController(EmbeddingSearchOptions options)
+    public AccountController(IOptions<EmbeddingSearchOptions> options)
     {
-        _options = options.SimpleAuth;
+        _options = options.Value.SimpleAuth;
     }
 
     [HttpGet("Login")]

src/Server/Models/ConfigModels.cs

@@ -1,16 +1,16 @@
 using System.Configuration;
 using ElmahCore;
+using Shared.Models;
 
 namespace Server.Models;
 
-public class EmbeddingSearchOptions
+public class EmbeddingSearchOptions : ApiKeyOptions
 {
     public required ConnectionStringsSection ConnectionStrings { get; set; }
     public ElmahOptions? Elmah { get; set; }
     public required long EmbeddingCacheMaxCount { get; set; }
-    public required AiProvider[] AiProviders { get; set; }
+    public required Dictionary<string, AiProvider> AiProviders { get; set; }
     public required SimpleAuthOptions SimpleAuth { get; set; }
-    public string[]? ApiKeys { get; set; }
     public required bool UseHttpsRedirection { get; set; }
 }
 
@@ -18,7 +18,7 @@ public class AiProvider
 {
     public required string Handler { get; set; }
     public required string BaseURL { get; set; }
-    public required string ApiKey { get; set; }
+    public string? ApiKey { get; set; }
     public required string[] Allowlist { get; set; }
     public required string[] Denylist { get; set; }
 }

src/Server/Program.cs

@@ -12,6 +12,7 @@ using System.Text.Json.Serialization;
 using System.Reflection;
 using System.Configuration;
 using Microsoft.OpenApi.Models;
+using Shared.Models;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -29,6 +30,7 @@ IConfigurationSection configurationSection = builder.Configuration.GetSection("E
 EmbeddingSearchOptions configuration = configurationSection.Get<EmbeddingSearchOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
 
 builder.Services.Configure<EmbeddingSearchOptions>(configurationSection);
+builder.Services.Configure<ApiKeyOptions>(configurationSection);
 
 // Add Localization
 builder.Services.AddLocalization(options => options.ResourcesPath = "Resources");
@@ -133,8 +135,6 @@ app.MapHealthChecks("/healthz/AIProvider", new Microsoft.AspNetCore.Diagnostics.
 });
 
 bool IsDevelopment = app.Environment.IsDevelopment();
-bool useSwagger = app.Configuration.GetValue<bool>("UseSwagger");
-bool? UseMiddleware = app.Configuration.GetValue<bool?>("UseMiddleware");
 
 app.UseSwagger();
 app.UseSwaggerUI(options =>
@@ -145,7 +145,19 @@ app.UseSwaggerUI(options =>
 
 if (configuration.ApiKeys is not null)
 {
-    app.UseMiddleware<Shared.ApiKeyMiddleware>();
+    app.UseWhen(context =>
+    {
+        RouteData routeData = context.GetRouteData();
+        string controllerName = routeData.Values["controller"]?.ToString() ?? "StaticFile";
+        if (controllerName == "Account" || controllerName == "Home" || controllerName == "StaticFile")
+        {
+            return false;
+        }
+        return true;
+    }, appBuilder =>
+    {
+        appBuilder.UseMiddleware<Shared.ApiKeyMiddleware>();    
+    });
 }
 
 // Add localization

src/Shared/ApiKeyMiddleware.cs

@@ -1,21 +1,25 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Primitives;
+using Shared.Models;
 
 namespace Shared;
 
 public class ApiKeyMiddleware
 {
     private readonly RequestDelegate _next;
-    private readonly IConfiguration _configuration;
+    private readonly ApiKeyOptions _configuration;
 
-    public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration)
+    public ApiKeyMiddleware(RequestDelegate next, IOptions<ApiKeyOptions> configuration)
     {
         _next = next;
-        _configuration = configuration;
+        _configuration = configuration.Value;
     }
 
     public async Task InvokeAsync(HttpContext context)
+    {
+        if (!(context.User.Identity?.IsAuthenticated ?? false))
         {
             if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
             {
@@ -24,15 +28,14 @@ public class ApiKeyMiddleware
                 return;
             }
 
-        var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get<List<string>>();
+            string[]? validApiKeys = _configuration.ApiKeys;
-#pragma warning disable CS8604
+            if (validApiKeys == null || !validApiKeys.ToList().Contains(extractedApiKey))
-        if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be.
             {
                 context.Response.StatusCode = 403;
                 await context.Response.WriteAsync("Invalid API Key.");
                 return;
             }
-#pragma warning restore CS8604
+        }
 
         await _next(context);
     }

src/Shared/Models/OptionModels.cs

@@ -0,0 +1,13 @@
+namespace Shared.Models;
+
+public class ApiKeyOptions
+{
+    public string[]? ApiKeys { get; set; }
+}
+
+public class ServerOptions
+{
+    public required string BaseUri { get; set; }
+    public string? ApiKey { get; set; }
+    public string? Searchdomain { get; set; }
+}