lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added allowlist and denylist, fixed patchy configuration with proper options models, fixed api middleware authorization issues

Browse Source
This commit is contained in:
LD-Reborn
2025-12-31 03:47:40 +01:00
parent 8d56883e7e
commit aa95308f61
13 changed files with 113 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/Client/Client.cs
View File

@@ -9,6 +9,7 @@ using Microsoft.Extensions.Configuration;
using System.Reflection.Metadata.Ecma335;
using Shared.Models;
using System.Net;
using Microsoft.Extensions.Options;
namespace Client;
@@ -25,12 +26,12 @@ public class Client
this.searchdomain = searchdomain;
}
public Client(IConfiguration configuration)
public Client(IOptions<ServerOptions> configuration)
{
string? baseUri = configuration.GetSection("Embeddingsearch").GetValue<string>("BaseUri");
string? apiKey = configuration.GetSection("Embeddingsearch").GetValue<string>("ApiKey");
string? searchdomain = configuration.GetSection("Embeddingsearch").GetValue<string>("Searchdomain");
this.baseUri = baseUri ?? "";
string baseUri = configuration.Value.BaseUri;
string? apiKey = configuration.Value.ApiKey;
string? searchdomain = configuration.Value.Searchdomain;
this.baseUri = baseUri;
this.apiKey = apiKey ?? "";
this.searchdomain = searchdomain ?? "";
}

1
src/Indexer/Indexer.csproj
View File

@@ -15,6 +15,7 @@
<PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
<PackageReference Include="System.Configuration.ConfigurationManager" Version="9.0.3" />
<PackageReference Include="Python" Version="3.13.3" />
<PackageReference Include="Pythonnet" Version="3.0.5" />
</ItemGroup>

9
src/Indexer/Models/OptionModels.cs Normal file
View File

@@ -0,0 +1,9 @@
using Shared.Models;
namespace Indexer.Models;
public class IndexerOptions : ApiKeyOptions
{
public required WorkerConfig[] Workers { get; set; }
public required ServerOptions Server { get; set;}
public required string PythonRuntime { get; set; } = "libpython3.13.so";
}

4
src/Indexer/Models/ScriptModels.cs
View File

@@ -15,11 +15,11 @@ public class ScriptToolSet
public Client.Client Client;
public LoggerWrapper Logger;
public ICallbackInfos? CallbackInfos;
public IConfiguration Configuration;
public IndexerOptions Configuration;
public CancellationToken CancellationToken;
public string Name;
public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IConfiguration configuration, CancellationToken cancellationToken, string name)
public ScriptToolSet(string filePath, Client.Client client, ILogger<WorkerManager> logger, IndexerOptions configuration, CancellationToken cancellationToken, string name)
{
Configuration = configuration;
Name = name;

8
src/Indexer/Program.cs
View File

@@ -6,6 +6,8 @@ using ElmahCore.Mvc;
using ElmahCore.Mvc.Logger;
using Serilog;
using Quartz;
using System.Configuration;
using Shared.Models;
var builder = WebApplication.CreateBuilder(args);
@@ -21,6 +23,12 @@ Log.Logger = new LoggerConfiguration()
builder.Logging.AddSerilog();
builder.Services.AddHttpContextAccessor();
builder.Services.AddSingleton<IConfigurationRoot>(builder.Configuration);
IConfigurationSection configurationSection = builder.Configuration.GetSection("Indexer");
IndexerOptions configuration = configurationSection.Get<IndexerOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
builder.Services.Configure<IndexerOptions>(configurationSection);
builder.Services.Configure<ServerOptions>(configurationSection.GetSection("Server"));
builder.Services.Configure<ApiKeyOptions>(configurationSection);
builder.Services.AddSingleton<Client.Client>();
builder.Services.AddSingleton<WorkerManager>();
builder.Services.AddHostedService<IndexerService>();

5
src/Indexer/ScriptContainers/PythonScriptContainer.cs
View File

@@ -15,11 +15,8 @@ public class PythonScriptable : IScriptContainer
public ILogger _logger { get; set; }
public PythonScriptable(ScriptToolSet toolSet, ILogger logger)
{
string? runtime = toolSet.Configuration.GetValue<string>("EmbeddingsearchIndexer:PythonRuntime");
if (runtime is not null)
{
string runtime = toolSet.Configuration.PythonRuntime;
Runtime.PythonDLL ??= runtime;
}
_logger = logger;
SourceLoaded = false;
if (!PythonEngine.IsInitialized)

24
src/Indexer/WorkerManager.cs
View File

@@ -1,21 +1,22 @@
using Indexer.Exceptions;
using Indexer.Models;
using Indexer.ScriptContainers;
using Microsoft.Extensions.Options;
public class WorkerManager
{
public Dictionary<string, Worker> Workers;
public List<Type> types;
private readonly ILogger<WorkerManager> _logger;
private readonly IConfiguration _configuration;
private readonly IndexerOptions _configuration;
private readonly Client.Client client;
public WorkerManager(ILogger<WorkerManager> logger, IConfiguration configuration, Client.Client client)
public WorkerManager(ILogger<WorkerManager> logger, IOptions<IndexerOptions> configuration, Client.Client client)
{
Workers = [];
types = [typeof(PythonScriptable), typeof(CSharpScriptable)];
_logger = logger;
_configuration = configuration;
_configuration = configuration.Value;
this.client = client;
}
@@ -23,28 +24,13 @@ public class WorkerManager
{
_logger.LogInformation("Initializing workers");
// Load and configure all workers
var sectionMain = _configuration.GetSection("EmbeddingsearchIndexer");
if (!sectionMain.Exists())
{
_logger.LogCritical("Unable to load section \"EmbeddingsearchIndexer\"");
throw new IndexerConfigurationException("Unable to load section \"EmbeddingsearchIndexer\"");
}
WorkerCollectionConfig? sectionWorker = (WorkerCollectionConfig?)sectionMain.Get(typeof(WorkerCollectionConfig)); //GetValue<WorkerCollectionConfig>("Worker");
if (sectionWorker is not null)
{
foreach (WorkerConfig workerConfig in sectionWorker.Worker)
foreach (WorkerConfig workerConfig in _configuration.Workers)
{
CancellationTokenSource cancellationTokenSource = new();
ScriptToolSet toolSet = new(workerConfig.Script, client, _logger, _configuration, cancellationTokenSource.Token, workerConfig.Name);
InitializeWorker(toolSet, workerConfig, cancellationTokenSource);
}
}
else
{
_logger.LogCritical("Unable to load section \"Worker\"");
throw new IndexerConfigurationException("Unable to load section \"Worker\"");
}
_logger.LogInformation("Initialized workers");
}

35
src/Server/AIProvider.cs
View File

@@ -1,24 +1,25 @@
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Server.Exceptions;
using Server.Models;
namespace Server;
public class AIProvider
{
private readonly ILogger<AIProvider> _logger;
private readonly IConfiguration _configuration;
public AIProvidersConfiguration aIProvidersConfiguration;
private readonly EmbeddingSearchOptions _configuration;
public Dictionary<string, AiProvider> aIProvidersConfiguration;
public AIProvider(ILogger<AIProvider> logger, IConfiguration configuration)
public AIProvider(ILogger<AIProvider> logger, IOptions<EmbeddingSearchOptions> configuration)
{
_logger = logger;
_configuration = configuration;
AIProvidersConfiguration? retrievedAiProvidersConfiguration = _configuration
.GetSection("Embeddingsearch")
.Get<AIProvidersConfiguration>();
_configuration = configuration.Value;
Dictionary<string, AiProvider>? retrievedAiProvidersConfiguration = _configuration.AiProviders;
if (retrievedAiProvidersConfiguration is null)
{
_logger.LogCritical("Unable to build AIProvidersConfiguration. Please check your configuration.");
@@ -35,8 +36,8 @@ public class AIProvider
Uri uri = new(modelUri);
string provider = uri.Scheme;
string model = uri.AbsolutePath;
AIProviderConfiguration? aIProvider = aIProvidersConfiguration.AiProviders
.FirstOrDefault(x => String.Equals(x.Key.ToLower(), provider.ToLower()))
AiProvider? aIProvider = aIProvidersConfiguration
.FirstOrDefault(x => string.Equals(x.Key.ToLower(), provider.ToLower()))
.Value;
if (aIProvider is null)
{
@@ -119,12 +120,12 @@ public class AIProvider
public string[] GetModels()
{
var aIProviders = aIProvidersConfiguration.AiProviders;
var aIProviders = aIProvidersConfiguration;
List<string> results = [];
foreach (KeyValuePair<string, AIProviderConfiguration> aIProviderKV in aIProviders)
foreach (KeyValuePair<string, AiProvider> aIProviderKV in aIProviders)
{
string aIProviderName = aIProviderKV.Key;
AIProviderConfiguration aIProvider = aIProviderKV.Value;
AiProvider aIProvider = aIProviderKV.Value;
using var httpClient = new HttpClient();
@@ -178,9 +179,14 @@ public class AIProvider
foreach (string? result in aIProviderResult)
{
if (result is null) continue;
bool isInAllowList = ElementMatchesAnyRegexInList(result, aIProvider.Allowlist);
bool isInDenyList = ElementMatchesAnyRegexInList(result, aIProvider.Denylist);
if (isInAllowList && !isInDenyList)
{
results.Add(aIProviderName + ":" + result);
}
}
}
catch (Exception ex)
{
_logger.LogError("Unable to parse the response to valid models. {ex.Message}", [ex.Message]);
@@ -189,6 +195,11 @@ public class AIProvider
}
return [.. results];
}
private static bool ElementMatchesAnyRegexInList(string element, string[] list)
{
return list?.Any(pattern => pattern != null && Regex.IsMatch(element, pattern)) ?? false;
}
}
public class AIProvidersConfiguration

4
src/Server/Controllers/AccountController.cs
View File

@@ -12,9 +12,9 @@ public class AccountController : Controller
{
private readonly SimpleAuthOptions _options;
public AccountController(EmbeddingSearchOptions options)
public AccountController(IOptions<EmbeddingSearchOptions> options)
{
_options = options.SimpleAuth;
_options = options.Value.SimpleAuth;
}
[HttpGet("Login")]

8
src/Server/Models/ConfigModels.cs
View File

@@ -1,16 +1,16 @@
using System.Configuration;
using ElmahCore;
using Shared.Models;
namespace Server.Models;
public class EmbeddingSearchOptions
public class EmbeddingSearchOptions : ApiKeyOptions
{
public required ConnectionStringsSection ConnectionStrings { get; set; }
public ElmahOptions? Elmah { get; set; }
public required long EmbeddingCacheMaxCount { get; set; }
public required AiProvider[] AiProviders { get; set; }
public required Dictionary<string, AiProvider> AiProviders { get; set; }
public required SimpleAuthOptions SimpleAuth { get; set; }
public string[]? ApiKeys { get; set; }
public required bool UseHttpsRedirection { get; set; }
}
@@ -18,7 +18,7 @@ public class AiProvider
{
public required string Handler { get; set; }
public required string BaseURL { get; set; }
public required string ApiKey { get; set; }
public string? ApiKey { get; set; }
public required string[] Allowlist { get; set; }
public required string[] Denylist { get; set; }
}

18
src/Server/Program.cs
View File

@@ -12,6 +12,7 @@ using System.Text.Json.Serialization;
using System.Reflection;
using System.Configuration;
using Microsoft.OpenApi.Models;
using Shared.Models;
var builder = WebApplication.CreateBuilder(args);
@@ -29,6 +30,7 @@ IConfigurationSection configurationSection = builder.Configuration.GetSection("E
EmbeddingSearchOptions configuration = configurationSection.Get<EmbeddingSearchOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
builder.Services.Configure<EmbeddingSearchOptions>(configurationSection);
builder.Services.Configure<ApiKeyOptions>(configurationSection);
// Add Localization
builder.Services.AddLocalization(options => options.ResourcesPath = "Resources");
@@ -133,8 +135,6 @@ app.MapHealthChecks("/healthz/AIProvider", new Microsoft.AspNetCore.Diagnostics.
});
bool IsDevelopment = app.Environment.IsDevelopment();
bool useSwagger = app.Configuration.GetValue<bool>("UseSwagger");
bool? UseMiddleware = app.Configuration.GetValue<bool?>("UseMiddleware");
app.UseSwagger();
app.UseSwaggerUI(options =>
@@ -145,7 +145,19 @@ app.UseSwaggerUI(options =>
if (configuration.ApiKeys is not null)
{
app.UseMiddleware<Shared.ApiKeyMiddleware>();
app.UseWhen(context =>
{
RouteData routeData = context.GetRouteData();
string controllerName = routeData.Values["controller"]?.ToString() ?? "StaticFile";
if (controllerName == "Account" || controllerName == "Home" || controllerName == "StaticFile")
{
return false;
}
return true;
}, appBuilder =>
{
appBuilder.UseMiddleware<Shared.ApiKeyMiddleware>();
});
}
// Add localization

17
src/Shared/ApiKeyMiddleware.cs
View File

@@ -1,21 +1,25 @@
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Primitives;
using Shared.Models;
namespace Shared;
public class ApiKeyMiddleware
{
private readonly RequestDelegate _next;
private readonly IConfiguration _configuration;
private readonly ApiKeyOptions _configuration;
public ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration)
public ApiKeyMiddleware(RequestDelegate next, IOptions<ApiKeyOptions> configuration)
{
_next = next;
_configuration = configuration;
_configuration = configuration.Value;
}
public async Task InvokeAsync(HttpContext context)
{
if (!(context.User.Identity?.IsAuthenticated ?? false))
{
if (!context.Request.Headers.TryGetValue("X-API-KEY", out StringValues extractedApiKey))
{
@@ -24,15 +28,14 @@ public class ApiKeyMiddleware
return;
}
var validApiKeys = _configuration.GetSection("Embeddingsearch").GetSection("ApiKeys").Get<List<string>>();
#pragma warning disable CS8604
if (validApiKeys == null || !validApiKeys.Contains(extractedApiKey)) // CS8604 extractedApiKey is not null here, but the compiler still thinks that it might be.
string[]? validApiKeys = _configuration.ApiKeys;
if (validApiKeys == null || !validApiKeys.ToList().Contains(extractedApiKey))
{
context.Response.StatusCode = 403;
await context.Response.WriteAsync("Invalid API Key.");
return;
}
#pragma warning restore CS8604
}
await _next(context);
}

13
src/Shared/Models/OptionModels.cs Normal file
View File

@@ -0,0 +1,13 @@
namespace Shared.Models;
public class ApiKeyOptions
{
public string[]? ApiKeys { get; set; }
}
public class ServerOptions
{
public required string BaseUri { get; set; }
public string? ApiKey { get; set; }
public string? Searchdomain { get; set; }
}