lucretia/embeddingsearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added proper server config model, added proper apikey authorization with swagger integration, added allowlist and denylist to config

Browse Source
This commit is contained in:
LD-Reborn
2025-12-30 22:18:26 +01:00
parent b5db4bc1e4
commit bc293bf7ec
7 changed files with 126 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
src/Client/Client.cs
View File

@@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Configuration;
using System.Reflection.Metadata.Ecma335;
using Shared.Models;
using System.Net;
namespace Client;
@@ -41,8 +42,8 @@ public class Client
public async Task<EntityListResults> EntityListAsync(string searchdomain, bool returnEmbeddings = false)
{
var url = $"{baseUri}/Entities?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
return await GetUrlAndProcessJson<EntityListResults>(url);
var url = $"{baseUri}/Entities?searchdomain={HttpUtility.UrlEncode(searchdomain)}&returnEmbeddings={HttpUtility.UrlEncode(returnEmbeddings.ToString())}";
return await FetchUrlAndProcessJson<EntityListResults>(HttpMethod.Get, url);
}
public async Task<EntityIndexResult> EntityIndexAsync(List<JSONEntity> jsonEntity)
@@ -53,7 +54,7 @@ public class Client
public async Task<EntityIndexResult> EntityIndexAsync(string jsonEntity)
{
var content = new StringContent(jsonEntity, Encoding.UTF8, "application/json");
return await PutUrlAndProcessJson<EntityIndexResult>(GetUrl($"{baseUri}", "Entities", apiKey, []), content);
return await FetchUrlAndProcessJson<EntityIndexResult>(HttpMethod.Put, GetUrl($"{baseUri}", "Entities", []), content);
}
public async Task<EntityDeleteResults> EntityDeleteAsync(string entityName)
@@ -64,12 +65,12 @@ public class Client
public async Task<EntityDeleteResults> EntityDeleteAsync(string searchdomain, string entityName)
{
var url = $"{baseUri}/Entity?apiKey={HttpUtility.UrlEncode(apiKey)}&searchdomain={HttpUtility.UrlEncode(searchdomain)}&entity={HttpUtility.UrlEncode(entityName)}";
return await DeleteUrlAndProcessJson<EntityDeleteResults>(url);
return await FetchUrlAndProcessJson<EntityDeleteResults>(HttpMethod.Delete, url);
}
public async Task<SearchdomainListResults> SearchdomainListAsync()
{
return await GetUrlAndProcessJson<SearchdomainListResults>(GetUrl($"{baseUri}", "Searchdomains", apiKey, []));
return await FetchUrlAndProcessJson<SearchdomainListResults>(HttpMethod.Get, GetUrl($"{baseUri}", "Searchdomains", []));
}
public async Task<SearchdomainCreateResults> SearchdomainCreateAsync()
@@ -79,7 +80,7 @@ public class Client
public async Task<SearchdomainCreateResults> SearchdomainCreateAsync(string searchdomain, SearchdomainSettings searchdomainSettings = new())
{
return await PostUrlAndProcessJson<SearchdomainCreateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
return await FetchUrlAndProcessJson<SearchdomainCreateResults>(HttpMethod.Post, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
{
{"searchdomain", searchdomain}
}), new StringContent(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json"));
@@ -92,7 +93,7 @@ public class Client
public async Task<SearchdomainDeleteResults> SearchdomainDeleteAsync(string searchdomain)
{
return await DeleteUrlAndProcessJson<SearchdomainDeleteResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
return await FetchUrlAndProcessJson<SearchdomainDeleteResults>(HttpMethod.Delete, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
{
{"searchdomain", searchdomain}
}));
@@ -112,7 +113,7 @@ public class Client
public async Task<SearchdomainUpdateResults> SearchdomainUpdateAsync(string searchdomain, string newName, string settings = "{}")
{
return await PutUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}", "Searchdomain", apiKey, new Dictionary<string, string>()
return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}", "Searchdomain", new Dictionary<string, string>()
{
{"searchdomain", searchdomain},
{"newName", newName}
@@ -125,7 +126,7 @@ public class Client
{
{"searchdomain", searchdomain}
};
return await GetUrlAndProcessJson<SearchdomainSearchesResults>(GetUrl($"{baseUri}/Searchdomain", "Queries", apiKey, parameters));
return await FetchUrlAndProcessJson<SearchdomainSearchesResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Queries", parameters));
}
public async Task<EntityQueryResults> SearchdomainQueryAsync(string query)
@@ -143,7 +144,7 @@ public class Client
if (topN is not null) parameters.Add("topN", ((int)topN).ToString());
if (returnAttributes) parameters.Add("returnAttributes", returnAttributes.ToString());
return await PostUrlAndProcessJson<EntityQueryResults>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters), null);
return await FetchUrlAndProcessJson<EntityQueryResults>(HttpMethod.Post, GetUrl($"{baseUri}/Searchdomain", "Query", parameters), null);
}
public async Task<SearchdomainDeleteSearchResult> SearchdomainDeleteQueryAsync(string searchdomain, string query)
@@ -153,7 +154,7 @@ public class Client
{"searchdomain", searchdomain},
{"query", query}
};
return await DeleteUrlAndProcessJson<SearchdomainDeleteSearchResult>(GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters));
return await FetchUrlAndProcessJson<SearchdomainDeleteSearchResult>(HttpMethod.Delete, GetUrl($"{baseUri}/Searchdomain", "Query", parameters));
}
public async Task<SearchdomainUpdateSearchResult> SearchdomainUpdateQueryAsync(string searchdomain, string query, List<ResultItem> results)
@@ -163,8 +164,9 @@ public class Client
{"searchdomain", searchdomain},
{"query", query}
};
return await PatchUrlAndProcessJson<SearchdomainUpdateSearchResult>(
GetUrl($"{baseUri}/Searchdomain", "Query", apiKey, parameters),
return await FetchUrlAndProcessJson<SearchdomainUpdateSearchResult>(
HttpMethod.Patch,
GetUrl($"{baseUri}/Searchdomain", "Query", parameters),
new StringContent(JsonSerializer.Serialize(results), Encoding.UTF8, "application/json"));
}
@@ -174,7 +176,7 @@ public class Client
{
{"searchdomain", searchdomain}
};
return await GetUrlAndProcessJson<SearchdomainSettingsResults>(GetUrl($"{baseUri}/Searchdomain", "Settings", apiKey, parameters));
return await FetchUrlAndProcessJson<SearchdomainSettingsResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters));
}
public async Task<SearchdomainUpdateResults> SearchdomainUpdateSettingsAsync(string searchdomain, SearchdomainSettings searchdomainSettings)
@@ -184,7 +186,7 @@ public class Client
{"searchdomain", searchdomain}
};
StringContent content = new(JsonSerializer.Serialize(searchdomainSettings), Encoding.UTF8, "application/json");
return await PutUrlAndProcessJson<SearchdomainUpdateResults>(GetUrl($"{baseUri}/Searchdomain", "Settings", apiKey, parameters), content);
return await FetchUrlAndProcessJson<SearchdomainUpdateResults>(HttpMethod.Put, GetUrl($"{baseUri}/Searchdomain", "Settings", parameters), content);
}
public async Task<SearchdomainSearchCacheSizeResults> SearchdomainGetQueryCacheSizeAsync(string searchdomain)
@@ -193,7 +195,7 @@ public class Client
{
{"searchdomain", searchdomain}
};
return await GetUrlAndProcessJson<SearchdomainSearchCacheSizeResults>(GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", apiKey, parameters));
return await FetchUrlAndProcessJson<SearchdomainSearchCacheSizeResults>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Size", parameters));
}
public async Task<SearchdomainInvalidateCacheResults> SearchdomainClearQueryCache(string searchdomain)
@@ -202,7 +204,7 @@ public class Client
{
{"searchdomain", searchdomain}
};
return await PostUrlAndProcessJson<SearchdomainInvalidateCacheResults>(GetUrl($"{baseUri}/Searchdomain/QueryCache", "Clear", apiKey, parameters), null);
return await FetchUrlAndProcessJson<SearchdomainInvalidateCacheResults>(HttpMethod.Post, GetUrl($"{baseUri}/Searchdomain/QueryCache", "Clear", parameters), null);
}
public async Task<SearchdomainGetDatabaseSizeResult> SearchdomainGetDatabaseSizeAsync(string searchdomain)
@@ -211,74 +213,40 @@ public class Client
{
{"searchdomain", searchdomain}
};
return await GetUrlAndProcessJson<SearchdomainGetDatabaseSizeResult>(GetUrl($"{baseUri}/Searchdomain/Database", "Size", apiKey, parameters));
return await FetchUrlAndProcessJson<SearchdomainGetDatabaseSizeResult>(HttpMethod.Get, GetUrl($"{baseUri}/Searchdomain/Database", "Size", parameters));
}
public async Task<ServerGetModelsResult> ServerGetModelsAsync()
{
return await GetUrlAndProcessJson<ServerGetModelsResult>(GetUrl($"{baseUri}/Server", "Models", apiKey, []));
return await FetchUrlAndProcessJson<ServerGetModelsResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server", "Models", []));
}
public async Task<ServerGetEmbeddingCacheSizeResult> ServerGetEmbeddingCacheSizeAsync()
{
return await GetUrlAndProcessJson<ServerGetEmbeddingCacheSizeResult>(GetUrl($"{baseUri}/Server/EmbeddingCache", "Size", apiKey, []));
return await FetchUrlAndProcessJson<ServerGetEmbeddingCacheSizeResult>(HttpMethod.Get, GetUrl($"{baseUri}/Server/EmbeddingCache", "Size", []));
}
private static async Task<T> GetUrlAndProcessJson<T>(string url)
private async Task<T> FetchUrlAndProcessJson<T>(HttpMethod httpMethod, string url, HttpContent? content = null)
{
HttpRequestMessage requestMessage = new(httpMethod, url)
{
Content = content,
};
requestMessage.Headers.Add("X-API-KEY", apiKey);
using var client = new HttpClient();
var response = await client.GetAsync(url);
var response = await client.SendAsync(requestMessage);
string responseContent = await response.Content.ReadAsStringAsync();
if (response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Unauthorized) throw new UnauthorizedAccessException(responseContent); // TODO implement distinct exceptions
if (response.StatusCode == HttpStatusCode.InternalServerError) throw new Exception($"Request was unsuccessful due to an internal server error: {responseContent}"); // TODO implement proper InternalServerErrorException
var result = JsonSerializer.Deserialize<T>(responseContent)
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
return result;
}
private static async Task<T> PostUrlAndProcessJson<T>(string url, HttpContent? content)
{
using var client = new HttpClient();
var response = await client.PostAsync(url, content);
string responseContent = await response.Content.ReadAsStringAsync();
var result = JsonSerializer.Deserialize<T>(responseContent)
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
return result;
}
private static async Task<T> PutUrlAndProcessJson<T>(string url, HttpContent content)
{
using var client = new HttpClient();
var response = await client.PutAsync(url, content);
string responseContent = await response.Content.ReadAsStringAsync();
var result = JsonSerializer.Deserialize<T>(responseContent)
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
return result;
}
private static async Task<T> PatchUrlAndProcessJson<T>(string url, HttpContent content)
{
using var client = new HttpClient();
var response = await client.PatchAsync(url, content);
string responseContent = await response.Content.ReadAsStringAsync();
var result = JsonSerializer.Deserialize<T>(responseContent)
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
return result;
}
private static async Task<T> DeleteUrlAndProcessJson<T>(string url)
{
using var client = new HttpClient();
var response = await client.DeleteAsync(url);
string responseContent = await response.Content.ReadAsStringAsync();
var result = JsonSerializer.Deserialize<T>(responseContent)
?? throw new Exception($"Failed to deserialize JSON to type {typeof(T).Name}");
return result;
}
public static string GetUrl(string baseUri, string endpoint, string apiKey, Dictionary<string, string> parameters)
public static string GetUrl(string baseUri, string endpoint, Dictionary<string, string> parameters)
{
var uriBuilder = new UriBuilder($"{baseUri}/{endpoint}");
var query = HttpUtility.ParseQueryString(uriBuilder.Query);
if (apiKey.Length > 0) query["apiKey"] = apiKey;
foreach (var param in parameters)
{
query[param.Key] = param.Value;

4
src/Server/Controllers/AccountController.cs
View File

@@ -12,9 +12,9 @@ public class AccountController : Controller
{
private readonly SimpleAuthOptions _options;
public AccountController(IOptions<SimpleAuthOptions> options)
public AccountController(EmbeddingSearchOptions options)
{
_options = options.Value;
_options = options.SimpleAuth;
}
[HttpGet("Login")]

13
src/Server/Models/Auth.cs
View File

@@ -1,13 +0,0 @@
namespace Server.Models;
public class SimpleAuthOptions
{
public List<SimpleUser> Users { get; set; } = new();
}
public class SimpleUser
{
public string Username { get; set; } = "";
public string Password { get; set; } = "";
public string[] Roles { get; set; } = Array.Empty<string>();
}

36
src/Server/Models/ConfigModels.cs Normal file
View File

@@ -0,0 +1,36 @@
using System.Configuration;
using ElmahCore;
namespace Server.Models;
public class EmbeddingSearchOptions
{
public required ConnectionStringsSection ConnectionStrings { get; set; }
public ElmahOptions? Elmah { get; set; }
public required long EmbeddingCacheMaxCount { get; set; }
public required AiProvider[] AiProviders { get; set; }
public required SimpleAuthOptions SimpleAuth { get; set; }
public string[]? ApiKeys { get; set; }
public required bool UseHttpsRedirection { get; set; }
}
public class AiProvider
{
public required string Handler { get; set; }
public required string BaseURL { get; set; }
public required string ApiKey { get; set; }
public required string[] Allowlist { get; set; }
public required string[] Denylist { get; set; }
}
public class SimpleAuthOptions
{
public List<SimpleUser> Users { get; set; } = [];
}
public class SimpleUser
{
public string Username { get; set; } = "";
public string Password { get; set; } = "";
public string[] Roles { get; set; } = [];
}

86
src/Server/Program.cs
View File

@@ -10,11 +10,12 @@ using Server.Models;
using Server.Services;
using System.Text.Json.Serialization;
using System.Reflection;
using System.Configuration;
using Microsoft.OpenApi.Models;
var builder = WebApplication.CreateBuilder(args);
// Add services to the container.
// Add Controllers with views & string conversion for enums
builder.Services.AddControllersWithViews()
.AddJsonOptions(options =>
{
@@ -23,6 +24,12 @@ builder.Services.AddControllersWithViews()
);
});
// Add Configuration
IConfigurationSection configurationSection = builder.Configuration.GetSection("Embeddingsearch");
EmbeddingSearchOptions configuration = configurationSection.Get<EmbeddingSearchOptions>() ?? throw new ConfigurationErrorsException("Unable to start server due to an invalid configration");
builder.Services.Configure<EmbeddingSearchOptions>(configurationSection);
// Add Localization
builder.Services.AddLocalization(options => options.ResourcesPath = "Resources");
builder.Services.Configure<RequestLocalizationOptions>(options =>
@@ -43,6 +50,31 @@ builder.Services.AddSwaggerGen(c =>
var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml";
var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile);
c.IncludeXmlComments(xmlPath);
if (configuration.ApiKeys is not null)
{
c.AddSecurityDefinition("ApiKey", new OpenApiSecurityScheme
{
Description = "ApiKey must appear in header",
Type = SecuritySchemeType.ApiKey,
Name = "X-API-KEY",
In = ParameterLocation.Header,
Scheme = "ApiKeyScheme"
});
var key = new OpenApiSecurityScheme()
{
Reference = new OpenApiReference
{
Type = ReferenceType.SecurityScheme,
Id = "ApiKey"
},
In = ParameterLocation.Header
};
var requirement = new OpenApiSecurityRequirement
{
{ key, []}
};
c.AddSecurityRequirement(requirement);
}
});
Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(builder.Configuration)
@@ -58,7 +90,12 @@ builder.Services.AddHealthChecks()
builder.Services.AddElmah<XmlFileErrorLog>(Options =>
{
Options.LogPath = builder.Configuration.GetValue<string>("Embeddingsearch:Elmah:LogFolder") ?? "~/logs";
Options.OnPermissionCheck = context =>
context.User.Claims.Any(claim =>
claim.Value.Equals("Admin", StringComparison.OrdinalIgnoreCase)
|| claim.Value.Equals("Elmah", StringComparison.OrdinalIgnoreCase)
);
Options.LogPath = configuration.Elmah?.LogPath ?? "~/logs";
});
builder.Services
@@ -76,35 +113,11 @@ builder.Services.AddAuthorization(options =>
policy => policy.RequireRole("Admin"));
});
IConfigurationSection simpleAuthSection = builder.Configuration.GetSection("Embeddingsearch:SimpleAuth");
if (simpleAuthSection.Exists()) builder.Services.Configure<SimpleAuthOptions>(simpleAuthSection);
var app = builder.Build();
List<string>? allowedIps = builder.Configuration.GetSection("Embeddingsearch:Elmah:AllowedHosts")
.Get<List<string>>();
app.Use(async (context, next) =>
{
bool requestIsElmah = context.Request.Path.StartsWithSegments("/elmah");
bool requestIsSwagger = context.Request.Path.StartsWithSegments("/swagger");
if (requestIsElmah || requestIsSwagger)
{
var remoteIp = context.Connection.RemoteIpAddress?.ToString();
bool blockRequest = allowedIps is null
|| remoteIp is null
|| !allowedIps.Contains(remoteIp);
if (blockRequest)
{
context.Response.StatusCode = 403;
await context.Response.WriteAsync("Forbidden");
return;
}
}
await next();
});
app.UseAuthentication();
app.UseAuthorization();
app.UseElmah();
@@ -123,14 +136,14 @@ bool IsDevelopment = app.Environment.IsDevelopment();
bool useSwagger = app.Configuration.GetValue<bool>("UseSwagger");
bool? UseMiddleware = app.Configuration.GetValue<bool?>("UseMiddleware");
// Configure the HTTP request pipeline.
if (IsDevelopment || useSwagger)
{
app.UseSwagger();
app.UseSwaggerUI();
app.UseSwaggerUI(options =>
{
options.EnablePersistAuthorization();
});
//app.UseElmahExceptionPage(); // Messes with JSON response for API calls. Leaving this here so I don't accidentally put this in again later on.
}
if (UseMiddleware == true && !IsDevelopment)
if (configuration.ApiKeys is not null)
{
app.UseMiddleware<Shared.ApiKeyMiddleware>();
}
@@ -143,9 +156,6 @@ var localizationOptions = new RequestLocalizationOptions()
.AddSupportedUICultures(supportedCultures);
app.UseRequestLocalization(localizationOptions);
app.UseAuthentication();
app.UseAuthorization();
app.MapControllers();
app.UseStaticFiles();

15
src/Server/appsettings.Development.json
View File

@@ -18,22 +18,21 @@
"SQL": "server=localhost;database=embeddingsearch;uid=embeddingsearch;pwd=somepassword!;"
},
"Elmah": {
"AllowedHosts": [
"127.0.0.1",
"::1",
"172.17.0.1"
]
"LogPath": "~/logs"
},
"EmbeddingCacheMaxCount": 10000000,
"AiProviders": {
"ollama": {
"handler": "ollama",
"baseURL": "http://localhost:11434"
"baseURL": "http://192.168.0.101:11434",
"Allowlist": ["*"],
"Denylist": ["qwen3-coder:latest", "qwen3:0.6b", "deepseek-v3.1:671b-cloud", "qwen3-vl", "deepseek-ocr"]
},
"localAI": {
"handler": "openai",
"baseURL": "http://localhost:8080",
"ApiKey": "Some API key here"
"ApiKey": "Some API key here",
"Allowlist": ["*"],
"Denylist": ["cross-encoder", "kitten-tts", "jina-reranker-v1-tiny-en", "whisper-small", "qwen3-vl-2b-instruct"]
}
},
"SimpleAuth": {

9
src/Server/appsettings.json
View File

@@ -16,14 +16,5 @@
"Application": "Embeddingsearch.Server"
}
},
"EmbeddingsearchIndexer": {
"Elmah": {
"AllowedHosts": [
"127.0.0.1",
"::1"
],
"LogFolder": "./logs"
}
},
"AllowedHosts": "*"
}