From e4a711fcbd290dd111e2af1f5fef4544998d00e4 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sat, 20 Dec 2025 20:13:28 +0100 Subject: [PATCH 01/18] Added filtering query output to top n elements --- src/Server/Controllers/EntityController.cs | 4 ++-- src/Server/Searchdomain.cs | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Server/Controllers/EntityController.cs b/src/Server/Controllers/EntityController.cs index 4765311..36ac1a8 100644 --- a/src/Server/Controllers/EntityController.cs +++ b/src/Server/Controllers/EntityController.cs @@ -25,7 +25,7 @@ public class EntityController : ControllerBase } [HttpGet("Query")] - public ActionResult Query(string searchdomain, string query) + public ActionResult Query(string searchdomain, string query, int? topN) { Searchdomain searchdomain_; try @@ -40,7 +40,7 @@ public class EntityController : ControllerBase _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]); return Ok(new EntityQueryResults() {Results = [], Success = false, Message = "Unable to retrieve the searchdomain - it likely exists, but some other error happened." }); } - var results = searchdomain_.Search(query); + List<(float, string)> results = searchdomain_.Search(query, topN); List queryResults = [.. results.Select(r => new EntityQueryResult { Name = r.Item2, diff --git a/src/Server/Searchdomain.cs b/src/Server/Searchdomain.cs index 019eb79..7566363 100644 --- a/src/Server/Searchdomain.cs +++ b/src/Server/Searchdomain.cs @@ -154,7 +154,7 @@ public class Searchdomain embeddingCache = []; // TODO remove this and implement proper remediation to improve performance } - public List<(float, string)> Search(string query) + public List<(float, string)> Search(string query, int? topN = null) { if (searchCache.TryGetValue(query, out DateTimedSearchResult cachedResult)) { @@ -190,9 +190,14 @@ public class Searchdomain } result.Add((entity.probMethod(datapointProbs), entity.name)); } - List<(float, string)> results = [.. result.OrderByDescending(s => s.Item1)]; + IEnumerable<(float, string)> sortedResults = result.OrderByDescending(s => s.Item1); + if (topN is not null) + { + sortedResults = sortedResults.Take(topN ?? 0); + } + List<(float, string)> results = [.. sortedResults]; List searchResult = new( - [.. results.Select(r => + [.. sortedResults.Select(r => new ResultItem(r.Item1, r.Item2 ))] ); searchCache[query] = new DateTimedSearchResult(DateTime.Now, searchResult); From ae2b9e9f41d99c11922d4458af16f1da5f2eac11 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sat, 20 Dec 2025 22:02:57 +0100 Subject: [PATCH 02/18] Added models listing endpoint --- src/Server/AIProvider.cs | 73 ++++++++++++++++++++++ src/Server/Controllers/ServerController.cs | 38 +++++++++++ src/Shared/Models/ServerModels.cs | 15 +++++ 3 files changed, 126 insertions(+) create mode 100644 src/Server/Controllers/ServerController.cs create mode 100644 src/Shared/Models/ServerModels.cs diff --git a/src/Server/AIProvider.cs b/src/Server/AIProvider.cs index 0a244a2..1e5c485 100644 --- a/src/Server/AIProvider.cs +++ b/src/Server/AIProvider.cs @@ -116,6 +116,79 @@ public class AIProvider throw; } } + + public string[] GetModels() + { + var aIProviders = aIProvidersConfiguration.AiProviders; + List results = []; + foreach (KeyValuePair aIProviderKV in aIProviders) + { + string aIProviderName = aIProviderKV.Key; + AIProviderConfiguration aIProvider = aIProviderKV.Value; + + using var httpClient = new HttpClient(); + + string modelNameJsonPath = ""; + Uri baseUri = new(aIProvider.BaseURL); + Uri requestUri; + string[][] requestHeaders = []; + switch (aIProvider.Handler) + { + case "ollama": + modelNameJsonPath = "$.models[*].name"; + requestUri = new Uri(baseUri, "/api/tags"); + break; + case "openai": + modelNameJsonPath = "$.data[*].id"; + requestUri = new Uri(baseUri, "/v1/models"); + if (aIProvider.ApiKey is not null) + { + requestHeaders = [ + ["Authorization", $"Bearer {aIProvider.ApiKey}"] + ]; + } + break; + default: + _logger.LogError("Unknown handler {aIProvider.Handler} in AiProvider {provider}.", [aIProvider.Handler, aIProvider]); + throw new ServerConfigurationException($"Unknown handler {aIProvider.Handler} in AiProvider {aIProvider}."); + } + + var request = new HttpRequestMessage() + { + RequestUri = requestUri, + Method = HttpMethod.Post + }; + + foreach (var header in requestHeaders) + { + request.Headers.Add(header[0], header[1]); + } + HttpResponseMessage response = httpClient.GetAsync(requestUri).Result; + string responseContent = response.Content.ReadAsStringAsync().Result; + try + { + JObject responseContentJson = JObject.Parse(responseContent); + IEnumerable? responseContentTokens = responseContentJson.SelectTokens(modelNameJsonPath); + if (responseContentTokens is null) + { + _logger.LogError("Unable to select tokens using JSONPath {modelNameJsonPath} for string: {responseContent}.", [modelNameJsonPath, responseContent]); + throw new JSONPathSelectionException(modelNameJsonPath, responseContent); + } + IEnumerable aIProviderResult = responseContentTokens.Values(); + foreach (string? result in aIProviderResult) + { + if (result is null) continue; + results.Add(aIProviderName + ":" + result); + } + } + catch (Exception ex) + { + _logger.LogError("Unable to parse the response to valid models. {ex.Message}", [ex.Message]); + throw; + } + } + return [.. results]; + } } public class AIProvidersConfiguration diff --git a/src/Server/Controllers/ServerController.cs b/src/Server/Controllers/ServerController.cs new file mode 100644 index 0000000..ca4d9b8 --- /dev/null +++ b/src/Server/Controllers/ServerController.cs @@ -0,0 +1,38 @@ +namespace Server.Controllers; + +using System.Text.Json; +using ElmahCore; +using Microsoft.AspNetCore.Mvc; +using Server.Exceptions; +using Server.Helper; +using Shared.Models; + +[ApiController] +[Route("[controller]")] +public class ServerController : ControllerBase +{ + private readonly ILogger _logger; + private readonly IConfiguration _config; + private AIProvider _aIProvider; + + public ServerController(ILogger logger, IConfiguration config, AIProvider aIProvider) + { + _logger = logger; + _config = config; + _aIProvider = aIProvider; + } + + [HttpGet("GetModels")] + public ActionResult GetModels() + { + try + { + string[] models = _aIProvider.GetModels(); + return new ServerGetModelsResult() { Models = models, Success = true }; + } catch (Exception ex) + { + _logger.LogError("Unable to get models due to exception {ex.Message} - {ex.StackTrace}", [ex.Message, ex.StackTrace]); + return new ServerGetModelsResult() { Success = false, Message = ex.Message}; + } + } +} diff --git a/src/Shared/Models/ServerModels.cs b/src/Shared/Models/ServerModels.cs new file mode 100644 index 0000000..278582e --- /dev/null +++ b/src/Shared/Models/ServerModels.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace Shared.Models; + +public class ServerGetModelsResult +{ + [JsonPropertyName("Success")] + public required bool Success { get; set; } + + [JsonPropertyName("Message")] + public string? Message { get; set; } + + [JsonPropertyName("Models")] + public string[]? Models { get; set; } +} \ No newline at end of file From 20fc309b07eefe925a8d0c12ba95c58f38f0d02a Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sun, 21 Dec 2025 00:15:28 +0100 Subject: [PATCH 03/18] Added SimilarityMethodsEnum --- src/Server/SimilarityMethods.cs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/Server/SimilarityMethods.cs b/src/Server/SimilarityMethods.cs index 9521825..9d02c22 100644 --- a/src/Server/SimilarityMethods.cs +++ b/src/Server/SimilarityMethods.cs @@ -21,20 +21,28 @@ public class SimilarityMethod } } +public enum SimilarityMethodEnum +{ + Cosine, + Euclidian, + Manhattan, + Pearson +} + public static class SimilarityMethods { public delegate float similarityMethodProtoDelegate(float[] vector1, float[] vector2); public delegate float similarityMethodDelegate(float[] vector1, float[] vector2); - public static readonly Dictionary probMethods; + public static readonly Dictionary probMethods; static SimilarityMethods() { - probMethods = new Dictionary + probMethods = new Dictionary { - ["Cosine"] = CosineSimilarity, - ["Euclidian"] = EuclidianDistance, - ["Manhattan"] = ManhattanDistance, - ["Pearson"] = PearsonCorrelation + [SimilarityMethodEnum.Cosine] = CosineSimilarity, + [SimilarityMethodEnum.Euclidian] = EuclidianDistance, + [SimilarityMethodEnum.Manhattan] = ManhattanDistance, + [SimilarityMethodEnum.Pearson] = PearsonCorrelation }; } @@ -42,7 +50,12 @@ public static class SimilarityMethods { string methodName = name; - if (!probMethods.TryGetValue(methodName, out similarityMethodProtoDelegate? method)) + SimilarityMethodEnum probMethodEnum = (SimilarityMethodEnum)Enum.Parse( + typeof(SimilarityMethodEnum), + methodName + ); + + if (!probMethods.TryGetValue(probMethodEnum, out similarityMethodProtoDelegate? method)) { return null; } From 8329beeca7be99365f2b31e6d6fb6cbe1da9e8b2 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sun, 21 Dec 2025 00:57:47 +0100 Subject: [PATCH 04/18] Added entity creation --- src/Server/Controllers/EntityController.cs | 2 +- src/Server/Views/Home/Index.cshtml | 305 +++++++++++++++++++++ 2 files changed, 306 insertions(+), 1 deletion(-) diff --git a/src/Server/Controllers/EntityController.cs b/src/Server/Controllers/EntityController.cs index 36ac1a8..e5866ff 100644 --- a/src/Server/Controllers/EntityController.cs +++ b/src/Server/Controllers/EntityController.cs @@ -85,7 +85,7 @@ public class EntityController : ControllerBase } catch (Exception ex) { if (ex.InnerException is not null) ex = ex.InnerException; - _logger.LogError("Unable to index the provided entities. {ex.Message}", [ex.Message]); + _logger.LogError("Unable to index the provided entities. {ex.Message} - {ex.StackTrace}", [ex.Message, ex.StackTrace]); return Ok(new EntityIndexResult() { Success = false, Message = ex.Message }); } diff --git a/src/Server/Views/Home/Index.cshtml b/src/Server/Views/Home/Index.cshtml index 022967b..59bac16 100644 --- a/src/Server/Views/Home/Index.cshtml +++ b/src/Server/Views/Home/Index.cshtml @@ -1,11 +1,17 @@ @using Server.Models @using System.Web @using Server.Services +@using Server + @inject LocalizationService T +@inject AIProvider AIProvider @model HomeIndexViewModel @{ ViewData["Title"] = "Home Page"; int i = 0; + string[] probMethods = [.. Enum.GetValues(typeof(ProbMethodEnum)).Cast().Select(e => e.ToString())]; + string[] similarityMethods = [.. Enum.GetValues(typeof(SimilarityMethodEnum)).Cast().Select(e => e.ToString())]; + string[] models = AIProvider.GetModels(); Dictionary domains = []; Model.Searchdomains.ForEach(domain => { @@ -130,6 +136,7 @@ + @T["Add new entity"] @@ -314,11 +321,90 @@ + + + \ No newline at end of file From bf6265882c0d2f73909238916e552313e40db805 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sun, 21 Dec 2025 00:58:28 +0100 Subject: [PATCH 05/18] Improved size estimation display units --- src/Server/Views/Home/Index.cshtml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Server/Views/Home/Index.cshtml b/src/Server/Views/Home/Index.cshtml index 59bac16..0fc4a0e 100644 --- a/src/Server/Views/Home/Index.cshtml +++ b/src/Server/Views/Home/Index.cshtml @@ -837,7 +837,7 @@ if (cacheUtilization != null && cacheUtilization.SearchCacheSizeBytes != null) { document.querySelector('#cacheUtilization').innerText = - `${(cacheUtilization.SearchCacheSizeBytes / (1024 * 1024)).toFixed(2)}MiB`; + `${NumberOfBytesAsHumanReadable(cacheUtilization.SearchCacheSizeBytes)}`; } else { // TODO add toast console.error('Failed to fetch searchdomain cache utilization'); @@ -848,7 +848,7 @@ if (databaseUtilization != null && databaseUtilization.SearchdomainDatabaseSizeBytes != null) { document.querySelector('#databaseUtilization').innerText = - `${(databaseUtilization.SearchdomainDatabaseSizeBytes / (1024 * 1024)).toFixed(2)}MiB`; + `${NumberOfBytesAsHumanReadable(databaseUtilization.SearchdomainDatabaseSizeBytes)}`; } else { // TODO add toast console.error('Failed to fetch searchdomain database utilization'); @@ -1065,6 +1065,16 @@ modal.show(); } + function NumberOfBytesAsHumanReadable(bytes, decimals = 2) { + if (bytes === 0) return '0 B'; + if (bytes > 1.20892581961*(10**27)) return "∞ B"; + const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; // + const unitIndex = Math.floor(Math.log2(bytes) / 10); + const unit = units[Math.min(unitIndex, units.length - 1)]; + const value = bytes / Math.pow(1024, unitIndex); + + return `${value.toFixed(decimals)} ${unit}`; + } function getEntityAttributes(containerName = 'createEntityAttributesContainer') { const container = document.getElementById(containerName); From 29f910a65a0c76292e6f5c5af87e891e0251d9c4 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Sun, 21 Dec 2025 00:58:44 +0100 Subject: [PATCH 06/18] Fixed typos --- src/Server/Views/Home/Index.cshtml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Server/Views/Home/Index.cshtml b/src/Server/Views/Home/Index.cshtml index 0fc4a0e..15dfbf6 100644 --- a/src/Server/Views/Home/Index.cshtml +++ b/src/Server/Views/Home/Index.cshtml @@ -63,7 +63,7 @@

@T["Settings"]

- +
@@ -305,7 +305,7 @@ - +
+ + \ No newline at end of file From ee12986fef895d703652e8ab3e20672308fac035 Mon Sep 17 00:00:00 2001 From: LD-Reborn Date: Mon, 22 Dec 2025 14:22:16 +0100 Subject: [PATCH 16/18] Added query deleting --- .../Controllers/SearchdomainController.cs | 29 +++++++ src/Server/Views/Home/Index.cshtml | 76 +++++++++++++++++-- src/Shared/Models/SearchdomainResults.cs | 9 +++ 3 files changed, 109 insertions(+), 5 deletions(-) diff --git a/src/Server/Controllers/SearchdomainController.cs b/src/Server/Controllers/SearchdomainController.cs index 6d75dbf..20f0f2e 100644 --- a/src/Server/Controllers/SearchdomainController.cs +++ b/src/Server/Controllers/SearchdomainController.cs @@ -1,5 +1,6 @@ using System.Text.Json; using ElmahCore; +using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.AspNetCore.Mvc; using Server.Exceptions; using Server.Helper; @@ -156,6 +157,34 @@ public class SearchdomainController : ControllerBase return Ok(new SearchdomainSearchesResults() { Searches = searchCache, Success = true }); } + [HttpDelete("Searches")] + public ActionResult DeleteSearch(string searchdomain, string query) + { + Searchdomain searchdomain_; + try + { + searchdomain_ = _domainManager.GetSearchdomain(searchdomain); + } + catch (SearchdomainNotFoundException) + { + _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - it likely does not exist yet", [searchdomain]); + return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = "Searchdomain not found" }); + } + catch (Exception ex) + { + _logger.LogError("Unable to retrieve the searchdomain {searchdomain} - {ex.Message} - {ex.StackTrace}", [searchdomain, ex.Message, ex.StackTrace]); + return Ok(new SearchdomainDeleteSearchResult() { Success = false, Message = ex.Message }); + } + Dictionary searchCache = searchdomain_.searchCache; + bool containsKey = searchCache.ContainsKey(query); + if (containsKey) + { + searchCache.Remove(query); + return Ok(new SearchdomainDeleteSearchResult() {Success = true}); + } + return Ok(new SearchdomainDeleteSearchResult() {Success = false, Message = "Query not found in search cache"}); + } + [HttpGet("GetSettings")] public ActionResult GetSettings(string searchdomain) { diff --git a/src/Server/Views/Home/Index.cshtml b/src/Server/Views/Home/Index.cshtml index 7848b8b..240cd0e 100644 --- a/src/Server/Views/Home/Index.cshtml +++ b/src/Server/Views/Home/Index.cshtml @@ -502,6 +502,33 @@ + + + \ No newline at end of file