Merge pull request #64 from LD-Reborn/62-add-an-embedding-cache-size-label-to-front-end

Added home page dashboard, added embedding cache size estimation and …

src/Client/Client.cs

@@ -219,6 +219,11 @@ public class Client
             return await GetUrlAndProcessJson<ServerGetModelsResult>(GetUrl($"{baseUri}/Server", "Models", apiKey, []));
         }
 
+        public async Task<ServerGetEmbeddingCacheSizeResult> ServerGetEmbeddingCacheSizeAsync()
+        {
+            return await GetUrlAndProcessJson<ServerGetEmbeddingCacheSizeResult>(GetUrl($"{baseUri}/Server/EmbeddingCache", "Size", apiKey, []));
+        }
+
         private static async Task<T> GetUrlAndProcessJson<T>(string url)
         {
             using var client = new HttpClient();

src/Server/Controllers/Frontend/HomeController.cs

@@ -8,7 +8,7 @@ using Server.Models;
 namespace Server.Controllers;
 
 [ApiExplorerSettings(IgnoreApi = true)]
-[Route("/")]
+[Route("[Controller]")]
 public class HomeController : Controller
 {
     private readonly ILogger<EntityController> _logger;
@@ -23,6 +23,13 @@ public class HomeController : Controller
     [Authorize]
     [HttpGet("/")]
     public IActionResult Index()
+    {
+        return View();
+    }
+
+    [Authorize]
+    [HttpGet("Searchdomains")]
+    public IActionResult Searchdomains()
     {
         HomeIndexViewModel viewModel = new()
         {

src/Server/Controllers/ServerController.cs

@@ -1,6 +1,8 @@
 namespace Server.Controllers;
 
+using System.Reflection;
 using System.Text.Json;
+using AdaptiveExpressions;
 using ElmahCore;
 using Microsoft.AspNetCore.Mvc;
 using Server.Exceptions;
@@ -14,12 +16,14 @@ public class ServerController : ControllerBase
     private readonly ILogger<ServerController> _logger;
     private readonly IConfiguration _config;
     private AIProvider _aIProvider;
+    private readonly SearchdomainManager _searchdomainManager;
 
-    public ServerController(ILogger<ServerController> logger, IConfiguration config, AIProvider aIProvider)
+    public ServerController(ILogger<ServerController> logger, IConfiguration config, AIProvider aIProvider, SearchdomainManager searchdomainManager)
     {
         _logger = logger;
         _config = config;
         _aIProvider = aIProvider;
+        _searchdomainManager = searchdomainManager;
     }
 
     /// <summary>
@@ -41,4 +45,51 @@ public class ServerController : ControllerBase
             return new ServerGetModelsResult() { Success = false, Message = ex.Message};
         }
     }
+
+    /// <summary>
+    /// Gets the total memory size of the embedding cache
+    /// </summary>
+    [HttpGet("EmbeddingCache/Size")]
+    public ActionResult<ServerGetEmbeddingCacheSizeResult> GetEmbeddingCacheSize()
+    {
+        long size = 0;
+        long elementCount = 0;
+        long embeddingsCount = 0;
+        LRUCache<string, Dictionary<string, float[]>> embeddingCache = _searchdomainManager.embeddingCache;
+        var cacheListField = embeddingCache.GetType()
+            .GetField("_cacheList", BindingFlags.Instance | BindingFlags.NonPublic) ?? throw new InvalidOperationException("_cacheList field not found"); // TODO Remove this unsafe reflection atrocity
+        LinkedList<string> cacheListOriginal = (LinkedList<string>)cacheListField.GetValue(embeddingCache)!;
+        LinkedList<string> cacheList = new(cacheListOriginal);
+
+        foreach (string key in cacheList)
+        {
+            if (!embeddingCache.TryGet(key, out var entry))
+                continue;
+
+            // estimate size
+            size += EstimateEntrySize(key, entry);
+            elementCount++;
+            embeddingsCount += entry.Keys.Count;
+        }
+        return new ServerGetEmbeddingCacheSizeResult() { Success = true, SizeInBytes = size, MaxElementCount = _searchdomainManager.EmbeddingCacheMaxCount, ElementCount = elementCount, EmbeddingsCount = embeddingsCount};
+    }
+
+    private static long EstimateEntrySize(string key, Dictionary<string, float[]> value)
+    {
+        int stringOverhead = MemorySizes.Align(MemorySizes.ObjectHeader + sizeof(int));
+        int arrayOverhead = MemorySizes.ArrayHeader;
+        int dictionaryOverhead = MemorySizes.ObjectHeader;
+        long size = 0;
+
+        size += stringOverhead + key.Length * sizeof(char);
+        size += dictionaryOverhead;
+
+        foreach (var kv in value)
+        {
+            size += stringOverhead + kv.Key.Length * sizeof(char);
+            size += arrayOverhead + kv.Value.Length * sizeof(float);
+        }
+
+        return size;
+    }
 }

src/Server/Program.cs

@@ -53,8 +53,8 @@ builder.Services.AddSingleton<SearchdomainHelper>();
 builder.Services.AddSingleton<SearchdomainManager>();
 builder.Services.AddSingleton<AIProvider>();
 builder.Services.AddHealthChecks()
-    .AddCheck<DatabaseHealthCheck>("DatabaseHealthCheck")
+    .AddCheck<DatabaseHealthCheck>("DatabaseHealthCheck", tags: ["Database"])
-    .AddCheck<AIProviderHealthCheck>("AIProviderHealthCheck");
+    .AddCheck<AIProviderHealthCheck>("AIProviderHealthCheck", tags: ["AIProvider"]);
 
 builder.Services.AddElmah<XmlFileErrorLog>(Options =>
 {
@@ -109,6 +109,15 @@ app.Use(async (context, next) =>
 app.UseElmah();
 
 app.MapHealthChecks("/healthz");
+app.MapHealthChecks("/healthz/Database", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions
+{
+    Predicate = c => c.Name.Contains("Database")
+});
+
+app.MapHealthChecks("/healthz/AIProvider", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions
+{
+    Predicate = c => c.Name.Contains("AIProvider")
+});
 
 bool IsDevelopment = app.Environment.IsDevelopment();
 bool useSwagger = app.Configuration.GetValue<bool>("UseSwagger");

src/Server/Views/Shared/_Layout.cshtml

@@ -8,6 +8,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>@ViewData["Title"] - embeddingsearch</title>
     <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.1/font/bootstrap-icons.css">
     <link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
     <script>
         window.appTranslations = {
@@ -29,16 +30,19 @@
                         @if (User.Identity?.IsAuthenticated == true)
                         {
                             <li class="nav-item">
-                                <a class="nav-link text-dark" asp-area="" asp-controller="Home" asp-action="Index">Home</a>
+                                <a class="nav-link text-dark" asp-area="" asp-controller="Home" asp-action="Index">@T["Home"]</a>
                             </li>
                             <li class="nav-item">
-                                <a class="nav-link text-dark" asp-area="" asp-controller="Account" asp-action="Logout">Logout</a>
+                                <a class="nav-link text-dark" asp-area="" asp-controller="Home" asp-action="Searchdomains">@T["Searchdomains"]</a>
+                            </li>
+                            <li class="nav-item">
+                                <a class="nav-link text-dark" asp-area="" asp-controller="Account" asp-action="Logout">@T["Logout"]</a>
                             </li>
                         }
                         else
                         {
                             <li class="nav-item">
-                                <a class="nav-link text-dark" asp-area="" asp-controller="Account" asp-action="Login">Login</a>
+                                <a class="nav-link text-dark" asp-area="" asp-controller="Account" asp-action="Login">@T["Login"]</a>
                             </li>
                         }
                     </ul>

src/Shared/Models/SearchdomainModels.cs

@@ -101,7 +101,7 @@ public struct SearchdomainSettings(bool cacheReconciliation = false)
     public bool CacheReconciliation { get; set; } = cacheReconciliation;
 }
 
-internal static class MemorySizes
+public static class MemorySizes
 {
     public static readonly int PointerSize = IntPtr.Size;
     public static readonly int ObjectHeader = PointerSize * 2;

src/Shared/Models/ServerModels.cs

@@ -7,3 +7,15 @@ public class ServerGetModelsResult : SuccesMessageBaseModel
     [JsonPropertyName("Models")]
     public string[]? Models { get; set; }    
 }
+
+public class ServerGetEmbeddingCacheSizeResult : SuccesMessageBaseModel
+{
+    [JsonPropertyName("SizeInBytes")]
+    public required long? SizeInBytes { get; set; }
+    [JsonPropertyName("MaxElementCount")]
+    public required long? MaxElementCount { get; set; }
+    [JsonPropertyName("ElementCount")]
+    public required long? ElementCount { get; set; }
+    [JsonPropertyName("EmbeddingsCount")]
+    public required long? EmbeddingsCount { get; set; }    
+}