fix(server): fixes models not being automatically loaded

src/Ngino.Server/KeepaliveCoordinator.cs

@@ -0,0 +1,111 @@
+using System.Text.RegularExpressions;
+
+namespace Ngino.Server;
+
+internal static class KeepaliveCoordinator
+{
+    public static IReadOnlyList<KeepaliveAction> PlanActions(
+        IEnumerable<GroupClientInfo> members,
+        IEnumerable<KeepaliveCandidate> candidates)
+    {
+        var actions = new List<KeepaliveAction>();
+        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        var candidateList = candidates.ToList();
+
+        foreach (var member in members)
+        {
+            if (string.IsNullOrWhiteSpace(member.Model))
+            {
+                continue;
+            }
+
+            var policy = member.KeepalivePolicy ?? GroupClientKeepalivePolicy.Default;
+            var targetCount = Math.Max(0, policy.InstancesToKeepAlive);
+            var matching = candidateList
+                .Where(candidate => MatchesMember(candidate.ClientId, member))
+                .ToList();
+
+            if (matching.Count == 0)
+            {
+                continue;
+            }
+
+            var active = matching.Where(candidate => candidate.HasActiveModel).ToList();
+            var activeCount = active.Count;
+
+            if (activeCount < targetCount)
+            {
+                var toLoad = matching
+                    .Where(candidate => !candidate.HasActiveModel && candidate.HasListedModel)
+                    .OrderBy(candidate => candidate.ClientId, StringComparer.OrdinalIgnoreCase)
+                    .Take(targetCount - activeCount);
+
+                foreach (var candidate in toLoad)
+                {
+                    var key = $"{candidate.ClientId}:{member.Model}";
+                    if (seen.Add(key))
+                    {
+                        actions.Add(new KeepaliveAction(candidate.ClientId, "load", member.Model));
+                    }
+                }
+            }
+            else if (activeCount > targetCount)
+            {
+                var toUnload = active
+                    .OrderByDescending(candidate => candidate.ClientId, StringComparer.OrdinalIgnoreCase)
+                    .Skip(targetCount)
+                    .Take(activeCount - targetCount);
+
+                foreach (var candidate in toUnload)
+                {
+                    var key = $"{candidate.ClientId}:{member.Model}";
+                    if (seen.Add(key))
+                    {
+                        actions.Add(new KeepaliveAction(candidate.ClientId, "unload", member.Model));
+                    }
+                }
+            }
+        }
+
+        return actions;
+    }
+
+    private static bool MatchesMember(string clientId, GroupClientInfo member)
+    {
+        if (string.IsNullOrWhiteSpace(member.ClientId)
+            && string.IsNullOrWhiteSpace(member.ClientPattern))
+        {
+            return true;
+        }
+
+        if (!string.IsNullOrWhiteSpace(member.ClientId)
+            && string.Equals(clientId, member.ClientId, StringComparison.OrdinalIgnoreCase))
+        {
+            return true;
+        }
+
+        if (string.IsNullOrWhiteSpace(member.ClientPattern))
+        {
+            return false;
+        }
+
+        try
+        {
+            return Regex.IsMatch(clientId, member.ClientPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled);
+        }
+        catch (RegexParseException)
+        {
+            return false;
+        }
+    }
+}
+
+internal sealed record KeepaliveCandidate(
+    string ClientId,
+    bool HasListedModel,
+    bool HasActiveModel);
+
+internal sealed record KeepaliveAction(
+    string ClientId,
+    string Command,
+    string Model);

src/Ngino.Server/KeepaliveService.cs

@@ -0,0 +1,125 @@
+namespace Ngino.Server;
+
+internal sealed class KeepaliveService : BackgroundService
+{
+    private static readonly TimeSpan CheckInterval = TimeSpan.FromSeconds(10);
+    private static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(30);
+
+    private readonly TunnelHub _hub;
+    private readonly ManagementStore _managementStore;
+    private readonly ILogger<KeepaliveService> _logger;
+
+    public KeepaliveService(TunnelHub hub, ManagementStore managementStore, ILogger<KeepaliveService> logger)
+    {
+        _hub = hub;
+        _managementStore = managementStore;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                await ApplyKeepaliveAsync(stoppingToken);
+            }
+            catch (Exception exception)
+            {
+                _logger.LogWarning(exception, "Keepalive cycle failed.");
+            }
+
+            await Task.Delay(CheckInterval, stoppingToken);
+        }
+    }
+
+    private async Task ApplyKeepaliveAsync(CancellationToken cancellationToken)
+    {
+        var members = _managementStore.ListAllGroupClients();
+        if (members.Count == 0)
+        {
+            return;
+        }
+
+        var snapshots = _hub.ClientSnapshots;
+        if (snapshots.Count == 0)
+        {
+            return;
+        }
+
+        foreach (var member in members)
+        {
+            if (string.IsNullOrWhiteSpace(member.Model))
+            {
+                continue;
+            }
+
+            var matchingCandidates = snapshots
+                .Select(snapshot => new KeepaliveCandidate(
+                    snapshot.Id,
+                    HasModel(snapshot.Models, member.Model),
+                    HasModel(snapshot.ActiveModels, member.Model)))
+                .ToList();
+
+            var actions = KeepaliveCoordinator.PlanActions([member], matchingCandidates);
+            foreach (var action in actions)
+            {
+                try
+                {
+                    var connection = _hub.Get(action.ClientId);
+                    if (connection is null)
+                    {
+                        continue;
+                    }
+
+                    var response = await connection.SendModelCommandAsync(
+                        action.Command,
+                        action.Model,
+                        payloadJson: null,
+                        CommandTimeout,
+                        cancellationToken);
+
+                    if (response.StatusCode is < 200 or >= 300)
+                    {
+                        _logger.LogWarning(
+                            "Keepalive {Command} for model {Model} on client {ClientId} returned HTTP {StatusCode}.",
+                            action.Command,
+                            action.Model,
+                            action.ClientId,
+                            response.StatusCode);
+                    }
+                }
+                catch (Exception exception)
+                {
+                    _logger.LogWarning(exception,
+                        "Keepalive {Command} for model {Model} on client {ClientId} failed.",
+                        action.Command,
+                        action.Model,
+                        action.ClientId);
+                }
+            }
+        }
+    }
+
+    private static bool HasModel(IEnumerable<string> models, string requestedModel)
+    {
+        if (string.IsNullOrWhiteSpace(requestedModel))
+        {
+            return false;
+        }
+
+        var requested = requestedModel.Trim();
+        return models.Any(model => ModelNamesMatch(requested, model));
+    }
+
+    private static bool ModelNamesMatch(string requested, string available)
+    {
+        return string.Equals(requested, available, StringComparison.OrdinalIgnoreCase)
+            || string.Equals(StripLatestTag(requested), StripLatestTag(available), StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static string StripLatestTag(string model) =>
+        model.EndsWith(":latest", StringComparison.OrdinalIgnoreCase)
+            ? model[..^":latest".Length]
+            : model;
+}

src/Ngino.Server/ManagementStore.cs

@@ -834,6 +834,36 @@ internal sealed class ManagementStore
                 """;
             command.Parameters.AddWithValue("$group_id", groupId);
 
+            return ReadGroupClientInfos(command);
+        }
+    }
+
+    public IReadOnlyList<GroupClientInfo> ListAllGroupClients()
+    {
+        if (!_isAvailable)
+        {
+            return [];
+        }
+
+        lock (_lock)
+        {
+            using var connection = OpenConnection();
+            using var command = connection.CreateCommand();
+            command.CommandText = """
+                SELECT id, group_id, client_id, model, client_pattern,
+                       keepalive_instances_to_keep_alive,
+                       keepalive_max_parallelism_per_client,
+                       keepalive_parallelism_headroom
+                FROM group_members
+                ORDER BY client_id, model, client_pattern
+                """;
+
+            return ReadGroupClientInfos(command);
+        }
+    }
+
+    private List<GroupClientInfo> ReadGroupClientInfos(SqliteCommand command)
+    {
         var result = new List<GroupClientInfo>();
         using var reader = command.ExecuteReader();
         while (reader.Read())
@@ -849,7 +879,6 @@ internal sealed class ManagementStore
 
         return result;
     }
-    }
 
     public GroupClientInfo AddGroupClient(
         string groupId,

src/Ngino.Server/Program.cs

@@ -20,6 +20,7 @@ builder.Services.AddSingleton<TunnelHub>();
 builder.Services.AddSingleton<EmbeddingCache>();
 builder.Services.AddSingleton<ManagementStore>();
 builder.Services.AddSingleton<AuthRateLimiter>();
+builder.Services.AddHostedService<KeepaliveService>();
 builder.Services.AddElmah<ElmahCore.MySql.MySqlErrorLog>().Configure<ElmahOptions>(
     options => options.ConnectionString = builder.Configuration.GetConnectionString("ElmahConnection"));
 

src/Ngino.Server/wwwroot/admin/app.css

@@ -167,11 +167,48 @@ textarea {
 }
 
 .field label {
+  display: flex;
+  align-items: center;
+  gap: 6px;
   color: var(--muted);
   font-size: 12px;
   font-weight: 600;
 }
 
+.help-button {
+  display: inline-grid;
+  place-items: center;
+  width: 18px;
+  height: 18px;
+  border: 0;
+  border-radius: 999px;
+  background: var(--panel-alt);
+  color: var(--blue);
+  font-size: 12px;
+  font-weight: 800;
+  cursor: pointer;
+  padding: 0;
+}
+
+.help-button.active {
+  background: var(--blue);
+  color: #ffffff;
+}
+
+.help-popover {
+  position: relative;
+  z-index: 1;
+  margin-top: 4px;
+  padding: 8px 10px;
+  border: 1px solid var(--line);
+  border-radius: 6px;
+  background: #ffffff;
+  color: var(--text);
+  box-shadow: var(--shadow);
+  font-size: 12px;
+  line-height: 1.45;
+}
+
 .input,
 .select,
 .textarea {

src/Ngino.Server/wwwroot/admin/app.js

@@ -16,6 +16,14 @@ const sidebarMeta = document.getElementById("sidebarMeta");
 
 document.getElementById("refreshButton").addEventListener("click", () => refresh(true));
 window.addEventListener("hashchange", () => renderRoute());
+document.addEventListener("click", (event) => {
+  if (event.target.closest(".help-button") || event.target.closest(".help-popover")) {
+    return;
+  }
+
+  document.querySelectorAll(".help-popover").forEach((popover) => popover.remove());
+  document.querySelectorAll(".help-button.active").forEach((button) => button.classList.remove("active"));
+});
 
 const morphdomOptions = {
   childrenOnly: true,
@@ -37,6 +45,38 @@ function patchSidebar(html) {
   morphdom(sidebarMeta, temp, morphdomOptions);
 }
 
+function renderFieldLabel(forId, label, helpText) {
+  return `
+    <label for="${escapeAttr(forId)}">
+      <span>${escapeHtml(label)}</span>
+      <button class="help-button" type="button" data-action="toggle-help" data-help="${escapeAttr(helpText)}" aria-label="Explain ${escapeAttr(label)}">?</button>
+    </label>
+  `;
+}
+
+function toggleHelpPopover(button) {
+  const field = button.closest(".field");
+  if (!field) {
+    return;
+  }
+
+  const existingPopover = field.querySelector(".help-popover");
+  if (existingPopover) {
+    existingPopover.remove();
+    button.classList.remove("active");
+    return;
+  }
+
+  document.querySelectorAll(".help-popover").forEach((popover) => popover.remove());
+  document.querySelectorAll(".help-button.active").forEach((activeButton) => activeButton.classList.remove("active"));
+
+  const popover = document.createElement("div");
+  popover.className = "help-popover";
+  popover.textContent = button.dataset.help || "";
+  field.appendChild(popover);
+  button.classList.add("active");
+}
+
 content.addEventListener("click", async (event) => {
   const button = event.target.closest("button[data-action]");
   if (!button) {
@@ -45,6 +85,12 @@ content.addEventListener("click", async (event) => {
 
   const { action, clientId, model, keyId, groupId, memberId } = button.dataset;
 
+  if (action === "toggle-help") {
+    event.stopPropagation();
+    toggleHelpPopover(button);
+    return;
+  }
+
   try {
     setBusy(button, true);
 
@@ -985,15 +1031,15 @@ function renderGroupDetail() {
             <input class="input" id="addClientPattern" name="clientPattern" placeholder="GPU_[0-9]*">
           </div>
           <div class="field">
-            <label for="addClientKeepaliveInstances">Keepalive instances</label>
+            ${renderFieldLabel("addClientKeepaliveInstances", "Min always loaded instances", "The minimum number of warm model instances that should stay loaded and ready so requests do not wait for a cold start.")}
             <input class="input" id="addClientKeepaliveInstances" name="keepaliveInstancesToKeepAlive" type="number" min="1" step="1" value="1">
           </div>
           <div class="field">
-            <label for="addClientKeepaliveMaxParallelism">Max parallelism per client</label>
+            ${renderFieldLabel("addClientKeepaliveMaxParallelism", "Max parallelism per client", "The maximum number of concurrent requests this client can handle at once. Higher values let one client absorb more traffic, but too many can overload the GPU.")}
             <input class="input" id="addClientKeepaliveMaxParallelism" name="keepaliveMaxParallelismPerClient" type="number" min="1" step="1" value="1">
           </div>
           <div class="field">
-            <label for="addClientKeepaliveHeadroom">Parallelism headroom</label>
+            ${renderFieldLabel("addClientKeepaliveHeadroom", "Parallelism headroom", "How much spare parallelism to leave unused so traffic spikes can be absorbed without saturating the GPU. A larger headroom makes routing more conservative.")}
             <input class="input" id="addClientKeepaliveHeadroom" name="keepaliveParallelismHeadroom" type="number" min="1" step="1" value="1">
           </div>
           <button class="button" type="submit">Add</button>

tests/Ngino.Client.Tests/KeepaliveCoordinatorTests.cs

@@ -0,0 +1,45 @@
+using Ngino.Server;
+using Xunit;
+
+namespace ReverseLlama.Client.Tests;
+
+public class KeepaliveCoordinatorTests
+{
+    [Fact]
+    public void PlanActions_LoadsMissingKeepaliveInstances()
+    {
+        var policy = new GroupClientKeepalivePolicy(2, 1, 1);
+        var member = new GroupClientInfo(1, "group-1", "client-1", "bge-m3:latest", null, policy);
+        var candidates = new[]
+        {
+            new KeepaliveCandidate("client-1", true, false),
+            new KeepaliveCandidate("client-2", true, false),
+            new KeepaliveCandidate("client-3", true, true)
+        };
+
+        var actions = KeepaliveCoordinator.PlanActions([member], candidates);
+
+        Assert.Single(actions);
+        Assert.Equal("client-1", actions[0].ClientId);
+        Assert.Equal("load", actions[0].Command);
+        Assert.Equal("bge-m3:latest", actions[0].Model);
+    }
+
+    [Fact]
+    public void PlanActions_UnloadsWhenTooManyInstancesAreActive()
+    {
+        var policy = new GroupClientKeepalivePolicy(1, 1, 1);
+        var member = new GroupClientInfo(2, "group-1", null, "bge-m3:latest", null, policy);
+        var candidates = new[]
+        {
+            new KeepaliveCandidate("client-1", true, true),
+            new KeepaliveCandidate("client-2", true, true)
+        };
+
+        var actions = KeepaliveCoordinator.PlanActions([member], candidates);
+
+        Assert.Single(actions);
+        Assert.Equal("client-1", actions[0].ClientId);
+        Assert.Equal("unload", actions[0].Command);
+    }
+}