diff --git a/deploy/install-client.ps1 b/deploy/install-client.ps1 index 92082af..1d1fc7c 100644 --- a/deploy/install-client.ps1 +++ b/deploy/install-client.ps1 @@ -14,7 +14,11 @@ param( [string]$InstallDir = "$env:ProgramFiles\Ngino Client", [string]$ServiceName = "NginoClient", [switch]$InsecureSkipTlsVerify, - [switch]$NoOllama + [switch]$NoOllama, + [switch]$UseLlamaCppViaDocker, + [string]$UseOllamaModelsPath = "", + [string]$LlamaCppDockerImage = "", + [int]$LlamaCppBasePort = 0 ) $ErrorActionPreference = "Stop" @@ -54,6 +58,14 @@ if ([string]::IsNullOrWhiteSpace($Token)) { finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($tokenPointer) } } if ([string]::IsNullOrWhiteSpace($Token)) { throw "Token is required." } + +if ($UseLlamaCppViaDocker -and [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) { + $UseOllamaModelsPath = Read-Host "Ollama models path (e.g. C:\Users\user\.ollama\models)" +} +if ($UseLlamaCppViaDocker -and [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) { + throw "Ollama models path is required with -UseLlamaCppViaDocker." +} + if ($ServiceName -notmatch '^[A-Za-z0-9_.-]+$') { throw "ServiceName contains unsupported characters." } $scriptDir = $PSScriptRoot @@ -149,6 +161,18 @@ $serviceEnvironment = @( "DOTNET_CLI_TELEMETRY_OPTOUT=1", "DOTNET_NOLOGO=1" ) +if ($UseLlamaCppViaDocker) { + $serviceEnvironment += "NGINO_USE_LLAMA_CPP_VIA_DOCKER=true" + if (-not [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) { + $serviceEnvironment += "NGINO_USE_OLLAMA_MODELS_PATH=$UseOllamaModelsPath" + } + if (-not [string]::IsNullOrWhiteSpace($LlamaCppDockerImage)) { + $serviceEnvironment += "NGINO_LLAMA_CPP_DOCKER_IMAGE=$LlamaCppDockerImage" + } + if ($LlamaCppBasePort -gt 0) { + $serviceEnvironment += "NGINO_LLAMA_CPP_BASE_PORT=$LlamaCppBasePort" + } +} if ($InsecureSkipTlsVerify) { Write-Warn "Server TLS certificate validation is disabled for $ServiceName." } @@ -169,6 +193,9 @@ Write-Host " Client ID: $ClientId" Write-Host " Upstream: $Upstream" Write-Host " Service: $ServiceName" Write-Host " Install dir: $InstallDir" +if ($UseLlamaCppViaDocker) { + Write-Host " llama.cpp: enabled (models: $UseOllamaModelsPath)" +} Write-Host "" Write-Host " Manage: Get-Service $ServiceName | Start-Service/Stop-Service/Restart-Service" Write-Host " Logs: Get-WinEvent -LogName Application | Where-Object ProviderName -eq NginoClient" diff --git a/deploy/install-client.sh b/deploy/install-client.sh index 95174d8..23f757b 100755 --- a/deploy/install-client.sh +++ b/deploy/install-client.sh @@ -16,6 +16,10 @@ TOKEN="" CLIENT_ID="$(hostname -s 2>/dev/null || echo "linux-client")" UPSTREAM="$DEFAULT_UPSTREAM" SKIP_OLLAMA=false +USE_LLAMA_CPP_VIA_DOCKER=false +USE_OLLAMA_MODELS_PATH="" +LLAMA_CPP_DOCKER_IMAGE="" +LLAMA_CPP_BASE_PORT="" # ── Colors ──────────────────────────────────────────────────────────────────── RED='\033[0;31m' @@ -45,11 +49,21 @@ Optional: --install-dir Install directory; defaults to $DEFAULT_INSTALL_DIR --service-name systemd service name; defaults to $DEFAULT_SERVICE_NAME --no-ollama Skip Ollama installation and status check + --use-llama-cpp-via-docker + Use llama.cpp via Docker for inference instead of Ollama + --use-ollama-models-path + Path to Ollama models directory (manifests/blobs); required with --use-llama-cpp-via-docker + --llama-cpp-docker-image + llama.cpp Docker image; defaults to auto-detected (rocm/cuda/cpu) + --llama-cpp-base-port + Base port for llama.cpp containers; defaults to 8081 -h, --help Show this help message Examples: $0 --server http://gpu-server:5050 --token "my-secret" $0 --server http://gpu-server:5050 --token "my-secret" --no-ollama + $0 --server http://gpu-server:5050 --token "my-secret" \\ + --use-llama-cpp-via-docker --use-ollama-models-path /usr/share/ollama/.ollama/models EOF exit 0 } @@ -57,15 +71,19 @@ EOF # ── Argument parsing ────────────────────────────────────────────────────────── while [[ $# -gt 0 ]]; do case "$1" in - --server) SERVER_URL="$2"; shift 2 ;; - --token) TOKEN="$2"; shift 2 ;; - --client-id) CLIENT_ID="$2"; shift 2 ;; - --upstream) UPSTREAM="$2"; shift 2 ;; - --install-dir) INSTALL_DIR="$2"; shift 2 ;; - --service-name) SERVICE_NAME="$2"; shift 2 ;; - --no-ollama) SKIP_OLLAMA=true; shift ;; - -h|--help) usage ;; - *) die "Unknown option: $1" ;; + --server) SERVER_URL="$2"; shift 2 ;; + --token) TOKEN="$2"; shift 2 ;; + --client-id) CLIENT_ID="$2"; shift 2 ;; + --upstream) UPSTREAM="$2"; shift 2 ;; + --install-dir) INSTALL_DIR="$2"; shift 2 ;; + --service-name) SERVICE_NAME="$2"; shift 2 ;; + --no-ollama) SKIP_OLLAMA=true; shift ;; + --use-llama-cpp-via-docker) USE_LLAMA_CPP_VIA_DOCKER=true; shift ;; + --use-ollama-models-path) USE_OLLAMA_MODELS_PATH="$2"; shift 2 ;; + --llama-cpp-docker-image) LLAMA_CPP_DOCKER_IMAGE="$2"; shift 2 ;; + --llama-cpp-base-port) LLAMA_CPP_BASE_PORT="$2"; shift 2 ;; + -h|--help) usage ;; + *) die "Unknown option: $1" ;; esac done @@ -85,6 +103,13 @@ if [[ -z "$TOKEN" ]]; then die "Token is required." fi +if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" && -z "$USE_OLLAMA_MODELS_PATH" ]]; then + read -rp "Ollama models path (e.g. /usr/share/ollama/.ollama/models): " USE_OLLAMA_MODELS_PATH +fi +if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" && -z "$USE_OLLAMA_MODELS_PATH" ]]; then + die "Ollama models path is required with --use-llama-cpp-via-docker." +fi + # ── Root check ──────────────────────────────────────────────────────────────── if [[ $EUID -ne 0 ]]; then die "This script must be run as root (or with sudo)." @@ -265,7 +290,21 @@ info "Client installed to $INSTALL_DIR." # ── Write environment file (avoids shell injection in unit file) ───────────── ENV_DIR="/etc/ngino-client" mkdir -p "$ENV_DIR" -printf 'NGINO_TOKEN=%s\n' "$TOKEN" > "$ENV_DIR/env" +{ + printf 'NGINO_TOKEN=%s\n' "$TOKEN" + if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" ]]; then + printf 'NGINO_USE_LLAMA_CPP_VIA_DOCKER=true\n' + if [[ -n "$USE_OLLAMA_MODELS_PATH" ]]; then + printf 'NGINO_USE_OLLAMA_MODELS_PATH=%s\n' "$USE_OLLAMA_MODELS_PATH" + fi + if [[ -n "$LLAMA_CPP_DOCKER_IMAGE" ]]; then + printf 'NGINO_LLAMA_CPP_DOCKER_IMAGE=%s\n' "$LLAMA_CPP_DOCKER_IMAGE" + fi + if [[ -n "$LLAMA_CPP_BASE_PORT" ]]; then + printf 'NGINO_LLAMA_CPP_BASE_PORT=%s\n' "$LLAMA_CPP_BASE_PORT" + fi + fi +} > "$ENV_DIR/env" chmod 600 "$ENV_DIR/env" info "Environment file written to $ENV_DIR/env (mode 0600)." @@ -280,8 +319,8 @@ fi cat > "$SERVICE_FILE" < """ Ngino.Client options: - --server Server base URL, e.g. http://my-server:5050 - --upstream Local upstream URL, e.g. http://localhost:11434 - --token Optional token matching the server - --client-id Identifies this machine on the server; defaults to the machine name - --tunnel-path Defaults to /_ngino/tunnel - --reconnect-delay Defaults to 5 - --chunk-size Defaults to 65536 - --insecure-skip-tls-verify Disable server TLS certificate validation (unsafe) + --server Server base URL, e.g. http://my-server:5050 + --upstream Local upstream URL, e.g. http://localhost:11434 + --token Optional token matching the server + --client-id Identifies this machine on the server; defaults to the machine name + --tunnel-path Defaults to /_ngino/tunnel + --reconnect-delay Defaults to 5 + --chunk-size Defaults to 65536 + --insecure-skip-tls-verify Disable server TLS certificate validation (unsafe) + --use-llama-cpp-via-docker Use llama.cpp via Docker for inference instead of Ollama + --use-ollama-models-path Path to Ollama models directory (manifests/blobs), required with --use-llama-cpp-via-docker + --llama-cpp-docker-image llama.cpp Docker image; defaults to auto-detected (rocm/cuda/cpu) + --llama-cpp-base-port Base port for llama.cpp containers; defaults to 8081 """; private static Dictionary ParseArgs(string[] args) @@ -92,7 +108,7 @@ internal sealed class ClientOptions continue; } - if (keyValue[0].Equals("insecure-skip-tls-verify", StringComparison.OrdinalIgnoreCase)) + if (IsBoolFlag(keyValue[0])) { values[keyValue[0]] = "true"; continue; @@ -109,6 +125,16 @@ internal sealed class ClientOptions return values; } + private static bool IsBoolFlag(string key) + { + return key switch + { + "insecure-skip-tls-verify" => true, + "use-llama-cpp-via-docker" => true, + _ => false + }; + } + private static string? Read(Dictionary values, params string[] keys) { foreach (var key in keys) @@ -154,4 +180,14 @@ internal sealed class ClientOptions private static string NormalizePath(string path) => path.StartsWith('/') ? path : $"/{path}"; + + private static string? NormalizeDirectoryPath(string? path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return null; + } + + return Path.GetFullPath(path); + } } diff --git a/src/Ngino.Client/LlamaCppManager.cs b/src/Ngino.Client/LlamaCppManager.cs new file mode 100644 index 0000000..2770895 --- /dev/null +++ b/src/Ngino.Client/LlamaCppManager.cs @@ -0,0 +1,471 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Text.Json; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Ngino.Client; + +internal sealed partial class LlamaCppManager : IAsyncDisposable +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + private const int DefaultBasePort = 8081; + private const string NginoContainerLabel = "ngino-llamacpp"; + private static readonly TimeSpan DockerTimeout = TimeSpan.FromSeconds(60); + + private readonly string _blobsPath; + private readonly string _manifestsPath; + private readonly string _dockerImage; + private readonly int _basePort; + private readonly ILogger _logger; + private readonly ConcurrentDictionary _modelPorts = new(StringComparer.OrdinalIgnoreCase); + + public LlamaCppManager( + string ollamaModelsPath, + string? dockerImage, + int? basePort, + ILogger? logger = null) + { + _manifestsPath = Path.Combine(ollamaModelsPath, "manifests"); + _blobsPath = Path.Combine(ollamaModelsPath, "blobs"); + _dockerImage = dockerImage ?? GetDefaultDockerImage(); + _basePort = basePort ?? DefaultBasePort; + _logger = logger ?? NullLogger.Instance; + } + + public string DockerImage => _dockerImage; + + public List DiscoverModels() + { + var models = new List(); + + if (!Directory.Exists(_manifestsPath)) + { + _logger.LogWarning("Ollama manifests path not found: {ManifestsPath}", _manifestsPath); + return models; + } + + foreach (var manifestPath in Directory.EnumerateFiles(_manifestsPath, "*", SearchOption.AllDirectories)) + { + try + { + var model = ParseManifest(manifestPath); + if (model is not null) + { + models.Add(model); + } + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to parse manifest: {ManifestPath}", manifestPath); + } + } + + return models; + } + + public List DiscoverModelsWithBlob() + { + return DiscoverModels().Where(m => File.Exists(m.BlobPath)).ToList(); + } + + public bool IsModelActive(string ollamaModelName) + { + return _modelPorts.ContainsKey(ollamaModelName); + } + + public Uri? GetUpstream(string ollamaModelName) + { + if (_modelPorts.TryGetValue(ollamaModelName, out var port)) + { + return new Uri($"http://localhost:{port}"); + } + + return null; + } + + public async Task StartModelContainerAsync(LlamaCppModel model, CancellationToken cancellationToken) + { + var ollamaName = model.OllamaName; + if (string.IsNullOrWhiteSpace(ollamaName)) + { + _logger.LogWarning("Cannot start container: model has no Ollama name"); + return false; + } + + if (_modelPorts.ContainsKey(ollamaName)) + { + _logger.LogInformation("Model {Model} already has a running container", ollamaName); + return true; + } + + if (!File.Exists(model.BlobPath)) + { + _logger.LogError("Model blob not found: {BlobPath}", model.BlobPath); + return false; + } + + var port = FindAvailablePort(); + var containerName = SanitizeContainerName($"ngino-llamacpp-{ollamaName}"); + + var args = BuildDockerRunArgs(containerName, model, port); + _logger.LogInformation( + "Starting llama.cpp container for {Model} on port {Port}: docker {Args}", + ollamaName, port, string.Join(" ", args)); + + try + { + var (exitCode, output) = await RunDockerWithOutputAsync(args, cancellationToken); + if (exitCode != 0) + { + _logger.LogError( + "Failed to start llama.cpp container for {Model}, exit code: {ExitCode}, output: {Output}", + ollamaName, exitCode, output); + return false; + } + + _modelPorts[ollamaName] = port; + _logger.LogInformation("llama.cpp container for {Model} started on port {Port}", ollamaName, port); + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to start llama.cpp container for {Model}", ollamaName); + return false; + } + } + + public async Task StopModelContainerAsync(string ollamaModelName, CancellationToken cancellationToken) + { + if (!_modelPorts.TryRemove(ollamaModelName, out _)) + { + _logger.LogWarning("No running container found for model {Model}", ollamaModelName); + return false; + } + + var containerName = SanitizeContainerName($"ngino-llamacpp-{ollamaModelName}"); + + _logger.LogInformation("Stopping llama.cpp container {ContainerName}", containerName); + + try + { + await RunDockerAsync(["stop", "--time", "10", containerName], cancellationToken); + await RunDockerAsync(["rm", "-f", containerName], cancellationToken); + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to stop container {ContainerName}", containerName); + return false; + } + } + + public async Task StopAllContainersAsync() + { + _logger.LogInformation("Stopping all llama.cpp containers..."); + + try + { + var (exitCode, output) = await RunDockerWithOutputAsync( + ["ps", "-q", "--filter", $"label={NginoContainerLabel}"], + CancellationToken.None); + + if (exitCode == 0 && !string.IsNullOrWhiteSpace(output)) + { + var containerIds = output.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + foreach (var id in containerIds) + { + await RunDockerAsync(["stop", "--time", "10", id], CancellationToken.None); + await RunDockerAsync(["rm", "-f", id], CancellationToken.None); + } + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to stop all llama.cpp containers"); + } + + _modelPorts.Clear(); + } + + public async Task TestDockerAsync() + { + try + { + var (exitCode, _) = await RunDockerWithOutputAsync(["info", "--format", "{{.ServerVersion}}"], CancellationToken.None); + return exitCode == 0; + } + catch + { + return false; + } + } + + public async ValueTask DisposeAsync() + { + await StopAllContainersAsync(); + } + + private LlamaCppModel? ParseManifest(string manifestPath) + { + var json = File.ReadAllText(manifestPath); + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + + if (!root.TryGetProperty("layers", out var layers) || layers.ValueKind != JsonValueKind.Array) + { + return null; + } + + string? modelDigest = null; + foreach (var layer in layers.EnumerateArray()) + { + if (layer.TryGetProperty("mediaType", out var mediaType) + && mediaType.GetString() == "application/vnd.ollama.image.model" + && layer.TryGetProperty("digest", out var digest)) + { + modelDigest = digest.GetString(); + break; + } + } + + if (string.IsNullOrWhiteSpace(modelDigest)) + { + return null; + } + + var modelName = ResolveModelName(manifestPath); + if (modelName is null) + { + return null; + } + + var blobName = modelDigest.Replace(":", "-", StringComparison.Ordinal); + var blobPath = Path.GetFullPath(Path.Combine(_blobsPath, blobName)); + + return new LlamaCppModel + { + OllamaName = modelName, + BlobDigest = blobName, + BlobPath = blobPath, + ManifestPath = manifestPath + }; + } + + private static string? ResolveModelName(string manifestPath) + { + var normalizedPath = manifestPath.Replace('\\', '/'); + var parts = normalizedPath.Split('/', StringSplitOptions.RemoveEmptyEntries); + + var manifestIndex = Array.FindLastIndex(parts, p => + p.Equals("manifests", StringComparison.OrdinalIgnoreCase)); + + if (manifestIndex < 0 || manifestIndex >= parts.Length - 1) + { + return null; + } + + var relativeParts = parts[(manifestIndex + 1)..]; + + if (relativeParts.Length < 2) + { + return null; + } + + var registry = relativeParts[0]; + + if (registry.Equals("registry.ollama.ai", StringComparison.OrdinalIgnoreCase)) + { + if (relativeParts.Length < 3) + { + return null; + } + + if (relativeParts[1].Equals("library", StringComparison.OrdinalIgnoreCase) && relativeParts.Length >= 4) + { + return $"{relativeParts[2]}:{relativeParts[3]}"; + } + + if (relativeParts.Length == 3) + { + return $"{relativeParts[1]}:{relativeParts[2]}"; + } + + return null; + } + + if (relativeParts.Length >= 3) + { + var tag = relativeParts[^1]; + var modelPath = string.Join("/", relativeParts.Take(relativeParts.Length - 1)); + return $"{modelPath}:{tag}"; + } + + return null; + } + + private string[] BuildDockerRunArgs(string containerName, LlamaCppModel model, int port) + { + var blobsDir = Path.GetDirectoryName(Path.GetFullPath(model.BlobPath))!; + var blobFile = Path.GetFileName(model.BlobPath); + + var args = new List + { + "run", + "-d", + "--rm", + "--label", $"{NginoContainerLabel}=true", + "--name", containerName, + "-p", $"{port}:{port}", + "-v", $"{blobsDir}:/models/blobs:ro", + }; + + if (HasRocmDevices()) + { + args.Add("--device=/dev/kfd"); + args.Add("--device=/dev/dri"); + args.Add("--group-add=video"); + } + + if (HasNvidiaGpu() && !HasRocmDevices()) + { + args.Add("--gpus=all"); + } + + args.Add(_dockerImage); + args.Add("-m"); + args.Add($"/models/blobs/{blobFile}"); + args.Add("-ngl"); + args.Add("999"); + args.Add("--parallel"); + args.Add("4"); + args.Add("--host"); + args.Add("0.0.0.0"); + args.Add("--port"); + args.Add(port.ToString()); + + return [.. args]; + } + + private int FindAvailablePort() + { + var usedPorts = new HashSet(_modelPorts.Values); + var port = _basePort; + + while (usedPorts.Contains(port)) + { + port++; + } + + return port; + } + + private static string SanitizeContainerName(string name) + { + var sanitized = InvalidContainerNameChars().Replace(name, "_"); + return sanitized.Trim('_').ToLowerInvariant(); + } + + private async Task RunDockerAsync(string[] args, CancellationToken cancellationToken) + { + var (exitCode, _) = await RunDockerWithOutputAsync(args, cancellationToken); + return exitCode; + } + + private async Task<(int ExitCode, string Output)> RunDockerWithOutputAsync( + string[] args, CancellationToken cancellationToken) + { + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = "docker", + Arguments = string.Join(" ", args.Select(a => a.Contains(' ') ? $"\"{a}\"" : a)), + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + process.Start(); + + var readOutput = process.StandardOutput.ReadToEndAsync(cancellationToken); + var readError = process.StandardError.ReadToEndAsync(cancellationToken); + + var completed = await Task.WhenAny( + process.WaitForExitAsync(cancellationToken), + Task.Delay(DockerTimeout, cancellationToken)); + + string output; + string error; + + try + { + output = await readOutput; + error = await readError; + } + catch + { + output = ""; + error = "timed out"; + } + + if (completed is Task { IsCompleted: true } delayTask + && delayTask != process.WaitForExitAsync(cancellationToken)) + { + _logger.LogWarning("Docker command timed out: docker {Args}", string.Join(" ", args)); + try { process.Kill(entireProcessTree: true); } catch { } + return (-1, error); + } + + if (!string.IsNullOrWhiteSpace(error)) + { + output = $"{output}\n{error}".Trim(); + } + + return (process.ExitCode, output); + } + + private static string GetDefaultDockerImage() + { + if (HasRocmDevices()) + { + return "ghcr.io/ggml-org/llama.cpp:server-rocm"; + } + + if (HasNvidiaGpu()) + { + return "ghcr.io/ggml-org/llama.cpp:server-cuda"; + } + + return "ghcr.io/ggml-org/llama.cpp:server"; + } + + private static bool HasRocmDevices() => File.Exists("/dev/kfd") && File.Exists("/dev/dri"); + + private static bool HasNvidiaGpu() + { + try + { + return File.Exists("/proc/driver/nvidia/version") + || Directory.Exists("/proc/driver/nvidia/gpus"); + } + catch + { + return false; + } + } + + [GeneratedRegex(@"[^a-zA-Z0-9_.-]")] + private static partial Regex InvalidContainerNameChars(); +} + +internal sealed record LlamaCppModel +{ + public required string OllamaName { get; init; } + public required string BlobDigest { get; init; } + public required string BlobPath { get; init; } + public required string ManifestPath { get; init; } +} diff --git a/src/Ngino.Client/Program.cs b/src/Ngino.Client/Program.cs index 45668a4..b9617f4 100644 --- a/src/Ngino.Client/Program.cs +++ b/src/Ngino.Client/Program.cs @@ -16,6 +16,14 @@ try Console.WriteLine(" WARNING: server TLS certificate validation is disabled"); } + if (options.UseLlamaCppViaDocker) + { + Console.WriteLine($" llama.cpp via Docker: enabled"); + Console.WriteLine($" ollama models path: {options.UseOllamaModelsPath ?? "(not set)"}"); + Console.WriteLine($" llama.cpp docker image: {options.LlamaCppDockerImage ?? "(auto)"}"); + Console.WriteLine($" llama.cpp base port: {options.LlamaCppBasePort}"); + } + // Args are parsed by ClientOptions; keep them away from the host configuration. var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings { Args = [] }); builder.Services.AddSingleton(options); diff --git a/src/Ngino.Client/TunnelClient.cs b/src/Ngino.Client/TunnelClient.cs index 385ec53..f2dd5f3 100644 --- a/src/Ngino.Client/TunnelClient.cs +++ b/src/Ngino.Client/TunnelClient.cs @@ -21,6 +21,7 @@ internal sealed class TunnelClient private readonly ILogger _logger; private readonly object _modelSnapshotLock = new(); private readonly SemaphoreSlim _sendLock = new(1, 1); + private readonly LlamaCppManager? _llamaCppManager; private List _lastActiveModels = []; private List _lastModels = []; @@ -32,10 +33,39 @@ internal sealed class TunnelClient { Timeout = Timeout.InfiniteTimeSpan }; + + if (_options.UseLlamaCppViaDocker) + { + if (string.IsNullOrWhiteSpace(_options.UseOllamaModelsPath)) + { + throw new InvalidOperationException( + "--use-ollama-models-path is required when --use-llama-cpp-via-docker is set."); + } + + _llamaCppManager = new LlamaCppManager( + _options.UseOllamaModelsPath, + _options.LlamaCppDockerImage, + _options.LlamaCppBasePort, + _logger); + } } public async Task RunAsync(CancellationToken cancellationToken) { + if (_llamaCppManager is not null) + { + _logger.LogInformation("Testing Docker availability..."); + var dockerAvailable = await _llamaCppManager.TestDockerAsync(); + if (!dockerAvailable) + { + _logger.LogWarning("Docker is not available. llama.cpp via Docker will not work."); + } + else + { + _logger.LogInformation("Docker is available. Using llama.cpp image: {Image}", _llamaCppManager.DockerImage); + } + } + while (!cancellationToken.IsCancellationRequested) { using var socket = new ClientWebSocket(); @@ -154,6 +184,16 @@ internal sealed class TunnelClient private async Task> GetUpstreamModelsAsync(CancellationToken cancellationToken) { + if (_llamaCppManager is not null) + { + var models = _llamaCppManager.DiscoverModelsWithBlob(); + return models + .Select(m => m.OllamaName) + .Where(name => !string.IsNullOrWhiteSpace(name)) + .OrderBy(name => name, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_options.Upstream, "/api/tags")); using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); response.EnsureSuccessStatusCode(); @@ -166,6 +206,17 @@ internal sealed class TunnelClient private async Task> GetActiveUpstreamModelsAsync(CancellationToken cancellationToken) { + if (_llamaCppManager is not null) + { + var models = _llamaCppManager.DiscoverModelsWithBlob(); + return models + .Where(m => _llamaCppManager.IsModelActive(m.OllamaName)) + .Select(m => m.OllamaName) + .Where(name => !string.IsNullOrWhiteSpace(name)) + .OrderBy(name => name, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_options.Upstream, "/api/ps")); using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); @@ -363,6 +414,11 @@ internal sealed class TunnelClient throw new InvalidOperationException("Model command is missing a model name."); } + if (_llamaCppManager is not null) + { + return await ExecuteModelCommandWithLlamaCppAsync(message, cancellationToken); + } + using var request = BuildModelCommandRequest(message); using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); var body = await response.Content.ReadAsByteArrayAsync(cancellationToken); @@ -379,6 +435,118 @@ internal sealed class TunnelClient return BuildModelCommandResult(message.RequestId, response, body); } + private async Task ExecuteModelCommandWithLlamaCppAsync( + TunnelMessage message, CancellationToken cancellationToken) + { + var modelName = message.Model?.Trim(); + var normalizedCommand = NormalizeModelCommand(message.Command); + + switch (normalizedCommand) + { + case "load": + { + var models = _llamaCppManager!.DiscoverModelsWithBlob(); + var model = models.FirstOrDefault(m => + string.Equals(m.OllamaName, modelName, StringComparison.OrdinalIgnoreCase)); + + if (model is null) + { + return new TunnelMessage + { + Type = TunnelMessageTypes.ModelCommandResult, + RequestId = message.RequestId, + StatusCode = 404, + Error = $"Model '{modelName}' not found in Ollama models path." + }; + } + + var started = await _llamaCppManager.StartModelContainerAsync(model, cancellationToken); + if (!started) + { + return new TunnelMessage + { + Type = TunnelMessageTypes.ModelCommandResult, + RequestId = message.RequestId, + StatusCode = 500, + Error = $"Failed to start llama.cpp container for model '{modelName}'." + }; + } + + return BuildModelCommandResult(message.RequestId, 200, "OK", []); + } + + case "unload": + { + var stopped = await _llamaCppManager!.StopModelContainerAsync(modelName!, cancellationToken); + if (!stopped) + { + return new TunnelMessage + { + Type = TunnelMessageTypes.ModelCommandResult, + RequestId = message.RequestId, + StatusCode = 404, + Error = $"No running llama.cpp container for model '{modelName}'." + }; + } + + return BuildModelCommandResult(message.RequestId, 200, "OK", []); + } + + case "pull": + case "delete": + { + using var request = BuildModelCommandRequest(_options.Upstream, message.Command, message.Model); + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + var body = await response.Content.ReadAsByteArrayAsync(cancellationToken); + return BuildModelCommandResult(message.RequestId, response, body); + } + + case "show": + { + var models = _llamaCppManager!.DiscoverModelsWithBlob(); + var model = models.FirstOrDefault(m => + string.Equals(m.OllamaName, modelName, StringComparison.OrdinalIgnoreCase)); + + if (model is null) + { + return new TunnelMessage + { + Type = TunnelMessageTypes.ModelCommandResult, + RequestId = message.RequestId, + StatusCode = 404, + Error = $"Model '{modelName}' not found in Ollama models path." + }; + } + + var showResponse = new + { + modelfile = $"# llama.cpp via Docker\nFROM {model.BlobDigest}\n", + details = new + { + format = "gguf", + family = "llama", + parameter_size = "", + quantization_level = "" + }, + model_info = new { } + }; + + var body = JsonSerializer.SerializeToUtf8Bytes(showResponse, JsonOptions); + return new TunnelMessage + { + Type = TunnelMessageTypes.ModelCommandResult, + RequestId = message.RequestId, + StatusCode = 200, + ReasonPhrase = "OK", + Body = body + }; + } + + default: + throw new InvalidOperationException($"Unsupported model command '{message.Command}' with llama.cpp."); + } + } + private static TunnelMessage BuildModelCommandResult( string requestId, HttpResponseMessage response, @@ -394,6 +562,19 @@ internal sealed class TunnelClient }; } + private static TunnelMessage BuildModelCommandResult( + string requestId, int statusCode, string reasonPhrase, byte[] body) + { + return new TunnelMessage + { + Type = TunnelMessageTypes.ModelCommandResult, + RequestId = requestId, + StatusCode = statusCode, + ReasonPhrase = reasonPhrase, + Body = body + }; + } + private static bool ShouldRetryModelCommandWithEmbedding( string? command, HttpResponseMessage response, @@ -491,13 +672,31 @@ internal sealed class TunnelClient private void StartRequest(ClientWebSocket socket, TunnelMessage message, CancellationToken cancellationToken) { + Uri? effectiveUpstream = null; + + if (_llamaCppManager is not null) + { + var modelName = UpstreamRequest.ExtractModelName(message); + if (modelName is not null) + { + effectiveUpstream = _llamaCppManager.GetUpstream(modelName); + if (effectiveUpstream is null) + { + _logger.LogWarning( + "Request for model '{Model}' but no llama.cpp container is running for it. Falling back to default upstream.", + modelName); + } + } + } + var request = new UpstreamRequest( _options, _httpClient, message, (response, token) => SendAsync(socket, response, token), requestId => _activeRequests.TryRemove(requestId, out _), - cancellationToken); + cancellationToken, + effectiveUpstream: effectiveUpstream); if (!_activeRequests.TryAdd(message.RequestId, request)) { diff --git a/src/Ngino.Client/UpstreamRequest.cs b/src/Ngino.Client/UpstreamRequest.cs index 2c37270..0a94970 100644 --- a/src/Ngino.Client/UpstreamRequest.cs +++ b/src/Ngino.Client/UpstreamRequest.cs @@ -19,7 +19,8 @@ internal sealed class UpstreamRequest "Trailer", "Transfer-Encoding", "Upgrade", - ProtocolConstants.TokenHeader + ProtocolConstants.TokenHeader, + ProtocolConstants.ModelHeader }; private readonly CancellationTokenSource _cancellationTokenSource; @@ -33,7 +34,7 @@ internal sealed class UpstreamRequest private readonly HttpClient _httpClient; private readonly TunnelMessage _initialMessage; private readonly Action _onComplete; - private readonly ClientOptions _options; + private readonly Uri _upstream; private readonly Func _sendAsync; public UpstreamRequest( @@ -42,13 +43,14 @@ internal sealed class UpstreamRequest TunnelMessage initialMessage, Func sendAsync, Action onComplete, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + Uri? effectiveUpstream = null) { - _options = options; _httpClient = httpClient; _initialMessage = initialMessage; _sendAsync = sendAsync; _onComplete = onComplete; + _upstream = effectiveUpstream ?? options.Upstream; _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); if (!initialMessage.HasBody) @@ -110,10 +112,29 @@ internal sealed class UpstreamRequest } } + public static string? ExtractModelName(TunnelMessage message) + { + if (message.Headers is null) + { + return null; + } + + foreach (var header in message.Headers) + { + if (string.Equals(header.Name, ProtocolConstants.ModelHeader, StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(header.Value)) + { + return header.Value; + } + } + + return null; + } + private HttpRequestMessage BuildHttpRequest() { var method = new HttpMethod(_initialMessage.Method ?? HttpMethod.Get.Method); - var request = new HttpRequestMessage(method, BuildUpstreamUri(_options.Upstream, _initialMessage.PathAndQuery)); + var request = new HttpRequestMessage(method, BuildUpstreamUri(_upstream, _initialMessage.PathAndQuery)); if (_initialMessage.HasBody) { @@ -185,7 +206,7 @@ internal sealed class UpstreamRequest private async Task SendResponseBodyAsync(HttpResponseMessage response) { await using var stream = await response.Content.ReadAsStreamAsync(_cancellationTokenSource.Token); - var buffer = new byte[_options.ChunkSize]; + var buffer = new byte[_upstream switch { _ => 64 * 1024 }]; while (true) { diff --git a/src/Ngino.Protocol/ProtocolConstants.cs b/src/Ngino.Protocol/ProtocolConstants.cs index cf12645..7b649b9 100644 --- a/src/Ngino.Protocol/ProtocolConstants.cs +++ b/src/Ngino.Protocol/ProtocolConstants.cs @@ -7,4 +7,5 @@ public static class ProtocolConstants public const string TokenHeader = "X-Ngino-Token"; public const string ClientIdHeader = "X-Ngino-Client-Id"; public const string ReplacedCloseDescription = "ngino-replaced"; +public const string ModelHeader = "X-Ngino-Model"; } diff --git a/src/Ngino.Server/ReverseProxyEndpoint.cs b/src/Ngino.Server/ReverseProxyEndpoint.cs index ff7d704..e618c24 100644 --- a/src/Ngino.Server/ReverseProxyEndpoint.cs +++ b/src/Ngino.Server/ReverseProxyEndpoint.cs @@ -523,6 +523,12 @@ internal static class ReverseProxyEndpoint Headers = CollectRequestHeaders(context.Request, settings, managementStore) }; + if (!string.IsNullOrWhiteSpace(requestedModel)) + { + requestMessage.Headers ??= []; + requestMessage.Headers.Add(new HeaderPair(ProtocolConstants.ModelHeader, requestedModel)); + } + await connection.SendAsync(requestMessage, context.RequestAborted); requestBodyTask = ForwardRequestBodyAsync(context.Request, connection, requestId, hasBody, settings, logger); _ = requestBodyTask.ContinueWith(