feat(Client): adds llama.cpp backend

deploy/install-client.ps1

@@ -14,7 +14,11 @@ param(
     [string]$InstallDir = "$env:ProgramFiles\Ngino Client",
     [string]$ServiceName = "NginoClient",
     [switch]$InsecureSkipTlsVerify,
-    [switch]$NoOllama
+    [switch]$NoOllama,
+    [switch]$UseLlamaCppViaDocker,
+    [string]$UseOllamaModelsPath = "",
+    [string]$LlamaCppDockerImage = "",
+    [int]$LlamaCppBasePort = 0
 )
 
 $ErrorActionPreference = "Stop"
@@ -54,6 +58,14 @@ if ([string]::IsNullOrWhiteSpace($Token)) {
     finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($tokenPointer) }
 }
 if ([string]::IsNullOrWhiteSpace($Token)) { throw "Token is required." }
+
+if ($UseLlamaCppViaDocker -and [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) {
+    $UseOllamaModelsPath = Read-Host "Ollama models path (e.g. C:\Users\user\.ollama\models)"
+}
+if ($UseLlamaCppViaDocker -and [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) {
+    throw "Ollama models path is required with -UseLlamaCppViaDocker."
+}
+
 if ($ServiceName -notmatch '^[A-Za-z0-9_.-]+$') { throw "ServiceName contains unsupported characters." }
 
 $scriptDir = $PSScriptRoot
@@ -149,6 +161,18 @@ $serviceEnvironment = @(
     "DOTNET_CLI_TELEMETRY_OPTOUT=1",
     "DOTNET_NOLOGO=1"
 )
+if ($UseLlamaCppViaDocker) {
+    $serviceEnvironment += "NGINO_USE_LLAMA_CPP_VIA_DOCKER=true"
+    if (-not [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) {
+        $serviceEnvironment += "NGINO_USE_OLLAMA_MODELS_PATH=$UseOllamaModelsPath"
+    }
+    if (-not [string]::IsNullOrWhiteSpace($LlamaCppDockerImage)) {
+        $serviceEnvironment += "NGINO_LLAMA_CPP_DOCKER_IMAGE=$LlamaCppDockerImage"
+    }
+    if ($LlamaCppBasePort -gt 0) {
+        $serviceEnvironment += "NGINO_LLAMA_CPP_BASE_PORT=$LlamaCppBasePort"
+    }
+}
 if ($InsecureSkipTlsVerify) {
     Write-Warn "Server TLS certificate validation is disabled for $ServiceName."
 }
@@ -169,6 +193,9 @@ Write-Host "  Client ID:   $ClientId"
 Write-Host "  Upstream:    $Upstream"
 Write-Host "  Service:     $ServiceName"
 Write-Host "  Install dir: $InstallDir"
+if ($UseLlamaCppViaDocker) {
+    Write-Host "  llama.cpp:   enabled (models: $UseOllamaModelsPath)"
+}
 Write-Host ""
 Write-Host "  Manage:  Get-Service $ServiceName | Start-Service/Stop-Service/Restart-Service"
 Write-Host "  Logs:    Get-WinEvent -LogName Application | Where-Object ProviderName -eq NginoClient"

deploy/install-client.sh

@@ -16,6 +16,10 @@ TOKEN=""
 CLIENT_ID="$(hostname -s 2>/dev/null || echo "linux-client")"
 UPSTREAM="$DEFAULT_UPSTREAM"
 SKIP_OLLAMA=false
+USE_LLAMA_CPP_VIA_DOCKER=false
+USE_OLLAMA_MODELS_PATH=""
+LLAMA_CPP_DOCKER_IMAGE=""
+LLAMA_CPP_BASE_PORT=""
 
 # ── Colors ────────────────────────────────────────────────────────────────────
 RED='\033[0;31m'
@@ -45,11 +49,21 @@ Optional:
   --install-dir <dir>  Install directory; defaults to $DEFAULT_INSTALL_DIR
   --service-name <n>   systemd service name; defaults to $DEFAULT_SERVICE_NAME
   --no-ollama          Skip Ollama installation and status check
+  --use-llama-cpp-via-docker
+                       Use llama.cpp via Docker for inference instead of Ollama
+  --use-ollama-models-path <dir>
+                       Path to Ollama models directory (manifests/blobs); required with --use-llama-cpp-via-docker
+  --llama-cpp-docker-image <img>
+                       llama.cpp Docker image; defaults to auto-detected (rocm/cuda/cpu)
+  --llama-cpp-base-port <num>
+                       Base port for llama.cpp containers; defaults to 8081
   -h, --help           Show this help message
 
 Examples:
   $0 --server http://gpu-server:5050 --token "my-secret"
   $0 --server http://gpu-server:5050 --token "my-secret" --no-ollama
+  $0 --server http://gpu-server:5050 --token "my-secret" \\
+     --use-llama-cpp-via-docker --use-ollama-models-path /usr/share/ollama/.ollama/models
 EOF
     exit 0
 }
@@ -64,6 +78,10 @@ while [[ $# -gt 0 ]]; do
         --install-dir)              INSTALL_DIR="$2"; shift 2 ;;
         --service-name)             SERVICE_NAME="$2"; shift 2 ;;
         --no-ollama)                SKIP_OLLAMA=true; shift ;;
+        --use-llama-cpp-via-docker) USE_LLAMA_CPP_VIA_DOCKER=true; shift ;;
+        --use-ollama-models-path)   USE_OLLAMA_MODELS_PATH="$2"; shift 2 ;;
+        --llama-cpp-docker-image)   LLAMA_CPP_DOCKER_IMAGE="$2"; shift 2 ;;
+        --llama-cpp-base-port)      LLAMA_CPP_BASE_PORT="$2"; shift 2 ;;
         -h|--help)                  usage ;;
         *)                          die "Unknown option: $1" ;;
     esac
@@ -85,6 +103,13 @@ if [[ -z "$TOKEN" ]]; then
     die "Token is required."
 fi
 
+if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" && -z "$USE_OLLAMA_MODELS_PATH" ]]; then
+    read -rp "Ollama models path (e.g. /usr/share/ollama/.ollama/models): " USE_OLLAMA_MODELS_PATH
+fi
+if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" && -z "$USE_OLLAMA_MODELS_PATH" ]]; then
+    die "Ollama models path is required with --use-llama-cpp-via-docker."
+fi
+
 # ── Root check ────────────────────────────────────────────────────────────────
 if [[ $EUID -ne 0 ]]; then
     die "This script must be run as root (or with sudo)."
@@ -265,7 +290,21 @@ info "Client installed to $INSTALL_DIR."
 # ── Write environment file (avoids shell injection in unit file) ─────────────
 ENV_DIR="/etc/ngino-client"
 mkdir -p "$ENV_DIR"
-printf 'NGINO_TOKEN=%s\n' "$TOKEN" > "$ENV_DIR/env"
+{
+    printf 'NGINO_TOKEN=%s\n' "$TOKEN"
+    if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" ]]; then
+        printf 'NGINO_USE_LLAMA_CPP_VIA_DOCKER=true\n'
+        if [[ -n "$USE_OLLAMA_MODELS_PATH" ]]; then
+            printf 'NGINO_USE_OLLAMA_MODELS_PATH=%s\n' "$USE_OLLAMA_MODELS_PATH"
+        fi
+        if [[ -n "$LLAMA_CPP_DOCKER_IMAGE" ]]; then
+            printf 'NGINO_LLAMA_CPP_DOCKER_IMAGE=%s\n' "$LLAMA_CPP_DOCKER_IMAGE"
+        fi
+        if [[ -n "$LLAMA_CPP_BASE_PORT" ]]; then
+            printf 'NGINO_LLAMA_CPP_BASE_PORT=%s\n' "$LLAMA_CPP_BASE_PORT"
+        fi
+    fi
+} > "$ENV_DIR/env"
 chmod 600 "$ENV_DIR/env"
 info "Environment file written to $ENV_DIR/env (mode 0600)."
 
@@ -280,8 +319,8 @@ fi
 cat > "$SERVICE_FILE" <<EOF
 [Unit]
 Description=Ngino Tunnel Client
-After=network-online.target
-Wants=network-online.target
+After=network-online.target docker.service
+Wants=network-online.target docker.service
 $([ "$SKIP_OLLAMA" = "false" ] && echo "After=ollama.service")
 $([ "$SKIP_OLLAMA" = "false" ] && echo "Wants=ollama.service")
 
@@ -318,6 +357,9 @@ echo "  Client ID:   $CLIENT_ID"
 echo "  Upstream:    $UPSTREAM"
 echo "  Service:     $SERVICE_NAME"
 echo "  Install dir: $INSTALL_DIR"
+if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" ]]; then
+    echo "  llama.cpp:   enabled (models: $USE_OLLAMA_MODELS_PATH)"
+fi
 echo
 echo "  Manage:  systemctl {start|stop|restart|status} $SERVICE_NAME"
 echo "  Logs:    journalctl -u $SERVICE_NAME -f"

src/Ngino.Client/ClientOptions.cs

@@ -20,6 +20,14 @@ internal sealed class ClientOptions
 
     public bool InsecureSkipTlsVerify { get; init; }
 
+    public bool UseLlamaCppViaDocker { get; init; }
+
+    public string? UseOllamaModelsPath { get; init; }
+
+    public string? LlamaCppDockerImage { get; init; }
+
+    public int LlamaCppBasePort { get; init; } = 8081;
+
     public Uri TunnelUri
     {
         get
@@ -56,7 +64,11 @@ internal sealed class ClientOptions
             ClientId = Read(values, "client-id", "NGINO_CLIENT_ID") ?? Environment.MachineName.ToLowerInvariant(),
             ReconnectDelay = TimeSpan.FromSeconds(ReadInt(values, 5, "reconnect-delay", "NGINO_RECONNECT_DELAY_SECONDS")),
             ChunkSize = ReadInt(values, 64 * 1024, "chunk-size", "NGINO_CHUNK_SIZE"),
-            InsecureSkipTlsVerify = ReadBool(values, false, "insecure-skip-tls-verify", "NGINO_INSECURE_SKIP_TLS_VERIFY")
+            InsecureSkipTlsVerify = ReadBool(values, false, "insecure-skip-tls-verify", "NGINO_INSECURE_SKIP_TLS_VERIFY"),
+            UseLlamaCppViaDocker = ReadBool(values, false, "use-llama-cpp-via-docker", "NGINO_USE_LLAMA_CPP_VIA_DOCKER"),
+            UseOllamaModelsPath = NormalizeDirectoryPath(Read(values, "use-ollama-models-path", "NGINO_USE_OLLAMA_MODELS_PATH")),
+            LlamaCppDockerImage = Read(values, "llama-cpp-docker-image", "NGINO_LLAMA_CPP_DOCKER_IMAGE"),
+            LlamaCppBasePort = ReadInt(values, 8081, "llama-cpp-base-port", "NGINO_LLAMA_CPP_BASE_PORT")
         };
     }
 
@@ -71,6 +83,10 @@ internal sealed class ClientOptions
           --reconnect-delay <sec>        Defaults to 5
           --chunk-size <bytes>           Defaults to 65536
           --insecure-skip-tls-verify     Disable server TLS certificate validation (unsafe)
+          --use-llama-cpp-via-docker     Use llama.cpp via Docker for inference instead of Ollama
+          --use-ollama-models-path <dir> Path to Ollama models directory (manifests/blobs), required with --use-llama-cpp-via-docker
+          --llama-cpp-docker-image <img> llama.cpp Docker image; defaults to auto-detected (rocm/cuda/cpu)
+          --llama-cpp-base-port <num>    Base port for llama.cpp containers; defaults to 8081
         """;
 
     private static Dictionary<string, string> ParseArgs(string[] args)
@@ -92,7 +108,7 @@ internal sealed class ClientOptions
                 continue;
             }
 
-            if (keyValue[0].Equals("insecure-skip-tls-verify", StringComparison.OrdinalIgnoreCase))
+            if (IsBoolFlag(keyValue[0]))
             {
                 values[keyValue[0]] = "true";
                 continue;
@@ -109,6 +125,16 @@ internal sealed class ClientOptions
         return values;
     }
 
+    private static bool IsBoolFlag(string key)
+    {
+        return key switch
+        {
+            "insecure-skip-tls-verify" => true,
+            "use-llama-cpp-via-docker" => true,
+            _ => false
+        };
+    }
+
     private static string? Read(Dictionary<string, string> values, params string[] keys)
     {
         foreach (var key in keys)
@@ -154,4 +180,14 @@ internal sealed class ClientOptions
 
     private static string NormalizePath(string path) =>
         path.StartsWith('/') ? path : $"/{path}";
+
+    private static string? NormalizeDirectoryPath(string? path)
+    {
+        if (string.IsNullOrWhiteSpace(path))
+        {
+            return null;
+        }
+
+        return Path.GetFullPath(path);
+    }
 }

src/Ngino.Client/LlamaCppManager.cs

@@ -0,0 +1,471 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Ngino.Client;
+
+internal sealed partial class LlamaCppManager : IAsyncDisposable
+{
+    private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
+    private const int DefaultBasePort = 8081;
+    private const string NginoContainerLabel = "ngino-llamacpp";
+    private static readonly TimeSpan DockerTimeout = TimeSpan.FromSeconds(60);
+
+    private readonly string _blobsPath;
+    private readonly string _manifestsPath;
+    private readonly string _dockerImage;
+    private readonly int _basePort;
+    private readonly ILogger _logger;
+    private readonly ConcurrentDictionary<string, int> _modelPorts = new(StringComparer.OrdinalIgnoreCase);
+
+    public LlamaCppManager(
+        string ollamaModelsPath,
+        string? dockerImage,
+        int? basePort,
+        ILogger? logger = null)
+    {
+        _manifestsPath = Path.Combine(ollamaModelsPath, "manifests");
+        _blobsPath = Path.Combine(ollamaModelsPath, "blobs");
+        _dockerImage = dockerImage ?? GetDefaultDockerImage();
+        _basePort = basePort ?? DefaultBasePort;
+        _logger = logger ?? NullLogger<LlamaCppManager>.Instance;
+    }
+
+    public string DockerImage => _dockerImage;
+
+    public List<LlamaCppModel> DiscoverModels()
+    {
+        var models = new List<LlamaCppModel>();
+
+        if (!Directory.Exists(_manifestsPath))
+        {
+            _logger.LogWarning("Ollama manifests path not found: {ManifestsPath}", _manifestsPath);
+            return models;
+        }
+
+        foreach (var manifestPath in Directory.EnumerateFiles(_manifestsPath, "*", SearchOption.AllDirectories))
+        {
+            try
+            {
+                var model = ParseManifest(manifestPath);
+                if (model is not null)
+                {
+                    models.Add(model);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogDebug(ex, "Failed to parse manifest: {ManifestPath}", manifestPath);
+            }
+        }
+
+        return models;
+    }
+
+    public List<LlamaCppModel> DiscoverModelsWithBlob()
+    {
+        return DiscoverModels().Where(m => File.Exists(m.BlobPath)).ToList();
+    }
+
+    public bool IsModelActive(string ollamaModelName)
+    {
+        return _modelPorts.ContainsKey(ollamaModelName);
+    }
+
+    public Uri? GetUpstream(string ollamaModelName)
+    {
+        if (_modelPorts.TryGetValue(ollamaModelName, out var port))
+        {
+            return new Uri($"http://localhost:{port}");
+        }
+
+        return null;
+    }
+
+    public async Task<bool> StartModelContainerAsync(LlamaCppModel model, CancellationToken cancellationToken)
+    {
+        var ollamaName = model.OllamaName;
+        if (string.IsNullOrWhiteSpace(ollamaName))
+        {
+            _logger.LogWarning("Cannot start container: model has no Ollama name");
+            return false;
+        }
+
+        if (_modelPorts.ContainsKey(ollamaName))
+        {
+            _logger.LogInformation("Model {Model} already has a running container", ollamaName);
+            return true;
+        }
+
+        if (!File.Exists(model.BlobPath))
+        {
+            _logger.LogError("Model blob not found: {BlobPath}", model.BlobPath);
+            return false;
+        }
+
+        var port = FindAvailablePort();
+        var containerName = SanitizeContainerName($"ngino-llamacpp-{ollamaName}");
+
+        var args = BuildDockerRunArgs(containerName, model, port);
+        _logger.LogInformation(
+            "Starting llama.cpp container for {Model} on port {Port}: docker {Args}",
+            ollamaName, port, string.Join(" ", args));
+
+        try
+        {
+            var (exitCode, output) = await RunDockerWithOutputAsync(args, cancellationToken);
+            if (exitCode != 0)
+            {
+                _logger.LogError(
+                    "Failed to start llama.cpp container for {Model}, exit code: {ExitCode}, output: {Output}",
+                    ollamaName, exitCode, output);
+                return false;
+            }
+
+            _modelPorts[ollamaName] = port;
+            _logger.LogInformation("llama.cpp container for {Model} started on port {Port}", ollamaName, port);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to start llama.cpp container for {Model}", ollamaName);
+            return false;
+        }
+    }
+
+    public async Task<bool> StopModelContainerAsync(string ollamaModelName, CancellationToken cancellationToken)
+    {
+        if (!_modelPorts.TryRemove(ollamaModelName, out _))
+        {
+            _logger.LogWarning("No running container found for model {Model}", ollamaModelName);
+            return false;
+        }
+
+        var containerName = SanitizeContainerName($"ngino-llamacpp-{ollamaModelName}");
+
+        _logger.LogInformation("Stopping llama.cpp container {ContainerName}", containerName);
+
+        try
+        {
+            await RunDockerAsync(["stop", "--time", "10", containerName], cancellationToken);
+            await RunDockerAsync(["rm", "-f", containerName], cancellationToken);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to stop container {ContainerName}", containerName);
+            return false;
+        }
+    }
+
+    public async Task StopAllContainersAsync()
+    {
+        _logger.LogInformation("Stopping all llama.cpp containers...");
+
+        try
+        {
+            var (exitCode, output) = await RunDockerWithOutputAsync(
+                ["ps", "-q", "--filter", $"label={NginoContainerLabel}"],
+                CancellationToken.None);
+
+            if (exitCode == 0 && !string.IsNullOrWhiteSpace(output))
+            {
+                var containerIds = output.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+                foreach (var id in containerIds)
+                {
+                    await RunDockerAsync(["stop", "--time", "10", id], CancellationToken.None);
+                    await RunDockerAsync(["rm", "-f", id], CancellationToken.None);
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to stop all llama.cpp containers");
+        }
+
+        _modelPorts.Clear();
+    }
+
+    public async Task<bool> TestDockerAsync()
+    {
+        try
+        {
+            var (exitCode, _) = await RunDockerWithOutputAsync(["info", "--format", "{{.ServerVersion}}"], CancellationToken.None);
+            return exitCode == 0;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        await StopAllContainersAsync();
+    }
+
+    private LlamaCppModel? ParseManifest(string manifestPath)
+    {
+        var json = File.ReadAllText(manifestPath);
+        using var document = JsonDocument.Parse(json);
+        var root = document.RootElement;
+
+        if (!root.TryGetProperty("layers", out var layers) || layers.ValueKind != JsonValueKind.Array)
+        {
+            return null;
+        }
+
+        string? modelDigest = null;
+        foreach (var layer in layers.EnumerateArray())
+        {
+            if (layer.TryGetProperty("mediaType", out var mediaType)
+                && mediaType.GetString() == "application/vnd.ollama.image.model"
+                && layer.TryGetProperty("digest", out var digest))
+            {
+                modelDigest = digest.GetString();
+                break;
+            }
+        }
+
+        if (string.IsNullOrWhiteSpace(modelDigest))
+        {
+            return null;
+        }
+
+        var modelName = ResolveModelName(manifestPath);
+        if (modelName is null)
+        {
+            return null;
+        }
+
+        var blobName = modelDigest.Replace(":", "-", StringComparison.Ordinal);
+        var blobPath = Path.GetFullPath(Path.Combine(_blobsPath, blobName));
+
+        return new LlamaCppModel
+        {
+            OllamaName = modelName,
+            BlobDigest = blobName,
+            BlobPath = blobPath,
+            ManifestPath = manifestPath
+        };
+    }
+
+    private static string? ResolveModelName(string manifestPath)
+    {
+        var normalizedPath = manifestPath.Replace('\\', '/');
+        var parts = normalizedPath.Split('/', StringSplitOptions.RemoveEmptyEntries);
+
+        var manifestIndex = Array.FindLastIndex(parts, p =>
+            p.Equals("manifests", StringComparison.OrdinalIgnoreCase));
+
+        if (manifestIndex < 0 || manifestIndex >= parts.Length - 1)
+        {
+            return null;
+        }
+
+        var relativeParts = parts[(manifestIndex + 1)..];
+
+        if (relativeParts.Length < 2)
+        {
+            return null;
+        }
+
+        var registry = relativeParts[0];
+
+        if (registry.Equals("registry.ollama.ai", StringComparison.OrdinalIgnoreCase))
+        {
+            if (relativeParts.Length < 3)
+            {
+                return null;
+            }
+
+            if (relativeParts[1].Equals("library", StringComparison.OrdinalIgnoreCase) && relativeParts.Length >= 4)
+            {
+                return $"{relativeParts[2]}:{relativeParts[3]}";
+            }
+
+            if (relativeParts.Length == 3)
+            {
+                return $"{relativeParts[1]}:{relativeParts[2]}";
+            }
+
+            return null;
+        }
+
+        if (relativeParts.Length >= 3)
+        {
+            var tag = relativeParts[^1];
+            var modelPath = string.Join("/", relativeParts.Take(relativeParts.Length - 1));
+            return $"{modelPath}:{tag}";
+        }
+
+        return null;
+    }
+
+    private string[] BuildDockerRunArgs(string containerName, LlamaCppModel model, int port)
+    {
+        var blobsDir = Path.GetDirectoryName(Path.GetFullPath(model.BlobPath))!;
+        var blobFile = Path.GetFileName(model.BlobPath);
+
+        var args = new List<string>
+        {
+            "run",
+            "-d",
+            "--rm",
+            "--label", $"{NginoContainerLabel}=true",
+            "--name", containerName,
+            "-p", $"{port}:{port}",
+            "-v", $"{blobsDir}:/models/blobs:ro",
+        };
+
+        if (HasRocmDevices())
+        {
+            args.Add("--device=/dev/kfd");
+            args.Add("--device=/dev/dri");
+            args.Add("--group-add=video");
+        }
+
+        if (HasNvidiaGpu() && !HasRocmDevices())
+        {
+            args.Add("--gpus=all");
+        }
+
+        args.Add(_dockerImage);
+        args.Add("-m");
+        args.Add($"/models/blobs/{blobFile}");
+        args.Add("-ngl");
+        args.Add("999");
+        args.Add("--parallel");
+        args.Add("4");
+        args.Add("--host");
+        args.Add("0.0.0.0");
+        args.Add("--port");
+        args.Add(port.ToString());
+
+        return [.. args];
+    }
+
+    private int FindAvailablePort()
+    {
+        var usedPorts = new HashSet<int>(_modelPorts.Values);
+        var port = _basePort;
+
+        while (usedPorts.Contains(port))
+        {
+            port++;
+        }
+
+        return port;
+    }
+
+    private static string SanitizeContainerName(string name)
+    {
+        var sanitized = InvalidContainerNameChars().Replace(name, "_");
+        return sanitized.Trim('_').ToLowerInvariant();
+    }
+
+    private async Task<int> RunDockerAsync(string[] args, CancellationToken cancellationToken)
+    {
+        var (exitCode, _) = await RunDockerWithOutputAsync(args, cancellationToken);
+        return exitCode;
+    }
+
+    private async Task<(int ExitCode, string Output)> RunDockerWithOutputAsync(
+        string[] args, CancellationToken cancellationToken)
+    {
+        var process = new Process
+        {
+            StartInfo = new ProcessStartInfo
+            {
+                FileName = "docker",
+                Arguments = string.Join(" ", args.Select(a => a.Contains(' ') ? $"\"{a}\"" : a)),
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            }
+        };
+
+        process.Start();
+
+        var readOutput = process.StandardOutput.ReadToEndAsync(cancellationToken);
+        var readError = process.StandardError.ReadToEndAsync(cancellationToken);
+
+        var completed = await Task.WhenAny(
+            process.WaitForExitAsync(cancellationToken),
+            Task.Delay(DockerTimeout, cancellationToken));
+
+        string output;
+        string error;
+
+        try
+        {
+            output = await readOutput;
+            error = await readError;
+        }
+        catch
+        {
+            output = "";
+            error = "timed out";
+        }
+
+        if (completed is Task { IsCompleted: true } delayTask
+            && delayTask != process.WaitForExitAsync(cancellationToken))
+        {
+            _logger.LogWarning("Docker command timed out: docker {Args}", string.Join(" ", args));
+            try { process.Kill(entireProcessTree: true); } catch { }
+            return (-1, error);
+        }
+
+        if (!string.IsNullOrWhiteSpace(error))
+        {
+            output = $"{output}\n{error}".Trim();
+        }
+
+        return (process.ExitCode, output);
+    }
+
+    private static string GetDefaultDockerImage()
+    {
+        if (HasRocmDevices())
+        {
+            return "ghcr.io/ggml-org/llama.cpp:server-rocm";
+        }
+
+        if (HasNvidiaGpu())
+        {
+            return "ghcr.io/ggml-org/llama.cpp:server-cuda";
+        }
+
+        return "ghcr.io/ggml-org/llama.cpp:server";
+    }
+
+    private static bool HasRocmDevices() => File.Exists("/dev/kfd") && File.Exists("/dev/dri");
+
+    private static bool HasNvidiaGpu()
+    {
+        try
+        {
+            return File.Exists("/proc/driver/nvidia/version")
+                || Directory.Exists("/proc/driver/nvidia/gpus");
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
+    [GeneratedRegex(@"[^a-zA-Z0-9_.-]")]
+    private static partial Regex InvalidContainerNameChars();
+}
+
+internal sealed record LlamaCppModel
+{
+    public required string OllamaName { get; init; }
+    public required string BlobDigest { get; init; }
+    public required string BlobPath { get; init; }
+    public required string ManifestPath { get; init; }
+}

src/Ngino.Client/Program.cs

@@ -16,6 +16,14 @@ try
         Console.WriteLine("  WARNING: server TLS certificate validation is disabled");
     }
 
+    if (options.UseLlamaCppViaDocker)
+    {
+        Console.WriteLine($"  llama.cpp via Docker: enabled");
+        Console.WriteLine($"  ollama models path: {options.UseOllamaModelsPath ?? "(not set)"}");
+        Console.WriteLine($"  llama.cpp docker image: {options.LlamaCppDockerImage ?? "(auto)"}");
+        Console.WriteLine($"  llama.cpp base port: {options.LlamaCppBasePort}");
+    }
+
     // Args are parsed by ClientOptions; keep them away from the host configuration.
     var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings { Args = [] });
     builder.Services.AddSingleton(options);

src/Ngino.Client/TunnelClient.cs

@@ -21,6 +21,7 @@ internal sealed class TunnelClient
     private readonly ILogger<TunnelClient> _logger;
     private readonly object _modelSnapshotLock = new();
     private readonly SemaphoreSlim _sendLock = new(1, 1);
+    private readonly LlamaCppManager? _llamaCppManager;
     private List<string> _lastActiveModels = [];
     private List<string> _lastModels = [];
 
@@ -32,10 +33,39 @@ internal sealed class TunnelClient
         {
             Timeout = Timeout.InfiniteTimeSpan
         };
+
+        if (_options.UseLlamaCppViaDocker)
+        {
+            if (string.IsNullOrWhiteSpace(_options.UseOllamaModelsPath))
+            {
+                throw new InvalidOperationException(
+                    "--use-ollama-models-path is required when --use-llama-cpp-via-docker is set.");
+            }
+
+            _llamaCppManager = new LlamaCppManager(
+                _options.UseOllamaModelsPath,
+                _options.LlamaCppDockerImage,
+                _options.LlamaCppBasePort,
+                _logger);
+        }
     }
 
     public async Task RunAsync(CancellationToken cancellationToken)
     {
+        if (_llamaCppManager is not null)
+        {
+            _logger.LogInformation("Testing Docker availability...");
+            var dockerAvailable = await _llamaCppManager.TestDockerAsync();
+            if (!dockerAvailable)
+            {
+                _logger.LogWarning("Docker is not available. llama.cpp via Docker will not work.");
+            }
+            else
+            {
+                _logger.LogInformation("Docker is available. Using llama.cpp image: {Image}", _llamaCppManager.DockerImage);
+            }
+        }
+
         while (!cancellationToken.IsCancellationRequested)
         {
             using var socket = new ClientWebSocket();
@@ -154,6 +184,16 @@ internal sealed class TunnelClient
 
     private async Task<List<string>> GetUpstreamModelsAsync(CancellationToken cancellationToken)
     {
+        if (_llamaCppManager is not null)
+        {
+            var models = _llamaCppManager.DiscoverModelsWithBlob();
+            return models
+                .Select(m => m.OllamaName)
+                .Where(name => !string.IsNullOrWhiteSpace(name))
+                .OrderBy(name => name, StringComparer.OrdinalIgnoreCase)
+                .ToList();
+        }
+
         using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_options.Upstream, "/api/tags"));
         using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
         response.EnsureSuccessStatusCode();
@@ -166,6 +206,17 @@ internal sealed class TunnelClient
 
     private async Task<List<string>> GetActiveUpstreamModelsAsync(CancellationToken cancellationToken)
     {
+        if (_llamaCppManager is not null)
+        {
+            var models = _llamaCppManager.DiscoverModelsWithBlob();
+            return models
+                .Where(m => _llamaCppManager.IsModelActive(m.OllamaName))
+                .Select(m => m.OllamaName)
+                .Where(name => !string.IsNullOrWhiteSpace(name))
+                .OrderBy(name => name, StringComparer.OrdinalIgnoreCase)
+                .ToList();
+        }
+
         using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_options.Upstream, "/api/ps"));
         using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
 
@@ -363,6 +414,11 @@ internal sealed class TunnelClient
             throw new InvalidOperationException("Model command is missing a model name.");
         }
 
+        if (_llamaCppManager is not null)
+        {
+            return await ExecuteModelCommandWithLlamaCppAsync(message, cancellationToken);
+        }
+
         using var request = BuildModelCommandRequest(message);
         using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
         var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
@@ -379,6 +435,118 @@ internal sealed class TunnelClient
         return BuildModelCommandResult(message.RequestId, response, body);
     }
 
+    private async Task<TunnelMessage> ExecuteModelCommandWithLlamaCppAsync(
+        TunnelMessage message, CancellationToken cancellationToken)
+    {
+        var modelName = message.Model?.Trim();
+        var normalizedCommand = NormalizeModelCommand(message.Command);
+
+        switch (normalizedCommand)
+        {
+            case "load":
+            {
+                var models = _llamaCppManager!.DiscoverModelsWithBlob();
+                var model = models.FirstOrDefault(m =>
+                    string.Equals(m.OllamaName, modelName, StringComparison.OrdinalIgnoreCase));
+
+                if (model is null)
+                {
+                    return new TunnelMessage
+                    {
+                        Type = TunnelMessageTypes.ModelCommandResult,
+                        RequestId = message.RequestId,
+                        StatusCode = 404,
+                        Error = $"Model '{modelName}' not found in Ollama models path."
+                    };
+                }
+
+                var started = await _llamaCppManager.StartModelContainerAsync(model, cancellationToken);
+                if (!started)
+                {
+                    return new TunnelMessage
+                    {
+                        Type = TunnelMessageTypes.ModelCommandResult,
+                        RequestId = message.RequestId,
+                        StatusCode = 500,
+                        Error = $"Failed to start llama.cpp container for model '{modelName}'."
+                    };
+                }
+
+                return BuildModelCommandResult(message.RequestId, 200, "OK", []);
+            }
+
+            case "unload":
+            {
+                var stopped = await _llamaCppManager!.StopModelContainerAsync(modelName!, cancellationToken);
+                if (!stopped)
+                {
+                    return new TunnelMessage
+                    {
+                        Type = TunnelMessageTypes.ModelCommandResult,
+                        RequestId = message.RequestId,
+                        StatusCode = 404,
+                        Error = $"No running llama.cpp container for model '{modelName}'."
+                    };
+                }
+
+                return BuildModelCommandResult(message.RequestId, 200, "OK", []);
+            }
+
+            case "pull":
+            case "delete":
+            {
+                using var request = BuildModelCommandRequest(_options.Upstream, message.Command, message.Model);
+                using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+                var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
+                return BuildModelCommandResult(message.RequestId, response, body);
+            }
+
+            case "show":
+            {
+                var models = _llamaCppManager!.DiscoverModelsWithBlob();
+                var model = models.FirstOrDefault(m =>
+                    string.Equals(m.OllamaName, modelName, StringComparison.OrdinalIgnoreCase));
+
+                if (model is null)
+                {
+                    return new TunnelMessage
+                    {
+                        Type = TunnelMessageTypes.ModelCommandResult,
+                        RequestId = message.RequestId,
+                        StatusCode = 404,
+                        Error = $"Model '{modelName}' not found in Ollama models path."
+                    };
+                }
+
+                var showResponse = new
+                {
+                    modelfile = $"# llama.cpp via Docker\nFROM {model.BlobDigest}\n",
+                    details = new
+                    {
+                        format = "gguf",
+                        family = "llama",
+                        parameter_size = "",
+                        quantization_level = ""
+                    },
+                    model_info = new { }
+                };
+
+                var body = JsonSerializer.SerializeToUtf8Bytes(showResponse, JsonOptions);
+                return new TunnelMessage
+                {
+                    Type = TunnelMessageTypes.ModelCommandResult,
+                    RequestId = message.RequestId,
+                    StatusCode = 200,
+                    ReasonPhrase = "OK",
+                    Body = body
+                };
+            }
+
+            default:
+                throw new InvalidOperationException($"Unsupported model command '{message.Command}' with llama.cpp.");
+        }
+    }
+
     private static TunnelMessage BuildModelCommandResult(
         string requestId,
         HttpResponseMessage response,
@@ -394,6 +562,19 @@ internal sealed class TunnelClient
         };
     }
 
+    private static TunnelMessage BuildModelCommandResult(
+        string requestId, int statusCode, string reasonPhrase, byte[] body)
+    {
+        return new TunnelMessage
+        {
+            Type = TunnelMessageTypes.ModelCommandResult,
+            RequestId = requestId,
+            StatusCode = statusCode,
+            ReasonPhrase = reasonPhrase,
+            Body = body
+        };
+    }
+
     private static bool ShouldRetryModelCommandWithEmbedding(
         string? command,
         HttpResponseMessage response,
@@ -491,13 +672,31 @@ internal sealed class TunnelClient
 
     private void StartRequest(ClientWebSocket socket, TunnelMessage message, CancellationToken cancellationToken)
     {
+        Uri? effectiveUpstream = null;
+
+        if (_llamaCppManager is not null)
+        {
+            var modelName = UpstreamRequest.ExtractModelName(message);
+            if (modelName is not null)
+            {
+                effectiveUpstream = _llamaCppManager.GetUpstream(modelName);
+                if (effectiveUpstream is null)
+                {
+                    _logger.LogWarning(
+                        "Request for model '{Model}' but no llama.cpp container is running for it. Falling back to default upstream.",
+                        modelName);
+                }
+            }
+        }
+
         var request = new UpstreamRequest(
             _options,
             _httpClient,
             message,
             (response, token) => SendAsync(socket, response, token),
             requestId => _activeRequests.TryRemove(requestId, out _),
-            cancellationToken);
+            cancellationToken,
+            effectiveUpstream: effectiveUpstream);
 
         if (!_activeRequests.TryAdd(message.RequestId, request))
         {

src/Ngino.Client/UpstreamRequest.cs

@@ -19,7 +19,8 @@ internal sealed class UpstreamRequest
         "Trailer",
         "Transfer-Encoding",
         "Upgrade",
-        ProtocolConstants.TokenHeader
+        ProtocolConstants.TokenHeader,
+        ProtocolConstants.ModelHeader
     };
 
     private readonly CancellationTokenSource _cancellationTokenSource;
@@ -33,7 +34,7 @@ internal sealed class UpstreamRequest
     private readonly HttpClient _httpClient;
     private readonly TunnelMessage _initialMessage;
     private readonly Action<string> _onComplete;
-    private readonly ClientOptions _options;
+    private readonly Uri _upstream;
     private readonly Func<TunnelMessage, CancellationToken, Task> _sendAsync;
 
     public UpstreamRequest(
@@ -42,13 +43,14 @@ internal sealed class UpstreamRequest
         TunnelMessage initialMessage,
         Func<TunnelMessage, CancellationToken, Task> sendAsync,
         Action<string> onComplete,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken,
+        Uri? effectiveUpstream = null)
     {
-        _options = options;
         _httpClient = httpClient;
         _initialMessage = initialMessage;
         _sendAsync = sendAsync;
         _onComplete = onComplete;
+        _upstream = effectiveUpstream ?? options.Upstream;
         _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
 
         if (!initialMessage.HasBody)
@@ -110,10 +112,29 @@ internal sealed class UpstreamRequest
         }
     }
 
+    public static string? ExtractModelName(TunnelMessage message)
+    {
+        if (message.Headers is null)
+        {
+            return null;
+        }
+
+        foreach (var header in message.Headers)
+        {
+            if (string.Equals(header.Name, ProtocolConstants.ModelHeader, StringComparison.OrdinalIgnoreCase)
+                && !string.IsNullOrWhiteSpace(header.Value))
+            {
+                return header.Value;
+            }
+        }
+
+        return null;
+    }
+
     private HttpRequestMessage BuildHttpRequest()
     {
         var method = new HttpMethod(_initialMessage.Method ?? HttpMethod.Get.Method);
-        var request = new HttpRequestMessage(method, BuildUpstreamUri(_options.Upstream, _initialMessage.PathAndQuery));
+        var request = new HttpRequestMessage(method, BuildUpstreamUri(_upstream, _initialMessage.PathAndQuery));
 
         if (_initialMessage.HasBody)
         {
@@ -185,7 +206,7 @@ internal sealed class UpstreamRequest
     private async Task SendResponseBodyAsync(HttpResponseMessage response)
     {
         await using var stream = await response.Content.ReadAsStreamAsync(_cancellationTokenSource.Token);
-        var buffer = new byte[_options.ChunkSize];
+        var buffer = new byte[_upstream switch { _ => 64 * 1024 }];
 
         while (true)
         {

src/Ngino.Protocol/ProtocolConstants.cs

@@ -7,4 +7,5 @@ public static class ProtocolConstants
     public const string TokenHeader = "X-Ngino-Token";
     public const string ClientIdHeader = "X-Ngino-Client-Id";
     public const string ReplacedCloseDescription = "ngino-replaced";
+public const string ModelHeader = "X-Ngino-Model";
 }

src/Ngino.Server/ReverseProxyEndpoint.cs

@@ -523,6 +523,12 @@ internal static class ReverseProxyEndpoint
                 Headers = CollectRequestHeaders(context.Request, settings, managementStore)
             };
 
+            if (!string.IsNullOrWhiteSpace(requestedModel))
+            {
+                requestMessage.Headers ??= [];
+                requestMessage.Headers.Add(new HeaderPair(ProtocolConstants.ModelHeader, requestedModel));
+            }
+
             await connection.SendAsync(requestMessage, context.RequestAborted);
             requestBodyTask = ForwardRequestBodyAsync(context.Request, connection, requestId, hasBody, settings, logger);
             _ = requestBodyTask.ContinueWith(