fix(client): fixes installer and inferencing issues

README.md

@@ -132,6 +132,15 @@ sudo bash deploy/install-client.sh --server http://your-server:5050 --token "cha
 
 Options: `--server`, `--token` (required); `--client-id`, `--upstream`, `--install-dir`, `--service-name`, `--no-ollama` (optional). Missing required values are prompted interactively.
 
+llama.cpp via Docker options (replaces Ollama for inferencing):
+
+| Option | Description |
+|--------|-------------|
+| `--use-llama-cpp-via-docker` | Use llama.cpp Docker containers instead of Ollama |
+| `--use-ollama-models-path <dir>` | Path to Ollama models directory (`manifests/blobs`); required with the flag above |
+| `--llama-cpp-docker-image <img>` | Docker image; defaults to auto-detected (rocm/cuda/cpu) |
+| `--llama-cpp-base-port <num>` | Base port for containers; defaults to `8081` |
+
 The script ensures .NET 10 and Ollama are installed, builds the client self-contained, installs it to `/opt/Ngino-client`, and creates a systemd service (`Ngino-client`). Logs: `journalctl -u Ngino-client -f`.
 
 ## Notes

deploy/install-client.ps1

@@ -0,0 +1,201 @@
+#Requires -Version 5.1
+#Requires -RunAsAdministrator
+
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $false)]
+    [string]$Server,
+
+    [Parameter(Mandatory = $false)]
+    [string]$Token,
+
+    [string]$ClientId = $(if ($env:COMPUTERNAME) { $env:COMPUTERNAME.ToLowerInvariant() } else { "windows-client" }),
+    [string]$Upstream = "http://localhost:11434",
+    [string]$InstallDir = "$env:ProgramFiles\Ngino Client",
+    [string]$ServiceName = "NginoClient",
+    [switch]$InsecureSkipTlsVerify,
+    [switch]$NoOllama,
+    [switch]$UseLlamaCppViaDocker,
+    [string]$UseOllamaModelsPath = "",
+    [string]$LlamaCppDockerImage = "",
+    [int]$LlamaCppBasePort = 0
+)
+
+$ErrorActionPreference = "Stop"
+$ProgressPreference = "SilentlyContinue"
+
+function Write-Info([string]$Message) { Write-Host "[INFO]  $Message" -ForegroundColor Green }
+function Write-Warn([string]$Message) { Write-Host "[WARN]  $Message" -ForegroundColor Yellow }
+
+function Find-DotNet {
+    $command = Get-Command dotnet -ErrorAction SilentlyContinue
+    if ($command) { return $command.Source }
+
+    $candidate = Join-Path $env:ProgramFiles "dotnet\dotnet.exe"
+    if (Test-Path -LiteralPath $candidate) { return $candidate }
+    return $null
+}
+
+function Install-WingetPackage([string]$Id, [string]$Name) {
+    if (-not (Get-Command winget.exe -ErrorAction SilentlyContinue)) {
+        throw "$Name is required but winget is unavailable. Install $Name manually and run this script again."
+    }
+
+    Write-Info "Installing $Name..."
+    & winget.exe install --id $Id --exact --accept-package-agreements --accept-source-agreements --silent
+    if ($LASTEXITCODE -ne 0) { throw "winget failed to install $Name (exit code $LASTEXITCODE)." }
+}
+
+if ([string]::IsNullOrWhiteSpace($Server)) {
+    $Server = Read-Host "Ngino server URL (e.g. http://my-server:5050)"
+}
+if ([string]::IsNullOrWhiteSpace($Server)) { throw "Server URL is required." }
+
+if ([string]::IsNullOrWhiteSpace($Token)) {
+    $secureToken = Read-Host "Server token" -AsSecureString
+    $tokenPointer = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secureToken)
+    try { $Token = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($tokenPointer) }
+    finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($tokenPointer) }
+}
+if ([string]::IsNullOrWhiteSpace($Token)) { throw "Token is required." }
+
+if ($UseLlamaCppViaDocker -and [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) {
+    $UseOllamaModelsPath = Read-Host "Ollama models path (e.g. C:\Users\user\.ollama\models)"
+}
+if ($UseLlamaCppViaDocker -and [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) {
+    throw "Ollama models path is required with -UseLlamaCppViaDocker."
+}
+
+if ($ServiceName -notmatch '^[A-Za-z0-9_.-]+$') { throw "ServiceName contains unsupported characters." }
+
+$scriptDir = $PSScriptRoot
+$repoRoot = Split-Path -Parent $scriptDir
+$clientProject = Join-Path $repoRoot "src\Ngino.Client\Ngino.Client.csproj"
+if (-not (Test-Path -LiteralPath $clientProject)) {
+    throw "Client source not found at $clientProject. Run this script from the repository."
+}
+
+$dotnet = Find-DotNet
+$dotnetVersion = if ($dotnet) { & $dotnet --version } else { $null }
+if (-not $dotnetVersion -or -not $dotnetVersion.StartsWith("10.")) {
+    if ($dotnetVersion) { Write-Warn "dotnet $dotnetVersion is installed, but version 10.x is required." }
+    Install-WingetPackage "Microsoft.DotNet.SDK.10" ".NET 10 SDK"
+    $dotnet = Find-DotNet
+    if (-not $dotnet) { throw ".NET was installed, but dotnet.exe could not be found." }
+    $dotnetVersion = & $dotnet --version
+}
+Write-Info "Using dotnet $dotnetVersion ($dotnet)."
+
+if ($NoOllama) {
+    Write-Info "Skipping Ollama check (-NoOllama)."
+} else {
+    $ollama = Get-Command ollama.exe -ErrorAction SilentlyContinue
+    if (-not $ollama) {
+        $ollamaCandidate = Join-Path $env:LOCALAPPDATA "Programs\Ollama\ollama.exe"
+        if (Test-Path -LiteralPath $ollamaCandidate) { $ollama = Get-Item $ollamaCandidate }
+    }
+    if (-not $ollama) {
+        Install-WingetPackage "Ollama.Ollama" "Ollama"
+        $ollamaCandidate = Join-Path $env:LOCALAPPDATA "Programs\Ollama\ollama.exe"
+        if (Test-Path -LiteralPath $ollamaCandidate) { $ollama = Get-Item $ollamaCandidate }
+    }
+    if (-not $ollama) { Write-Warn "Ollama was installed, but ollama.exe was not found in the current session." }
+    else {
+        $ollamaPath = if ($ollama.Source) { $ollama.Source } elseif ($ollama.FullName) { $ollama.FullName } else { $ollama.Path }
+        Write-Info "Ollama is installed ($ollamaPath)."
+    }
+}
+
+$architecture = $env:PROCESSOR_ARCHITECTURE
+$runtimeId = switch ($architecture) {
+    { $_ -in "AMD64", "x64" } { "win-x64"; break }
+    { $_ -in "ARM64", "Arm64" } { "win-arm64"; break }
+    { $_ -in "x86", "X86" } { "win-x86"; break }
+    default { throw "Unsupported architecture: $architecture" }
+}
+
+$buildDir = Join-Path ([IO.Path]::GetTempPath()) ("ngino-build-" + [Guid]::NewGuid().ToString("N"))
+New-Item -ItemType Directory -Path $buildDir | Out-Null
+try {
+    Write-Info "Building Ngino client (self-contained, $runtimeId)..."
+    & $dotnet publish $clientProject -c Release -r $runtimeId --self-contained true -o $buildDir
+    if ($LASTEXITCODE -ne 0) { throw "dotnet publish failed (exit code $LASTEXITCODE)." }
+
+    $executable = Join-Path $buildDir "Ngino.Client.exe"
+    if (-not (Test-Path -LiteralPath $executable)) { throw "Build failed: Ngino.Client.exe was not produced." }
+
+    $existingService = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
+    if ($existingService -and $existingService.Status -ne "Stopped") {
+        Write-Info "Stopping existing service $ServiceName..."
+        Stop-Service -Name $ServiceName -Force
+        (Get-Service -Name $ServiceName).WaitForStatus("Stopped", [TimeSpan]::FromSeconds(30))
+    }
+
+    Write-Info "Installing to $InstallDir..."
+    New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null
+    Copy-Item -Path (Join-Path $buildDir "*") -Destination $InstallDir -Recurse -Force
+} finally {
+    if (Test-Path -LiteralPath $buildDir) { Remove-Item -LiteralPath $buildDir -Recurse -Force }
+}
+
+$installedExecutable = Join-Path $InstallDir "Ngino.Client.exe"
+$binaryPath = '"{0}"' -f $installedExecutable
+if (-not (Get-Service -Name $ServiceName -ErrorAction SilentlyContinue)) {
+    Write-Info "Creating Windows service $ServiceName..."
+    & sc.exe create $ServiceName "binPath=" $binaryPath "start=" "auto" "DisplayName=" "Ngino Tunnel Client"
+    if ($LASTEXITCODE -ne 0) { throw "Could not create Windows service $ServiceName." }
+} else {
+    & sc.exe config $ServiceName "binPath=" $binaryPath "start=" "auto" "DisplayName=" "Ngino Tunnel Client" | Out-Null
+    if ($LASTEXITCODE -ne 0) { throw "Could not update Windows service $ServiceName." }
+}
+
+# A service-specific environment keeps the token out of the process command line.
+$serviceRegistryPath = "HKLM:\SYSTEM\CurrentControlSet\Services\$ServiceName"
+$insecureTlsValue = $InsecureSkipTlsVerify.IsPresent.ToString().ToLowerInvariant()
+$serviceEnvironment = @(
+    "NGINO_SERVER=$Server",
+    "NGINO_TOKEN=$Token",
+    "NGINO_CLIENT_ID=$ClientId",
+    "NGINO_UPSTREAM=$Upstream",
+    "NGINO_INSECURE_SKIP_TLS_VERIFY=$insecureTlsValue",
+    "DOTNET_CLI_TELEMETRY_OPTOUT=1",
+    "DOTNET_NOLOGO=1"
+)
+if ($UseLlamaCppViaDocker) {
+    $serviceEnvironment += "NGINO_USE_LLAMA_CPP_VIA_DOCKER=true"
+    if (-not [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) {
+        $serviceEnvironment += "NGINO_USE_OLLAMA_MODELS_PATH=$UseOllamaModelsPath"
+    }
+    if (-not [string]::IsNullOrWhiteSpace($LlamaCppDockerImage)) {
+        $serviceEnvironment += "NGINO_LLAMA_CPP_DOCKER_IMAGE=$LlamaCppDockerImage"
+    }
+    if ($LlamaCppBasePort -gt 0) {
+        $serviceEnvironment += "NGINO_LLAMA_CPP_BASE_PORT=$LlamaCppBasePort"
+    }
+}
+if ($InsecureSkipTlsVerify) {
+    Write-Warn "Server TLS certificate validation is disabled for $ServiceName."
+}
+New-ItemProperty -Path $serviceRegistryPath -Name Environment -PropertyType MultiString -Value $serviceEnvironment -Force | Out-Null
+& sc.exe description $ServiceName "Ngino outbound tunnel client" | Out-Null
+& sc.exe failure $ServiceName "reset=" "86400" "actions=" "restart/5000/restart/5000/restart/5000" | Out-Null
+
+Start-Service -Name $ServiceName
+$service = Get-Service -Name $ServiceName
+try { $service.WaitForStatus("Running", [TimeSpan]::FromSeconds(15)) } catch { }
+if ($service.Status -eq "Running") { Write-Info "Service $ServiceName is running." }
+else { Write-Warn "Service $ServiceName did not reach Running state. Check: Get-WinEvent -LogName Application" }
+
+Write-Host ""
+Write-Info "Installation complete."
+Write-Host "  Server:      $Server"
+Write-Host "  Client ID:   $ClientId"
+Write-Host "  Upstream:    $Upstream"
+Write-Host "  Service:     $ServiceName"
+Write-Host "  Install dir: $InstallDir"
+if ($UseLlamaCppViaDocker) {
+    Write-Host "  llama.cpp:   enabled (models: $UseOllamaModelsPath)"
+}
+Write-Host ""
+Write-Host "  Manage:  Get-Service $ServiceName | Start-Service/Stop-Service/Restart-Service"
+Write-Host "  Logs:    Get-WinEvent -LogName Application | Where-Object ProviderName -eq NginoClient"

deploy/install-client.sh

@@ -16,6 +16,10 @@ TOKEN=""
 CLIENT_ID="$(hostname -s 2>/dev/null || echo "linux-client")"
 UPSTREAM="$DEFAULT_UPSTREAM"
 SKIP_OLLAMA=false
+USE_LLAMA_CPP_VIA_DOCKER=false
+USE_OLLAMA_MODELS_PATH=""
+LLAMA_CPP_DOCKER_IMAGE=""
+LLAMA_CPP_BASE_PORT=""
 
 # ── Colors ────────────────────────────────────────────────────────────────────
 RED='\033[0;31m'
@@ -45,11 +49,21 @@ Optional:
   --install-dir <dir>  Install directory; defaults to $DEFAULT_INSTALL_DIR
   --service-name <n>   systemd service name; defaults to $DEFAULT_SERVICE_NAME
   --no-ollama          Skip Ollama installation and status check
+  --use-llama-cpp-via-docker
+                       Use llama.cpp via Docker for inference instead of Ollama
+  --use-ollama-models-path <dir>
+                       Path to Ollama models directory (manifests/blobs); required with --use-llama-cpp-via-docker
+  --llama-cpp-docker-image <img>
+                       llama.cpp Docker image; defaults to auto-detected (rocm/cuda/cpu)
+  --llama-cpp-base-port <num>
+                       Base port for llama.cpp containers; defaults to 8081
   -h, --help           Show this help message
 
 Examples:
   $0 --server http://gpu-server:5050 --token "my-secret"
   $0 --server http://gpu-server:5050 --token "my-secret" --no-ollama
+  $0 --server http://gpu-server:5050 --token "my-secret" \\
+     --use-llama-cpp-via-docker --use-ollama-models-path /usr/share/ollama/.ollama/models
 EOF
     exit 0
 }
@@ -57,15 +71,19 @@ EOF
 # ── Argument parsing ──────────────────────────────────────────────────────────
 while [[ $# -gt 0 ]]; do
     case "$1" in
-        --server)       SERVER_URL="$2"; shift 2 ;;
-        --token)        TOKEN="$2"; shift 2 ;;
-        --client-id)    CLIENT_ID="$2"; shift 2 ;;
-        --upstream)     UPSTREAM="$2"; shift 2 ;;
-        --install-dir)  INSTALL_DIR="$2"; shift 2 ;;
-        --service-name) SERVICE_NAME="$2"; shift 2 ;;
-        --no-ollama)    SKIP_OLLAMA=true; shift ;;
-        -h|--help)      usage ;;
-        *)              die "Unknown option: $1" ;;
+        --server)                   SERVER_URL="$2"; shift 2 ;;
+        --token)                    TOKEN="$2"; shift 2 ;;
+        --client-id)                CLIENT_ID="$2"; shift 2 ;;
+        --upstream)                 UPSTREAM="$2"; shift 2 ;;
+        --install-dir)              INSTALL_DIR="$2"; shift 2 ;;
+        --service-name)             SERVICE_NAME="$2"; shift 2 ;;
+        --no-ollama)                SKIP_OLLAMA=true; shift ;;
+        --use-llama-cpp-via-docker) USE_LLAMA_CPP_VIA_DOCKER=true; shift ;;
+        --use-ollama-models-path)   USE_OLLAMA_MODELS_PATH="$2"; shift 2 ;;
+        --llama-cpp-docker-image)   LLAMA_CPP_DOCKER_IMAGE="$2"; shift 2 ;;
+        --llama-cpp-base-port)      LLAMA_CPP_BASE_PORT="$2"; shift 2 ;;
+        -h|--help)                  usage ;;
+        *)                          die "Unknown option: $1" ;;
     esac
 done
 
@@ -85,6 +103,13 @@ if [[ -z "$TOKEN" ]]; then
     die "Token is required."
 fi
 
+if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" && -z "$USE_OLLAMA_MODELS_PATH" ]]; then
+    read -rp "Ollama models path (e.g. /usr/share/ollama/.ollama/models): " USE_OLLAMA_MODELS_PATH
+fi
+if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" && -z "$USE_OLLAMA_MODELS_PATH" ]]; then
+    die "Ollama models path is required with --use-llama-cpp-via-docker."
+fi
+
 # ── Root check ────────────────────────────────────────────────────────────────
 if [[ $EUID -ne 0 ]]; then
     die "This script must be run as root (or with sudo)."
@@ -254,6 +279,14 @@ fi
 
 info "Build successful."
 
+# ── Stop existing service before overwriting binary ─────────────────────────
+SERVICE_FILE="/etc/systemd/system/${SERVICE_NAME}.service"
+
+if systemctl list-unit-files "$SERVICE_NAME.service" &>/dev/null 2>&1 || systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
+    info "Stopping existing service $SERVICE_NAME..."
+    systemctl stop "$SERVICE_NAME" 2>/dev/null || true
+fi
+
 # ── Install ───────────────────────────────────────────────────────────────────
 info "Installing to $INSTALL_DIR..."
 mkdir -p "$INSTALL_DIR"
@@ -265,23 +298,32 @@ info "Client installed to $INSTALL_DIR."
 # ── Write environment file (avoids shell injection in unit file) ─────────────
 ENV_DIR="/etc/ngino-client"
 mkdir -p "$ENV_DIR"
-printf 'NGINO_TOKEN=%s\n' "$TOKEN" > "$ENV_DIR/env"
+{
+    printf 'NGINO_TOKEN=%s\n' "$TOKEN"
+    if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" ]]; then
+        printf 'NGINO_USE_LLAMA_CPP_VIA_DOCKER=true\n'
+        if [[ -n "$USE_OLLAMA_MODELS_PATH" ]]; then
+            printf 'NGINO_USE_OLLAMA_MODELS_PATH=%s\n' "$USE_OLLAMA_MODELS_PATH"
+        fi
+        if [[ -n "$LLAMA_CPP_DOCKER_IMAGE" ]]; then
+            printf 'NGINO_LLAMA_CPP_DOCKER_IMAGE=%s\n' "$LLAMA_CPP_DOCKER_IMAGE"
+        fi
+        if [[ -n "$LLAMA_CPP_BASE_PORT" ]]; then
+            printf 'NGINO_LLAMA_CPP_BASE_PORT=%s\n' "$LLAMA_CPP_BASE_PORT"
+        fi
+    fi
+} > "$ENV_DIR/env"
 chmod 600 "$ENV_DIR/env"
 info "Environment file written to $ENV_DIR/env (mode 0600)."
 
 # ── Create systemd service ───────────────────────────────────────────────────
 SERVICE_FILE="/etc/systemd/system/${SERVICE_NAME}.service"
 
-if systemctl list-unit-files "$SERVICE_NAME.service" &>/dev/null 2>&1; then
-    info "Stopping existing service $SERVICE_NAME..."
-    systemctl stop "$SERVICE_NAME" 2>/dev/null || true
-fi
-
 cat > "$SERVICE_FILE" <<EOF
 [Unit]
 Description=Ngino Tunnel Client
-After=network-online.target
-Wants=network-online.target
+After=network-online.target docker.service
+Wants=network-online.target docker.service
 $([ "$SKIP_OLLAMA" = "false" ] && echo "After=ollama.service")
 $([ "$SKIP_OLLAMA" = "false" ] && echo "Wants=ollama.service")
 
@@ -318,6 +360,9 @@ echo "  Client ID:   $CLIENT_ID"
 echo "  Upstream:    $UPSTREAM"
 echo "  Service:     $SERVICE_NAME"
 echo "  Install dir: $INSTALL_DIR"
+if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" ]]; then
+    echo "  llama.cpp:   enabled (models: $USE_OLLAMA_MODELS_PATH)"
+fi
 echo
 echo "  Manage:  systemctl {start|stop|restart|status} $SERVICE_NAME"
 echo "  Logs:    journalctl -u $SERVICE_NAME -f"

src/Ngino.Client/ClientOptions.cs

@@ -18,6 +18,16 @@ internal sealed class ClientOptions
 
     public int ChunkSize { get; init; } = 64 * 1024;
 
+    public bool InsecureSkipTlsVerify { get; init; }
+
+    public bool UseLlamaCppViaDocker { get; init; }
+
+    public string? UseOllamaModelsPath { get; init; }
+
+    public string? LlamaCppDockerImage { get; init; }
+
+    public int LlamaCppBasePort { get; init; } = 8081;
+
     public Uri TunnelUri
     {
         get
@@ -53,20 +63,30 @@ internal sealed class ClientOptions
             Token = Read(values, "token", "NGINO_TOKEN"),
             ClientId = Read(values, "client-id", "NGINO_CLIENT_ID") ?? Environment.MachineName.ToLowerInvariant(),
             ReconnectDelay = TimeSpan.FromSeconds(ReadInt(values, 5, "reconnect-delay", "NGINO_RECONNECT_DELAY_SECONDS")),
-            ChunkSize = ReadInt(values, 64 * 1024, "chunk-size", "NGINO_CHUNK_SIZE")
+            ChunkSize = ReadInt(values, 64 * 1024, "chunk-size", "NGINO_CHUNK_SIZE"),
+            InsecureSkipTlsVerify = ReadBool(values, false, "insecure-skip-tls-verify", "NGINO_INSECURE_SKIP_TLS_VERIFY"),
+            UseLlamaCppViaDocker = ReadBool(values, false, "use-llama-cpp-via-docker", "NGINO_USE_LLAMA_CPP_VIA_DOCKER"),
+            UseOllamaModelsPath = NormalizeDirectoryPath(Read(values, "use-ollama-models-path", "NGINO_USE_OLLAMA_MODELS_PATH")),
+            LlamaCppDockerImage = Read(values, "llama-cpp-docker-image", "NGINO_LLAMA_CPP_DOCKER_IMAGE"),
+            LlamaCppBasePort = ReadInt(values, 8081, "llama-cpp-base-port", "NGINO_LLAMA_CPP_BASE_PORT")
         };
     }
 
     public static string Usage =>
         """
         Ngino.Client options:
-          --server <url>             Server base URL, e.g. http://my-server:5050
-          --upstream <url>           Local upstream URL, e.g. http://localhost:11434
-          --token <value>            Optional token matching the server
-          --client-id <name>         Identifies this machine on the server; defaults to the machine name
-          --tunnel-path <path>       Defaults to /_ngino/tunnel
-          --reconnect-delay <sec>    Defaults to 5
-          --chunk-size <bytes>       Defaults to 65536
+          --server <url>                 Server base URL, e.g. http://my-server:5050
+          --upstream <url>               Local upstream URL, e.g. http://localhost:11434
+          --token <value>                Optional token matching the server
+          --client-id <name>             Identifies this machine on the server; defaults to the machine name
+          --tunnel-path <path>           Defaults to /_ngino/tunnel
+          --reconnect-delay <sec>        Defaults to 5
+          --chunk-size <bytes>           Defaults to 65536
+          --insecure-skip-tls-verify     Disable server TLS certificate validation (unsafe)
+          --use-llama-cpp-via-docker     Use llama.cpp via Docker for inference instead of Ollama
+          --use-ollama-models-path <dir> Path to Ollama models directory (manifests/blobs), required with --use-llama-cpp-via-docker
+          --llama-cpp-docker-image <img> llama.cpp Docker image; defaults to auto-detected (rocm/cuda/cpu)
+          --llama-cpp-base-port <num>    Base port for llama.cpp containers; defaults to 8081
         """;
 
     private static Dictionary<string, string> ParseArgs(string[] args)
@@ -88,6 +108,12 @@ internal sealed class ClientOptions
                 continue;
             }
 
+            if (IsBoolFlag(keyValue[0]))
+            {
+                values[keyValue[0]] = "true";
+                continue;
+            }
+
             if (i + 1 >= args.Length || args[i + 1].StartsWith("--", StringComparison.Ordinal))
             {
                 throw new ArgumentException($"Missing value for '{arg}'.");
@@ -99,6 +125,16 @@ internal sealed class ClientOptions
         return values;
     }
 
+    private static bool IsBoolFlag(string key)
+    {
+        return key switch
+        {
+            "insecure-skip-tls-verify" => true,
+            "use-llama-cpp-via-docker" => true,
+            _ => false
+        };
+    }
+
     private static string? Read(Dictionary<string, string> values, params string[] keys)
     {
         foreach (var key in keys)
@@ -124,6 +160,12 @@ internal sealed class ClientOptions
         return int.TryParse(value, out var parsed) && parsed > 0 ? parsed : fallback;
     }
 
+    private static bool ReadBool(Dictionary<string, string> values, bool fallback, params string[] keys)
+    {
+        var value = Read(values, keys);
+        return bool.TryParse(value, out var parsed) ? parsed : fallback;
+    }
+
     private static Uri ReadUri(Dictionary<string, string> values, string key, string envKey, string fallback)
     {
         var value = Read(values, key, envKey) ?? fallback;
@@ -138,4 +180,14 @@ internal sealed class ClientOptions
 
     private static string NormalizePath(string path) =>
         path.StartsWith('/') ? path : $"/{path}";
+
+    private static string? NormalizeDirectoryPath(string? path)
+    {
+        if (string.IsNullOrWhiteSpace(path))
+        {
+            return null;
+        }
+
+        return Path.GetFullPath(path);
+    }
 }

src/Ngino.Client/LlamaCppManager.cs

@@ -0,0 +1,531 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Ngino.Client;
+
+internal sealed partial class LlamaCppManager : IAsyncDisposable
+{
+    private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
+    private const int DefaultBasePort = 8081;
+    private const string NginoContainerLabel = "ngino-llamacpp";
+    private static readonly TimeSpan DockerTimeout = TimeSpan.FromSeconds(60);
+
+    private readonly string _blobsPath;
+    private readonly string _manifestsPath;
+    private readonly string _dockerImage;
+    private readonly int _basePort;
+    private readonly ILogger _logger;
+    private readonly ConcurrentDictionary<string, int> _modelPorts = new(StringComparer.OrdinalIgnoreCase);
+
+    public LlamaCppManager(
+        string ollamaModelsPath,
+        string? dockerImage,
+        int? basePort,
+        ILogger? logger = null)
+    {
+        _manifestsPath = Path.Combine(ollamaModelsPath, "manifests");
+        _blobsPath = Path.Combine(ollamaModelsPath, "blobs");
+        _dockerImage = dockerImage ?? GetDefaultDockerImage();
+        _basePort = basePort ?? DefaultBasePort;
+        _logger = logger ?? NullLogger<LlamaCppManager>.Instance;
+    }
+
+    public string DockerImage => _dockerImage;
+
+    public List<LlamaCppModel> DiscoverModels()
+    {
+        var models = new List<LlamaCppModel>();
+
+        if (!Directory.Exists(_manifestsPath))
+        {
+            _logger.LogWarning("Ollama manifests path not found: {ManifestsPath}", _manifestsPath);
+            return models;
+        }
+
+        foreach (var manifestPath in Directory.EnumerateFiles(_manifestsPath, "*", SearchOption.AllDirectories))
+        {
+            try
+            {
+                var model = ParseManifest(manifestPath);
+                if (model is not null)
+                {
+                    models.Add(model);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogDebug(ex, "Failed to parse manifest: {ManifestPath}", manifestPath);
+            }
+        }
+
+        return models;
+    }
+
+    public List<LlamaCppModel> DiscoverModelsWithBlob()
+    {
+        return DiscoverModels().Where(m => File.Exists(m.BlobPath)).ToList();
+    }
+
+    public bool IsModelActive(string ollamaModelName)
+    {
+        return _modelPorts.ContainsKey(ollamaModelName);
+    }
+
+    public Uri? GetUpstream(string ollamaModelName)
+    {
+        if (_modelPorts.TryGetValue(ollamaModelName, out var port))
+        {
+            return new Uri($"http://localhost:{port}");
+        }
+
+        return null;
+    }
+
+    public async Task<bool> StartModelContainerAsync(LlamaCppModel model, CancellationToken cancellationToken)
+    {
+        var ollamaName = model.OllamaName;
+        if (string.IsNullOrWhiteSpace(ollamaName))
+        {
+            _logger.LogWarning("Cannot start container: model has no Ollama name");
+            return false;
+        }
+
+        if (_modelPorts.ContainsKey(ollamaName))
+        {
+            _logger.LogInformation("Model {Model} already has a running container", ollamaName);
+            return true;
+        }
+
+        if (!File.Exists(model.BlobPath))
+        {
+            _logger.LogError("Model blob not found: {BlobPath}", model.BlobPath);
+            return false;
+        }
+
+        var port = FindAvailablePort();
+        var containerName = SanitizeContainerName($"ngino-llamacpp-{ollamaName}");
+
+        var existingPort = await FindExistingContainerPortAsync(containerName);
+        if (existingPort.HasValue)
+        {
+            _logger.LogInformation(
+                "Reusing existing container for {Model} on port {Port}", ollamaName, existingPort.Value);
+            _modelPorts[ollamaName] = existingPort.Value;
+            return true;
+        }
+
+        await RunDockerAsync(["rm", "-f", containerName], CancellationToken.None);
+
+        var args = BuildDockerRunArgs(containerName, model, port);
+        _logger.LogInformation(
+            "Starting llama.cpp container for {Model} on port {Port}: docker {Args}",
+            ollamaName, port, string.Join(" ", args));
+
+        try
+        {
+            var (exitCode, output) = await RunDockerWithOutputAsync(args, cancellationToken);
+            if (exitCode != 0)
+            {
+                _logger.LogError(
+                    "Failed to start llama.cpp container for {Model}, exit code: {ExitCode}, output: {Output}",
+                    ollamaName, exitCode, output);
+                return false;
+            }
+
+            _modelPorts[ollamaName] = port;
+            _logger.LogInformation("llama.cpp container for {Model} started on port {Port}", ollamaName, port);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to start llama.cpp container for {Model}", ollamaName);
+            return false;
+        }
+    }
+
+    public async Task<bool> StopModelContainerAsync(string ollamaModelName, CancellationToken cancellationToken)
+    {
+        if (!_modelPorts.TryRemove(ollamaModelName, out _))
+        {
+            _logger.LogWarning("No running container found for model {Model}", ollamaModelName);
+            return false;
+        }
+
+        var containerName = SanitizeContainerName($"ngino-llamacpp-{ollamaModelName}");
+
+        _logger.LogInformation("Stopping llama.cpp container {ContainerName}", containerName);
+
+        try
+        {
+            await RunDockerAsync(["stop", "--time", "10", containerName], cancellationToken);
+            await RunDockerAsync(["rm", "-f", containerName], cancellationToken);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to stop container {ContainerName}", containerName);
+            return false;
+        }
+    }
+
+    public async Task StopAllContainersAsync()
+    {
+        _logger.LogInformation("Stopping all llama.cpp containers...");
+
+        try
+        {
+            var (exitCode, output) = await RunDockerWithOutputAsync(
+                ["ps", "-q", "--filter", $"label={NginoContainerLabel}"],
+                CancellationToken.None);
+
+            if (exitCode == 0 && !string.IsNullOrWhiteSpace(output))
+            {
+                var containerIds = output.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+                foreach (var id in containerIds)
+                {
+                    await RunDockerAsync(["stop", "--time", "10", id], CancellationToken.None);
+                    await RunDockerAsync(["rm", "-f", id], CancellationToken.None);
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to stop all llama.cpp containers");
+        }
+
+        _modelPorts.Clear();
+    }
+
+    public async Task<bool> TestDockerAsync()
+    {
+        try
+        {
+            var (exitCode, _) = await RunDockerWithOutputAsync(["info", "--format", "{{.ServerVersion}}"], CancellationToken.None);
+            return exitCode == 0;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        await StopAllContainersAsync();
+    }
+
+    private LlamaCppModel? ParseManifest(string manifestPath)
+    {
+        var json = File.ReadAllText(manifestPath);
+        using var document = JsonDocument.Parse(json);
+        var root = document.RootElement;
+
+        if (!root.TryGetProperty("layers", out var layers) || layers.ValueKind != JsonValueKind.Array)
+        {
+            return null;
+        }
+
+        string? modelDigest = null;
+        foreach (var layer in layers.EnumerateArray())
+        {
+            if (layer.TryGetProperty("mediaType", out var mediaType)
+                && mediaType.GetString() == "application/vnd.ollama.image.model"
+                && layer.TryGetProperty("digest", out var digest))
+            {
+                modelDigest = digest.GetString();
+                break;
+            }
+        }
+
+        if (string.IsNullOrWhiteSpace(modelDigest))
+        {
+            return null;
+        }
+
+        var modelName = ResolveModelName(manifestPath);
+        if (modelName is null)
+        {
+            return null;
+        }
+
+        var blobName = modelDigest.Replace(":", "-", StringComparison.Ordinal);
+        var blobPath = Path.GetFullPath(Path.Combine(_blobsPath, blobName));
+
+        return new LlamaCppModel
+        {
+            OllamaName = modelName,
+            BlobDigest = blobName,
+            BlobPath = blobPath,
+            ManifestPath = manifestPath
+        };
+    }
+
+    private static string? ResolveModelName(string manifestPath)
+    {
+        var normalizedPath = manifestPath.Replace('\\', '/');
+        var parts = normalizedPath.Split('/', StringSplitOptions.RemoveEmptyEntries);
+
+        var manifestIndex = Array.FindLastIndex(parts, p =>
+            p.Equals("manifests", StringComparison.OrdinalIgnoreCase));
+
+        if (manifestIndex < 0 || manifestIndex >= parts.Length - 1)
+        {
+            return null;
+        }
+
+        var relativeParts = parts[(manifestIndex + 1)..];
+
+        if (relativeParts.Length < 2)
+        {
+            return null;
+        }
+
+        var registry = relativeParts[0];
+
+        if (registry.Equals("registry.ollama.ai", StringComparison.OrdinalIgnoreCase))
+        {
+            if (relativeParts.Length < 3)
+            {
+                return null;
+            }
+
+            if (relativeParts[1].Equals("library", StringComparison.OrdinalIgnoreCase) && relativeParts.Length >= 4)
+            {
+                return $"{relativeParts[2]}:{relativeParts[3]}";
+            }
+
+            if (relativeParts.Length == 3)
+            {
+                return $"{relativeParts[1]}:{relativeParts[2]}";
+            }
+
+            return null;
+        }
+
+        if (relativeParts.Length >= 3)
+        {
+            var tag = relativeParts[^1];
+            var modelPath = string.Join("/", relativeParts.Take(relativeParts.Length - 1));
+            return $"{modelPath}:{tag}";
+        }
+
+        return null;
+    }
+
+    private string[] BuildDockerRunArgs(string containerName, LlamaCppModel model, int port)
+    {
+        var blobsDir = Path.GetDirectoryName(Path.GetFullPath(model.BlobPath))!;
+        var blobFile = Path.GetFileName(model.BlobPath);
+
+        var args = new List<string>
+        {
+            "run",
+            "-d",
+            "--rm",
+            "--label", $"{NginoContainerLabel}=true",
+            "--name", containerName,
+            "-p", $"{port}:{port}",
+            "-v", $"{blobsDir}:/models/blobs:ro",
+        };
+
+        if (HasRocmDevices())
+        {
+            args.Add("--device=/dev/kfd");
+            args.Add("--device=/dev/dri");
+            args.Add("--group-add=video");
+        }
+
+        if (HasNvidiaGpu() && !HasRocmDevices())
+        {
+            args.Add("--gpus=all");
+        }
+
+        args.Add(_dockerImage);
+        args.Add("--embeddings");
+        args.Add("-m");
+        args.Add($"/models/blobs/{blobFile}");
+        args.Add("-ngl");
+        args.Add("999");
+        args.Add("--parallel");
+        args.Add("4");
+        args.Add("--host");
+        args.Add("0.0.0.0");
+        args.Add("--port");
+        args.Add(port.ToString());
+
+        return [.. args];
+    }
+
+    private async Task<int?> FindExistingContainerPortAsync(string containerName)
+    {
+        var (exitCode, output) = await RunDockerWithOutputAsync(
+            ["ps", "--filter", $"name=^{containerName}$", "--format", "{{.Ports}}"],
+            CancellationToken.None);
+
+        if (exitCode != 0 || string.IsNullOrWhiteSpace(output))
+        {
+            return null;
+        }
+
+        foreach (var line in output.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
+        {
+            var port = ParseHostPort(line);
+            if (port.HasValue)
+            {
+                return port;
+            }
+        }
+
+        return null;
+    }
+
+    private static int? ParseHostPort(string ports)
+    {
+        foreach (var mapping in ports.Split(','))
+        {
+            var trimmed = mapping.Trim();
+            var arrowIndex = trimmed.IndexOf("->", StringComparison.Ordinal);
+            if (arrowIndex < 0)
+            {
+                continue;
+            }
+
+            var hostPart = trimmed[..arrowIndex].Trim();
+            var colonIndex = hostPart.LastIndexOf(':');
+            if (colonIndex < 0)
+            {
+                continue;
+            }
+
+            if (int.TryParse(hostPart[(colonIndex + 1)..], out var port))
+            {
+                return port;
+            }
+        }
+
+        return null;
+    }
+
+    private int FindAvailablePort()
+    {
+        var usedPorts = new HashSet<int>(_modelPorts.Values);
+        var port = _basePort;
+
+        while (usedPorts.Contains(port))
+        {
+            port++;
+        }
+
+        return port;
+    }
+
+    private static string SanitizeContainerName(string name)
+    {
+        var sanitized = InvalidContainerNameChars().Replace(name, "_");
+        return sanitized.Trim('_').ToLowerInvariant();
+    }
+
+    private async Task<int> RunDockerAsync(string[] args, CancellationToken cancellationToken)
+    {
+        var (exitCode, _) = await RunDockerWithOutputAsync(args, cancellationToken);
+        return exitCode;
+    }
+
+    private async Task<(int ExitCode, string Output)> RunDockerWithOutputAsync(
+        string[] args, CancellationToken cancellationToken)
+    {
+        var process = new Process
+        {
+            StartInfo = new ProcessStartInfo
+            {
+                FileName = "docker",
+                Arguments = string.Join(" ", args.Select(a => a.Contains(' ') ? $"\"{a}\"" : a)),
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            }
+        };
+
+        process.Start();
+
+        var readOutput = process.StandardOutput.ReadToEndAsync(cancellationToken);
+        var readError = process.StandardError.ReadToEndAsync(cancellationToken);
+        var waitTask = process.WaitForExitAsync(cancellationToken);
+
+        var completed = await Task.WhenAny(waitTask, Task.Delay(DockerTimeout, cancellationToken));
+
+        string output;
+        string error;
+
+        try
+        {
+            output = await readOutput;
+            error = await readError;
+        }
+        catch
+        {
+            output = "";
+            error = "timed out";
+        }
+
+        if (completed != waitTask)
+        {
+            _logger.LogWarning("Docker command timed out: docker {Args}", string.Join(" ", args));
+            try { process.Kill(entireProcessTree: true); } catch { }
+            return (-1, error);
+        }
+
+        if (!string.IsNullOrWhiteSpace(error))
+        {
+            output = $"{output}\n{error}".Trim();
+        }
+
+        return (process.ExitCode, output);
+    }
+
+    private static string GetDefaultDockerImage()
+    {
+        if (HasRocmDevices())
+        {
+            return "ghcr.io/ggml-org/llama.cpp:server-rocm";
+        }
+
+        if (HasNvidiaGpu())
+        {
+            return "ghcr.io/ggml-org/llama.cpp:server-cuda";
+        }
+
+        return "ghcr.io/ggml-org/llama.cpp:server";
+    }
+
+    private static bool HasRocmDevices() => File.Exists("/dev/kfd") && Directory.Exists("/dev/dri");
+
+    private static bool HasNvidiaGpu()
+    {
+        try
+        {
+            return File.Exists("/proc/driver/nvidia/version")
+                || Directory.Exists("/proc/driver/nvidia/gpus");
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
+    [GeneratedRegex(@"[^a-zA-Z0-9_.-]")]
+    private static partial Regex InvalidContainerNameChars();
+}
+
+internal sealed record LlamaCppModel
+{
+    public required string OllamaName { get; init; }
+    public required string BlobDigest { get; init; }
+    public required string BlobPath { get; init; }
+    public required string ManifestPath { get; init; }
+}

src/Ngino.Client/OllamaToLlamaCppTranslator.cs

@@ -0,0 +1,605 @@
+using System.Text;
+using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using Ngino.Protocol;
+
+namespace Ngino.Client;
+
+internal sealed class OllamaToLlamaCppTranslator
+{
+    private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
+
+    private readonly string _modelName;
+    private readonly ILogger _logger;
+
+    public OllamaToLlamaCppTranslator(string modelName, ILogger logger)
+    {
+        _modelName = modelName;
+        _logger = logger;
+    }
+
+    public bool TryTranslatePath(string method, string pathAndQuery, out string newPath)
+    {
+        var path = pathAndQuery.Split('?')[0];
+        newPath = path switch
+        {
+            "/api/generate" => "/completion",
+            "/api/chat" => "/v1/chat/completions",
+            "/api/embed" or "/api/embeddings" => "/v1/embeddings",
+            _ => null!
+        };
+        return newPath is not null;
+    }
+
+    public byte[] TranslateBody(string pathAndQuery, byte[] body)
+    {
+        if (body is null || body.Length == 0)
+            return body;
+
+        var path = pathAndQuery.Split('?')[0];
+
+        try
+        {
+            return path switch
+            {
+                "/api/generate" => TranslateGenerateBody(body),
+                "/api/chat" => TranslateChatBody(body),
+                "/api/embed" or "/api/embeddings" => TranslateEmbedBody(body),
+                _ => body
+            };
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to translate request body for {Path}", path);
+            return body;
+        }
+    }
+
+    private byte[] TranslateGenerateBody(byte[] body)
+    {
+        using var doc = JsonDocument.Parse(body);
+        var root = doc.RootElement;
+
+        var result = new Dictionary<string, object?>();
+
+        if (root.TryGetProperty("prompt", out var prompt))
+            result["prompt"] = prompt.GetString() ?? "";
+
+        result["stream"] = true;
+
+        CopyOptions(root, result);
+
+        if (!result.ContainsKey("n_predict"))
+            result["n_predict"] = 2048;
+
+        return JsonSerializer.SerializeToUtf8Bytes(result, JsonOptions);
+    }
+
+    private byte[] TranslateChatBody(byte[] body)
+    {
+        using var doc = JsonDocument.Parse(body);
+        var root = doc.RootElement;
+
+        var result = new Dictionary<string, object?>();
+
+        if (root.TryGetProperty("messages", out var messages))
+            result["messages"] = messages.Deserialize<object>(JsonOptions);
+
+        result["stream"] = true;
+
+        if (root.TryGetProperty("model", out var model))
+            result["model"] = model.GetString();
+
+        CopyOptions(root, result, chat: true);
+
+        if (!result.ContainsKey("max_tokens"))
+            result["max_tokens"] = 2048;
+
+        return JsonSerializer.SerializeToUtf8Bytes(result, JsonOptions);
+    }
+
+    public static bool ExtractOriginalStream(byte[] originalBody, string pathAndQuery)
+    {
+        var path = pathAndQuery.Split('?')[0];
+        if (path is not "/api/generate" and not "/api/chat")
+            return true;
+
+        try
+        {
+            using var doc = JsonDocument.Parse(originalBody);
+            var root = doc.RootElement;
+            if (root.TryGetProperty("stream", out var stream))
+                return stream.ValueKind != JsonValueKind.False;
+        }
+        catch
+        {
+        }
+
+        return true;
+    }
+
+    private byte[] TranslateEmbedBody(byte[] body)
+    {
+        using var doc = JsonDocument.Parse(body);
+        var root = doc.RootElement;
+
+        var result = new Dictionary<string, object?>
+        {
+            ["model"] = _modelName
+        };
+
+        if (root.TryGetProperty("input", out var input))
+            result["input"] = input.Deserialize<object>(JsonOptions);
+
+        return JsonSerializer.SerializeToUtf8Bytes(result, JsonOptions);
+    }
+
+    public Func<HttpResponseMessage, CancellationToken, Task> CreateResponseHandler(
+        Func<TunnelMessage, CancellationToken, Task> sendAsync,
+        string requestId,
+        string originalPath,
+        Func<bool> originalRequestedStream) =>
+        async (response, ct) =>
+        {
+            await TranslateAndSendResponse(response, sendAsync, requestId, originalPath, originalRequestedStream(), ct);
+        };
+
+    private async Task TranslateAndSendResponse(
+        HttpResponseMessage httpResponse,
+        Func<TunnelMessage, CancellationToken, Task> sendAsync,
+        string requestId,
+        string originalPath,
+        bool originalRequestedStream,
+        CancellationToken cancellationToken)
+    {
+        var path = originalPath.Split('?')[0];
+
+        switch (path)
+        {
+            case "/api/tags":
+                await SynthesizeTagsResponse(sendAsync, requestId, cancellationToken);
+                return;
+
+            case "/api/ps":
+                await SynthesizePsResponse(sendAsync, requestId, cancellationToken);
+                return;
+        }
+
+        if (!httpResponse.IsSuccessStatusCode)
+        {
+            await ForwardRawResponse(httpResponse, sendAsync, requestId, cancellationToken);
+            return;
+        }
+
+        if (path is "/api/generate" or "/api/chat")
+        {
+            if (originalRequestedStream)
+            {
+                await TranslateStreaming(path, httpResponse, sendAsync, requestId, cancellationToken);
+            }
+            else
+            {
+                await TranslateNonStreaming(path, httpResponse, sendAsync, requestId, cancellationToken);
+            }
+        }
+        else
+        {
+            await ForwardRawResponse(httpResponse, sendAsync, requestId, cancellationToken);
+        }
+    }
+
+    private async Task ForwardRawResponse(
+        HttpResponseMessage httpResponse,
+        Func<TunnelMessage, CancellationToken, Task> sendAsync,
+        string requestId,
+        CancellationToken cancellationToken)
+    {
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseHeaders,
+            RequestId = requestId,
+            StatusCode = (int)httpResponse.StatusCode,
+            ReasonPhrase = httpResponse.ReasonPhrase
+        }, cancellationToken);
+
+        await using var stream = await httpResponse.Content.ReadAsStreamAsync(cancellationToken);
+        var buffer = new byte[64 * 1024];
+        while (true)
+        {
+            var bytesRead = await stream.ReadAsync(buffer, cancellationToken);
+            if (bytesRead == 0)
+                break;
+            await sendAsync(new TunnelMessage
+            {
+                Type = TunnelMessageTypes.HttpResponseBody,
+                RequestId = requestId,
+                Body = buffer.AsSpan(0, bytesRead).ToArray()
+            }, cancellationToken);
+        }
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseComplete,
+            RequestId = requestId
+        }, cancellationToken);
+    }
+
+    private async Task SynthesizeTagsResponse(
+        Func<TunnelMessage, CancellationToken, Task> sendAsync,
+        string requestId,
+        CancellationToken cancellationToken)
+    {
+        var modelList = new
+        {
+            models = new[]
+            {
+                new
+                {
+                    name = _modelName,
+                    model = _modelName,
+                    modified_at = DateTime.UtcNow.ToString("o"),
+                    size = 0L,
+                    digest = "sha256:" + _modelName,
+                    details = new
+                    {
+                        format = "gguf",
+                        family = "llama",
+                        parameter_size = "",
+                        quantization_level = ""
+                    }
+                }
+            }
+        };
+
+        var body = JsonSerializer.SerializeToUtf8Bytes(modelList, JsonOptions);
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseHeaders,
+            RequestId = requestId,
+            StatusCode = 200,
+            ReasonPhrase = "OK"
+        }, cancellationToken);
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseBody,
+            RequestId = requestId,
+            Body = body
+        }, cancellationToken);
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseComplete,
+            RequestId = requestId
+        }, cancellationToken);
+    }
+
+    private async Task SynthesizePsResponse(
+        Func<TunnelMessage, CancellationToken, Task> sendAsync,
+        string requestId,
+        CancellationToken cancellationToken)
+    {
+        var body = JsonSerializer.SerializeToUtf8Bytes(new { models = Array.Empty<object>() }, JsonOptions);
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseHeaders,
+            RequestId = requestId,
+            StatusCode = 200,
+            ReasonPhrase = "OK"
+        }, cancellationToken);
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseBody,
+            RequestId = requestId,
+            Body = body
+        }, cancellationToken);
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseComplete,
+            RequestId = requestId
+        }, cancellationToken);
+    }
+
+    private async Task TranslateNonStreaming(
+        string path,
+        HttpResponseMessage httpResponse,
+        Func<TunnelMessage, CancellationToken, Task> sendAsync,
+        string requestId,
+        CancellationToken cancellationToken)
+    {
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseHeaders,
+            RequestId = requestId,
+            StatusCode = (int)httpResponse.StatusCode,
+            ReasonPhrase = httpResponse.ReasonPhrase
+        }, cancellationToken);
+
+        var body = await httpResponse.Content.ReadAsByteArrayAsync(cancellationToken);
+        byte[] translatedBody;
+
+        try
+        {
+            translatedBody = path switch
+            {
+                "/api/generate" => TranslateNonStreamingGenerate(body),
+                "/api/chat" => TranslateNonStreamingChat(body),
+                _ => body
+            };
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to translate non-streaming response");
+            translatedBody = body;
+        }
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseBody,
+            RequestId = requestId,
+            Body = translatedBody
+        }, cancellationToken);
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseComplete,
+            RequestId = requestId
+        }, cancellationToken);
+    }
+
+    private async Task TranslateStreaming(
+        string path,
+        HttpResponseMessage httpResponse,
+        Func<TunnelMessage, CancellationToken, Task> sendAsync,
+        string requestId,
+        CancellationToken cancellationToken)
+    {
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseHeaders,
+            RequestId = requestId,
+            StatusCode = 200,
+            ReasonPhrase = "OK"
+        }, cancellationToken);
+
+        await using var stream = await httpResponse.Content.ReadAsStreamAsync(cancellationToken);
+        using var reader = new StreamReader(stream, Encoding.UTF8);
+
+        while (!cancellationToken.IsCancellationRequested)
+        {
+            var line = await reader.ReadLineAsync(cancellationToken);
+            if (line is null)
+                break;
+
+            if (!line.StartsWith("data: ", StringComparison.Ordinal))
+                continue;
+
+            var jsonStr = line[6..];
+            if (jsonStr == "[DONE]")
+                continue;
+
+            try
+            {
+                using var doc = JsonDocument.Parse(jsonStr);
+                var root = doc.RootElement;
+
+                byte[]? chunk = path switch
+                {
+                    "/api/generate" => TranslateGenerateStreamChunk(root),
+                    "/api/chat" => TranslateChatStreamChunk(root),
+                    _ => null
+                };
+
+                if (chunk is not null)
+                {
+                    await sendAsync(new TunnelMessage
+                    {
+                        Type = TunnelMessageTypes.HttpResponseBody,
+                        RequestId = requestId,
+                        Body = chunk
+                    }, cancellationToken);
+                }
+            }
+            catch (JsonException)
+            {
+            }
+        }
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseBody,
+            RequestId = requestId,
+            Body = Encoding.UTF8.GetBytes(
+                $"{{\"model\":\"{EscapeJson(_modelName)}\",\"response\":\"\",\"done\":true}}\n")
+        }, cancellationToken);
+
+        await sendAsync(new TunnelMessage
+        {
+            Type = TunnelMessageTypes.HttpResponseComplete,
+            RequestId = requestId
+        }, cancellationToken);
+    }
+
+    private byte[] TranslateNonStreamingGenerate(byte[] body)
+    {
+        using var doc = JsonDocument.Parse(body);
+        var root = doc.RootElement;
+
+        var result = new Dictionary<string, object?>
+        {
+            ["model"] = _modelName,
+            ["created_at"] = DateTime.UtcNow.ToString("o"),
+            ["response"] = root.TryGetProperty("content", out var content) ? content.GetString() : "",
+            ["done"] = root.TryGetProperty("stop", out var stop) && stop.GetBoolean()
+        };
+
+        CopyTimings(root, result);
+        return JsonSerializer.SerializeToUtf8Bytes(result, JsonOptions);
+    }
+
+    private byte[] TranslateNonStreamingChat(byte[] body)
+    {
+        using var doc = JsonDocument.Parse(body);
+        var root = doc.RootElement;
+
+        var message = root.GetProperty("choices")[0].GetProperty("message");
+        var content = message.GetProperty("content").GetString() ?? "";
+
+        var result = new Dictionary<string, object?>
+        {
+            ["model"] = _modelName,
+            ["created_at"] = DateTime.UtcNow.ToString("o"),
+            ["message"] = new Dictionary<string, object?>
+            {
+                ["role"] = "assistant",
+                ["content"] = content
+            },
+            ["done"] = true
+        };
+
+        if (root.TryGetProperty("usage", out var usage))
+        {
+            if (usage.TryGetProperty("completion_tokens", out var comp))
+                result["eval_count"] = comp.GetInt32();
+            if (usage.TryGetProperty("prompt_tokens", out var prompt))
+                result["prompt_eval_count"] = prompt.GetInt32();
+        }
+
+        CopyTimings(root, result);
+        return JsonSerializer.SerializeToUtf8Bytes(result, JsonOptions);
+    }
+
+    private byte[]? TranslateGenerateStreamChunk(JsonElement root)
+    {
+        var done = root.TryGetProperty("stop", out var stop) && stop.GetBoolean();
+        var text = root.TryGetProperty("content", out var content) ? content.GetString() ?? "" : "";
+
+        var result = new Dictionary<string, object?>
+        {
+            ["model"] = _modelName,
+            ["created_at"] = DateTime.UtcNow.ToString("o"),
+            ["response"] = text,
+            ["done"] = done
+        };
+
+        if (done)
+            CopyTimings(root, result);
+
+        var bytes = JsonSerializer.SerializeToUtf8Bytes(result, JsonOptions);
+        var withNewline = new byte[bytes.Length + 1];
+        bytes.CopyTo(withNewline, 0);
+        withNewline[^1] = (byte)'\n';
+        return withNewline;
+    }
+
+    private byte[]? TranslateChatStreamChunk(JsonElement root)
+    {
+        if (!root.TryGetProperty("choices", out var choices) || choices.GetArrayLength() == 0)
+            return null;
+
+        var choice = choices[0];
+        var delta = choice.GetProperty("delta");
+        var finishReason = choice.TryGetProperty("finish_reason", out var fr) ? fr.GetString() : null;
+        var done = finishReason is not null && finishReason != "null" && finishReason != "";
+
+        var content = delta.TryGetProperty("content", out var c) ? c.GetString() ?? "" : "";
+        var role = delta.TryGetProperty("role", out var r) ? r.GetString() : null;
+
+        var result = new Dictionary<string, object?>
+        {
+            ["model"] = _modelName,
+            ["created_at"] = DateTime.UtcNow.ToString("o"),
+            ["message"] = new Dictionary<string, object?>
+            {
+                ["role"] = role ?? "assistant",
+                ["content"] = content
+            },
+            ["done"] = done
+        };
+
+        var bytes = JsonSerializer.SerializeToUtf8Bytes(result, JsonOptions);
+        var withNewline = new byte[bytes.Length + 1];
+        bytes.CopyTo(withNewline, 0);
+        withNewline[^1] = (byte)'\n';
+        return withNewline;
+    }
+
+    private static void CopyOptions(JsonElement root, Dictionary<string, object?> target, bool chat = false)
+    {
+        if (!root.TryGetProperty("options", out var options) || options.ValueKind != JsonValueKind.Object)
+            return;
+
+        foreach (var opt in options.EnumerateObject())
+        {
+            var key = (chat ? MapChatOptionName(opt.Name) : MapGenerateOptionName(opt.Name)) ?? opt.Name;
+            target[key] = ValueToObject(opt.Value);
+        }
+    }
+
+    private static string? MapGenerateOptionName(string name) => name switch
+    {
+        "num_predict" => "n_predict",
+        "temperature" => "temperature",
+        "top_p" => "top_p",
+        "top_k" => "top_k",
+        "seed" => "seed",
+        "stop" => "stop",
+        "repeat_penalty" => "repeat_penalty",
+        "repeat_last_n" => "repeat_last_n",
+        "frequency_penalty" => "frequency_penalty",
+        "presence_penalty" => "presence_penalty",
+        "mirostat" => "mirostat",
+        "mirostat_tau" => "mirostat_tau",
+        "mirostat_eta" => "mirostat_eta",
+        "num_ctx" => "n_ctx",
+        "num_batch" => "n_batch",
+        _ => null
+    };
+
+    private static string? MapChatOptionName(string name) => name switch
+    {
+        "num_predict" => "max_tokens",
+        "temperature" => "temperature",
+        "top_p" => "top_p",
+        "seed" => "seed",
+        "stop" => "stop",
+        "frequency_penalty" => "frequency_penalty",
+        "presence_penalty" => "presence_penalty",
+        _ => null
+    };
+
+    private static void CopyTimings(JsonElement root, Dictionary<string, object?> target)
+    {
+        if (!root.TryGetProperty("timings", out var timings))
+            return;
+
+        if (timings.TryGetProperty("predicted_n", out var predN))
+            target["eval_count"] = predN.GetInt32();
+        if (timings.TryGetProperty("predicted_ms", out var predMs))
+            target["eval_duration"] = (long)(predMs.GetDouble() * 1_000_000);
+        if (timings.TryGetProperty("prompt_n", out var promptN))
+            target["prompt_eval_count"] = promptN.GetInt32();
+        if (timings.TryGetProperty("prompt_ms", out var promptMs))
+            target["prompt_eval_duration"] = (long)(promptMs.GetDouble() * 1_000_000);
+    }
+
+    private static object? ValueToObject(JsonElement element)
+    {
+        return element.ValueKind switch
+        {
+            JsonValueKind.String => element.GetString(),
+            JsonValueKind.Number => element.TryGetInt64(out var l) ? (object)l : element.GetDouble(),
+            JsonValueKind.True => true,
+            JsonValueKind.False => false,
+            JsonValueKind.Null => null,
+            _ => element.Deserialize<object>(JsonOptions)
+        };
+    }
+
+    private static string EscapeJson(string value) =>
+        value.Replace("\\", "\\\\").Replace("\"", "\\\"");
+}

src/Ngino.Client/Program.cs

@@ -11,6 +11,18 @@ try
     Console.WriteLine($"  client id: {options.ClientId}");
     Console.WriteLine($"  server tunnel: {options.TunnelUri}");
     Console.WriteLine($"  local upstream: {options.Upstream}");
+    if (options.InsecureSkipTlsVerify)
+    {
+        Console.WriteLine("  WARNING: server TLS certificate validation is disabled");
+    }
+
+    if (options.UseLlamaCppViaDocker)
+    {
+        Console.WriteLine($"  llama.cpp via Docker: enabled");
+        Console.WriteLine($"  ollama models path: {options.UseOllamaModelsPath ?? "(not set)"}");
+        Console.WriteLine($"  llama.cpp docker image: {options.LlamaCppDockerImage ?? "(auto)"}");
+        Console.WriteLine($"  llama.cpp base port: {options.LlamaCppBasePort}");
+    }
 
     // Args are parsed by ClientOptions; keep them away from the host configuration.
     var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings { Args = [] });

src/Ngino.Client/TunnelClient.cs

@@ -16,11 +16,13 @@ internal sealed class TunnelClient
     private const string EmbeddingWarmupInput = "Ngino warmup";
 
     private readonly ConcurrentDictionary<string, UpstreamRequest> _activeRequests = new();
+    private readonly ConcurrentDictionary<string, PendingRequestBody> _pendingRequestBodies = new();
     private readonly HttpClient _httpClient;
     private readonly ClientOptions _options;
     private readonly ILogger<TunnelClient> _logger;
     private readonly object _modelSnapshotLock = new();
     private readonly SemaphoreSlim _sendLock = new(1, 1);
+    private readonly LlamaCppManager? _llamaCppManager;
     private List<string> _lastActiveModels = [];
     private List<string> _lastModels = [];
 
@@ -32,15 +34,49 @@ internal sealed class TunnelClient
         {
             Timeout = Timeout.InfiniteTimeSpan
         };
+
+        if (_options.UseLlamaCppViaDocker)
+        {
+            if (string.IsNullOrWhiteSpace(_options.UseOllamaModelsPath))
+            {
+                throw new InvalidOperationException(
+                    "--use-ollama-models-path is required when --use-llama-cpp-via-docker is set.");
+            }
+
+            _llamaCppManager = new LlamaCppManager(
+                _options.UseOllamaModelsPath,
+                _options.LlamaCppDockerImage,
+                _options.LlamaCppBasePort,
+                _logger);
+        }
     }
 
     public async Task RunAsync(CancellationToken cancellationToken)
     {
+        if (_llamaCppManager is not null)
+        {
+            _logger.LogInformation("Testing Docker availability...");
+            var dockerAvailable = await _llamaCppManager.TestDockerAsync();
+            if (!dockerAvailable)
+            {
+                _logger.LogWarning("Docker is not available. llama.cpp via Docker will not work.");
+            }
+            else
+            {
+                _logger.LogInformation("Docker is available. Using llama.cpp image: {Image}", _llamaCppManager.DockerImage);
+            }
+        }
+
         while (!cancellationToken.IsCancellationRequested)
         {
             using var socket = new ClientWebSocket();
             socket.Options.KeepAliveInterval = TimeSpan.FromSeconds(30);
 
+            if (_options.InsecureSkipTlsVerify)
+            {
+                socket.Options.RemoteCertificateValidationCallback = static (_, _, _, _) => true;
+            }
+
             if (!string.IsNullOrWhiteSpace(_options.Token))
             {
                 socket.Options.SetRequestHeader(ProtocolConstants.TokenHeader, _options.Token);
@@ -149,6 +185,16 @@ internal sealed class TunnelClient
 
     private async Task<List<string>> GetUpstreamModelsAsync(CancellationToken cancellationToken)
     {
+        if (_llamaCppManager is not null)
+        {
+            var models = _llamaCppManager.DiscoverModelsWithBlob();
+            return models
+                .Select(m => m.OllamaName)
+                .Where(name => !string.IsNullOrWhiteSpace(name))
+                .OrderBy(name => name, StringComparer.OrdinalIgnoreCase)
+                .ToList();
+        }
+
         using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_options.Upstream, "/api/tags"));
         using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
         response.EnsureSuccessStatusCode();
@@ -161,6 +207,17 @@ internal sealed class TunnelClient
 
     private async Task<List<string>> GetActiveUpstreamModelsAsync(CancellationToken cancellationToken)
     {
+        if (_llamaCppManager is not null)
+        {
+            var models = _llamaCppManager.DiscoverModelsWithBlob();
+            return models
+                .Where(m => _llamaCppManager.IsModelActive(m.OllamaName))
+                .Select(m => m.OllamaName)
+                .Where(name => !string.IsNullOrWhiteSpace(name))
+                .OrderBy(name => name, StringComparer.OrdinalIgnoreCase)
+                .ToList();
+        }
+
         using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_options.Upstream, "/api/ps"));
         using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
 
@@ -285,7 +342,8 @@ internal sealed class TunnelClient
         switch (message.Type)
         {
             case TunnelMessageTypes.HttpRequest:
-                StartRequest(socket, message, cancellationToken);
+                _pendingRequestBodies[message.RequestId] = new PendingRequestBody();
+                _ = Task.Run(() => StartRequest(socket, message, cancellationToken), cancellationToken);
                 break;
 
             case TunnelMessageTypes.HttpRequestBody:
@@ -293,6 +351,10 @@ internal sealed class TunnelClient
                 {
                     requestWithBody.AddBody(message.Body ?? []);
                 }
+                else if (_pendingRequestBodies.TryGetValue(message.RequestId, out var pendingBody))
+                {
+                    pendingBody.AddBody(message.Body ?? []);
+                }
                 break;
 
             case TunnelMessageTypes.HttpRequestComplete:
@@ -300,6 +362,10 @@ internal sealed class TunnelClient
                 {
                     completedRequest.CompleteBody();
                 }
+                else if (_pendingRequestBodies.TryGetValue(message.RequestId, out var pendingBody))
+                {
+                    pendingBody.Complete();
+                }
                 break;
 
             case TunnelMessageTypes.Cancel:
@@ -307,6 +373,10 @@ internal sealed class TunnelClient
                 {
                     cancelledRequest.Cancel();
                 }
+                else
+                {
+                    _pendingRequestBodies.TryRemove(message.RequestId, out _);
+                }
                 break;
 
             case TunnelMessageTypes.ModelCommand:
@@ -358,6 +428,11 @@ internal sealed class TunnelClient
             throw new InvalidOperationException("Model command is missing a model name.");
         }
 
+        if (_llamaCppManager is not null)
+        {
+            return await ExecuteModelCommandWithLlamaCppAsync(message, cancellationToken);
+        }
+
         using var request = BuildModelCommandRequest(message);
         using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
         var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
@@ -374,6 +449,118 @@ internal sealed class TunnelClient
         return BuildModelCommandResult(message.RequestId, response, body);
     }
 
+    private async Task<TunnelMessage> ExecuteModelCommandWithLlamaCppAsync(
+        TunnelMessage message, CancellationToken cancellationToken)
+    {
+        var modelName = message.Model?.Trim();
+        var normalizedCommand = NormalizeModelCommand(message.Command);
+
+        switch (normalizedCommand)
+        {
+            case "load":
+            {
+                var models = _llamaCppManager!.DiscoverModelsWithBlob();
+                var model = models.FirstOrDefault(m =>
+                    string.Equals(m.OllamaName, modelName, StringComparison.OrdinalIgnoreCase));
+
+                if (model is null)
+                {
+                    return new TunnelMessage
+                    {
+                        Type = TunnelMessageTypes.ModelCommandResult,
+                        RequestId = message.RequestId,
+                        StatusCode = 404,
+                        Error = $"Model '{modelName}' not found in Ollama models path."
+                    };
+                }
+
+                var started = await _llamaCppManager.StartModelContainerAsync(model, cancellationToken);
+                if (!started)
+                {
+                    return new TunnelMessage
+                    {
+                        Type = TunnelMessageTypes.ModelCommandResult,
+                        RequestId = message.RequestId,
+                        StatusCode = 500,
+                        Error = $"Failed to start llama.cpp container for model '{modelName}'."
+                    };
+                }
+
+                return BuildModelCommandResult(message.RequestId, 200, "OK", []);
+            }
+
+            case "unload":
+            {
+                var stopped = await _llamaCppManager!.StopModelContainerAsync(modelName!, cancellationToken);
+                if (!stopped)
+                {
+                    return new TunnelMessage
+                    {
+                        Type = TunnelMessageTypes.ModelCommandResult,
+                        RequestId = message.RequestId,
+                        StatusCode = 404,
+                        Error = $"No running llama.cpp container for model '{modelName}'."
+                    };
+                }
+
+                return BuildModelCommandResult(message.RequestId, 200, "OK", []);
+            }
+
+            case "pull":
+            case "delete":
+            {
+                using var request = BuildModelCommandRequest(_options.Upstream, message.Command, message.Model);
+                using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+                var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
+                return BuildModelCommandResult(message.RequestId, response, body);
+            }
+
+            case "show":
+            {
+                var models = _llamaCppManager!.DiscoverModelsWithBlob();
+                var model = models.FirstOrDefault(m =>
+                    string.Equals(m.OllamaName, modelName, StringComparison.OrdinalIgnoreCase));
+
+                if (model is null)
+                {
+                    return new TunnelMessage
+                    {
+                        Type = TunnelMessageTypes.ModelCommandResult,
+                        RequestId = message.RequestId,
+                        StatusCode = 404,
+                        Error = $"Model '{modelName}' not found in Ollama models path."
+                    };
+                }
+
+                var showResponse = new
+                {
+                    modelfile = $"# llama.cpp via Docker\nFROM {model.BlobDigest}\n",
+                    details = new
+                    {
+                        format = "gguf",
+                        family = "llama",
+                        parameter_size = "",
+                        quantization_level = ""
+                    },
+                    model_info = new { }
+                };
+
+                var body = JsonSerializer.SerializeToUtf8Bytes(showResponse, JsonOptions);
+                return new TunnelMessage
+                {
+                    Type = TunnelMessageTypes.ModelCommandResult,
+                    RequestId = message.RequestId,
+                    StatusCode = 200,
+                    ReasonPhrase = "OK",
+                    Body = body
+                };
+            }
+
+            default:
+                throw new InvalidOperationException($"Unsupported model command '{message.Command}' with llama.cpp.");
+        }
+    }
+
     private static TunnelMessage BuildModelCommandResult(
         string requestId,
         HttpResponseMessage response,
@@ -389,6 +576,19 @@ internal sealed class TunnelClient
         };
     }
 
+    private static TunnelMessage BuildModelCommandResult(
+        string requestId, int statusCode, string reasonPhrase, byte[] body)
+    {
+        return new TunnelMessage
+        {
+            Type = TunnelMessageTypes.ModelCommandResult,
+            RequestId = requestId,
+            StatusCode = statusCode,
+            ReasonPhrase = reasonPhrase,
+            Body = body
+        };
+    }
+
     private static bool ShouldRetryModelCommandWithEmbedding(
         string? command,
         HttpResponseMessage response,
@@ -484,15 +684,89 @@ internal sealed class TunnelClient
     private static StringContent JsonContent<T>(T value) =>
         new(JsonSerializer.Serialize(value, JsonOptions), Encoding.UTF8, "application/json");
 
-    private void StartRequest(ClientWebSocket socket, TunnelMessage message, CancellationToken cancellationToken)
+    private async Task StartRequest(ClientWebSocket socket, TunnelMessage message, CancellationToken cancellationToken)
     {
+        Uri? effectiveUpstream = null;
+        var modelName = UpstreamRequest.ExtractModelName(message);
+
+        if (_llamaCppManager is not null)
+        {
+            if (modelName is not null)
+            {
+                effectiveUpstream = _llamaCppManager.GetUpstream(modelName);
+                if (effectiveUpstream is null)
+                {
+                    _logger.LogInformation(
+                        "Request for model '{Model}' but no llama.cpp container is running. Starting one on demand...",
+                        modelName);
+
+                    var model = _llamaCppManager.DiscoverModelsWithBlob()
+                        .FirstOrDefault(m => string.Equals(m.OllamaName, modelName, StringComparison.OrdinalIgnoreCase));
+
+                    if (model is not null)
+                    {
+                        var started = await _llamaCppManager.StartModelContainerAsync(model, cancellationToken);
+                        if (started)
+                        {
+                            effectiveUpstream = _llamaCppManager.GetUpstream(modelName);
+                        }
+                    }
+
+                    if (effectiveUpstream is null)
+                    {
+                        _logger.LogWarning(
+                            "Failed to start llama.cpp container for model '{Model}'. Falling back to default upstream.",
+                            modelName);
+                    }
+                }
+            }
+        }
+
+        // When llama.cpp backend is active, set up request/response translation
+        Func<string, string?>? pathTransform = null;
+        Func<byte[], byte[]>? bodyTransform = null;
+        Func<HttpResponseMessage, CancellationToken, Task>? responseHandler = null;
+        string? translatorModelName = null;
+
+        if (_llamaCppManager is not null && effectiveUpstream is not null && modelName is not null)
+        {
+            var translator = new OllamaToLlamaCppTranslator(modelName, _logger);
+            translatorModelName = modelName;
+            bool originalRequestedStream = true;
+
+            pathTransform = path =>
+            {
+                if (translator.TryTranslatePath(message.Method ?? "GET", path, out var newPath))
+                {
+                    return newPath;
+                }
+                return null;
+            };
+
+            bodyTransform = body =>
+            {
+                originalRequestedStream = OllamaToLlamaCppTranslator.ExtractOriginalStream(body, message.PathAndQuery ?? "/");
+                return translator.TranslateBody(message.PathAndQuery ?? "/", body);
+            };
+
+            responseHandler = translator.CreateResponseHandler(
+                (response, token) => SendAsync(socket, response, token),
+                message.RequestId,
+                message.PathAndQuery ?? "/",
+                () => originalRequestedStream);
+        }
+
         var request = new UpstreamRequest(
             _options,
             _httpClient,
             message,
             (response, token) => SendAsync(socket, response, token),
             requestId => _activeRequests.TryRemove(requestId, out _),
-            cancellationToken);
+            cancellationToken,
+            effectiveUpstream: effectiveUpstream,
+            responseHandler: responseHandler,
+            pathTransform: pathTransform,
+            bodyTransform: bodyTransform);
 
         if (!_activeRequests.TryAdd(message.RequestId, request))
         {
@@ -505,10 +779,16 @@ internal sealed class TunnelClient
                     Error = "Duplicate request id."
                 },
                 cancellationToken);
+            _pendingRequestBodies.TryRemove(message.RequestId, out _);
             return;
         }
 
-        _ = Task.Run(request.RunAsync, cancellationToken);
+        if (_pendingRequestBodies.TryRemove(message.RequestId, out var pendingBody))
+        {
+            pendingBody.TransferTo(request);
+        }
+
+        await request.RunAsync();
     }
 
     private async Task SendAsync(ClientWebSocket socket, TunnelMessage message, CancellationToken cancellationToken)
@@ -536,5 +816,51 @@ internal sealed class TunnelClient
                 request.Cancel();
             }
         }
+
+        _pendingRequestBodies.Clear();
+    }
+}
+
+internal sealed class PendingRequestBody
+{
+    private readonly List<byte[]> _chunks = [];
+    private bool _completed;
+    private readonly object _lock = new();
+
+    public void AddBody(byte[] chunk)
+    {
+        if (chunk.Length == 0)
+            return;
+
+        lock (_lock)
+        {
+            _chunks.Add(chunk);
+        }
+    }
+
+    public void Complete()
+    {
+        lock (_lock)
+        {
+            _completed = true;
+        }
+    }
+
+    public void TransferTo(UpstreamRequest request)
+    {
+        lock (_lock)
+        {
+            foreach (var chunk in _chunks)
+            {
+                request.AddBody(chunk);
+            }
+
+            _chunks.Clear();
+
+            if (_completed)
+            {
+                request.CompleteBody();
+            }
+        }
     }
 }

src/Ngino.Client/UpstreamRequest.cs

@@ -19,7 +19,8 @@ internal sealed class UpstreamRequest
         "Trailer",
         "Transfer-Encoding",
         "Upgrade",
-        ProtocolConstants.TokenHeader
+        ProtocolConstants.TokenHeader,
+        ProtocolConstants.ModelHeader
     };
 
     private readonly CancellationTokenSource _cancellationTokenSource;
@@ -33,8 +34,12 @@ internal sealed class UpstreamRequest
     private readonly HttpClient _httpClient;
     private readonly TunnelMessage _initialMessage;
     private readonly Action<string> _onComplete;
-    private readonly ClientOptions _options;
+    private readonly Uri _upstream;
     private readonly Func<TunnelMessage, CancellationToken, Task> _sendAsync;
+    private readonly Func<HttpResponseMessage, CancellationToken, Task>? _responseHandler;
+    private readonly Func<string, string?>? _pathTransform;
+    private readonly Func<byte[], byte[]>? _bodyTransform;
+    private readonly List<byte[]> _bufferedBody = [];
 
     public UpstreamRequest(
         ClientOptions options,
@@ -42,13 +47,20 @@ internal sealed class UpstreamRequest
         TunnelMessage initialMessage,
         Func<TunnelMessage, CancellationToken, Task> sendAsync,
         Action<string> onComplete,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken,
+        Uri? effectiveUpstream = null,
+        Func<HttpResponseMessage, CancellationToken, Task>? responseHandler = null,
+        Func<string, string?>? pathTransform = null,
+        Func<byte[], byte[]>? bodyTransform = null)
     {
-        _options = options;
         _httpClient = httpClient;
         _initialMessage = initialMessage;
         _sendAsync = sendAsync;
         _onComplete = onComplete;
+        _upstream = effectiveUpstream ?? options.Upstream;
+        _responseHandler = responseHandler;
+        _pathTransform = pathTransform;
+        _bodyTransform = bodyTransform;
         _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
 
         if (!initialMessage.HasBody)
@@ -61,12 +73,36 @@ internal sealed class UpstreamRequest
     {
         if (body.Length > 0)
         {
-            _requestBody.Writer.TryWrite(body);
+            if (_bodyTransform is not null)
+            {
+                _bufferedBody.Add(body);
+            }
+            else
+            {
+                _requestBody.Writer.TryWrite(body);
+            }
         }
     }
 
-    public void CompleteBody() =>
+    public void CompleteBody()
+    {
+        if (_bodyTransform is not null && _bufferedBody.Count > 0)
+        {
+            var totalLength = _bufferedBody.Sum(b => b.Length);
+            var concatenated = new byte[totalLength];
+            var offset = 0;
+            foreach (var chunk in _bufferedBody)
+            {
+                chunk.CopyTo(concatenated, offset);
+                offset += chunk.Length;
+            }
+
+            var transformed = _bodyTransform(concatenated);
+            _requestBody.Writer.TryWrite(transformed);
+        }
+
         _requestBody.Writer.TryComplete();
+    }
 
     public void Cancel()
     {
@@ -84,16 +120,23 @@ internal sealed class UpstreamRequest
                 HttpCompletionOption.ResponseHeadersRead,
                 _cancellationTokenSource.Token);
 
-            await SendResponseHeadersAsync(response);
-            await SendResponseBodyAsync(response);
+            if (_responseHandler is not null)
+            {
+                await _responseHandler(response, _cancellationTokenSource.Token);
+            }
+            else
+            {
+                await SendResponseHeadersAsync(response);
+                await SendResponseBodyAsync(response);
 
-            await _sendAsync(
-                new TunnelMessage
-                {
-                    Type = TunnelMessageTypes.HttpResponseComplete,
-                    RequestId = _initialMessage.RequestId
-                },
-                _cancellationTokenSource.Token);
+                await _sendAsync(
+                    new TunnelMessage
+                    {
+                        Type = TunnelMessageTypes.HttpResponseComplete,
+                        RequestId = _initialMessage.RequestId
+                    },
+                    _cancellationTokenSource.Token);
+            }
         }
         catch (OperationCanceledException) when (_cancellationTokenSource.IsCancellationRequested)
         {
@@ -110,10 +153,39 @@ internal sealed class UpstreamRequest
         }
     }
 
+    public static string? ExtractModelName(TunnelMessage message)
+    {
+        if (message.Headers is null)
+        {
+            return null;
+        }
+
+        foreach (var header in message.Headers)
+        {
+            if (string.Equals(header.Name, ProtocolConstants.ModelHeader, StringComparison.OrdinalIgnoreCase)
+                && !string.IsNullOrWhiteSpace(header.Value))
+            {
+                return header.Value;
+            }
+        }
+
+        return null;
+    }
+
     private HttpRequestMessage BuildHttpRequest()
     {
+        var path = _initialMessage.PathAndQuery ?? "/";
+        if (_pathTransform is not null)
+        {
+            var transformed = _pathTransform(path);
+            if (transformed is not null)
+            {
+                path = transformed;
+            }
+        }
+
         var method = new HttpMethod(_initialMessage.Method ?? HttpMethod.Get.Method);
-        var request = new HttpRequestMessage(method, BuildUpstreamUri(_options.Upstream, _initialMessage.PathAndQuery));
+        var request = new HttpRequestMessage(method, BuildUpstreamUri(_upstream, path));
 
         if (_initialMessage.HasBody)
         {
@@ -185,7 +257,7 @@ internal sealed class UpstreamRequest
     private async Task SendResponseBodyAsync(HttpResponseMessage response)
     {
         await using var stream = await response.Content.ReadAsStreamAsync(_cancellationTokenSource.Token);
-        var buffer = new byte[_options.ChunkSize];
+        var buffer = new byte[_upstream switch { _ => 64 * 1024 }];
 
         while (true)
         {

src/Ngino.Protocol/ProtocolConstants.cs

@@ -7,4 +7,5 @@ public static class ProtocolConstants
     public const string TokenHeader = "X-Ngino-Token";
     public const string ClientIdHeader = "X-Ngino-Client-Id";
     public const string ReplacedCloseDescription = "ngino-replaced";
+public const string ModelHeader = "X-Ngino-Model";
 }

src/Ngino.Server/ManagementStore.cs

@@ -1198,7 +1198,7 @@ internal sealed class ManagementStore
             using var connection = OpenConnection();
             using var command = connection.CreateCommand();
             command.CommandText = """
-                SELECT g.name, gm.client_id, gm.client_pattern
+                SELECT g.id, gm.client_id, gm.client_pattern
                 FROM group_members gm
                 INNER JOIN groups g ON gm.group_id = g.id
                 """;
@@ -1206,14 +1206,14 @@ internal sealed class ManagementStore
             using var reader = command.ExecuteReader();
             while (reader.Read())
             {
-                var groupName = reader.GetString(0);
+                var groupId = reader.GetString(0);
                 var explicitClientId = reader.IsDBNull(1) ? null : reader.GetString(1);
                 var pattern = reader.IsDBNull(2) ? null : reader.GetString(2);
 
                 if (!string.IsNullOrWhiteSpace(explicitClientId)
                     && result.TryGetValue(explicitClientId, out var explicitGroups))
                 {
-                    explicitGroups.Add(groupName);
+                    explicitGroups.Add(groupId);
                 }
                 else if (!string.IsNullOrWhiteSpace(pattern))
                 {
@@ -1231,7 +1231,7 @@ internal sealed class ManagementStore
                     {
                         if (regex.IsMatch(clientId) && result.TryGetValue(clientId, out var groups))
                         {
-                            groups.Add(groupName);
+                            groups.Add(groupId);
                         }
                     }
                 }

src/Ngino.Server/ReverseProxyEndpoint.cs

@@ -523,6 +523,12 @@ internal static class ReverseProxyEndpoint
                 Headers = CollectRequestHeaders(context.Request, settings, managementStore)
             };
 
+            if (!string.IsNullOrWhiteSpace(requestedModel))
+            {
+                requestMessage.Headers ??= [];
+                requestMessage.Headers.Add(new HeaderPair(ProtocolConstants.ModelHeader, requestedModel));
+            }
+
             await connection.SendAsync(requestMessage, context.RequestAborted);
             requestBodyTask = ForwardRequestBodyAsync(context.Request, connection, requestId, hasBody, settings, logger);
             _ = requestBodyTask.ContinueWith(

src/Ngino.Server/wwwroot/admin/app.js

@@ -483,6 +483,7 @@ function renderClients() {
 
 function clientsTable(clients) {
   const clientGroups = state.summary?.clientGroups || {};
+  const groupNames = Object.fromEntries((state.summary?.groups || []).map((g) => [g.id, g.name]));
   const rows = clients.map((client) => {
     const groups = clientGroups[client.id] || [];
     return `
@@ -507,7 +508,7 @@ function clientsTable(clients) {
       <td>${modelBadges(client.activeModels)}</td>
       <td>
         <div class="badge-row">
-          ${groups.length ? groups.map((g) => `<a class="badge" href="#groups/${encodeURIComponent(g)}">${escapeHtml(g)}</a>`).join("") : `<span class="cell-sub">None</span>`}
+          ${groups.length ? groups.map((g) => `<a class="badge" href="#groups/${encodeURIComponent(g)}">${escapeHtml(groupNames[g] || g)}</a>`).join("") : `<span class="cell-sub">None</span>`}
         </div>
       </td>
       <td>

tests/Ngino.Client.Tests/ClientOptionsTests.cs

@@ -0,0 +1,31 @@
+using Ngino.Client;
+using Xunit;
+
+namespace Ngino.Client.Tests;
+
+public sealed class ClientOptionsTests
+{
+    [Fact]
+    public void InsecureTlsIsDisabledByDefault()
+    {
+        var options = ClientOptions.Parse([]);
+
+        Assert.False(options.InsecureSkipTlsVerify);
+    }
+
+    [Fact]
+    public void InsecureTlsCanBeEnabledWithFlag()
+    {
+        var options = ClientOptions.Parse(["--insecure-skip-tls-verify"]);
+
+        Assert.True(options.InsecureSkipTlsVerify);
+    }
+
+    [Fact]
+    public void InsecureTlsCanBeEnabledWithEnvironmentStyleValue()
+    {
+        var options = ClientOptions.Parse(["--insecure-skip-tls-verify=true"]);
+
+        Assert.True(options.InsecureSkipTlsVerify);
+    }
+}