feat(Client): adds llama.cpp backend

deploy/install-client.ps1

@@ -14,7 +14,11 @@ param(
     [string]$InstallDir = "$env:ProgramFiles\Ngino Client",
     [string]$ServiceName = "NginoClient",
     [switch]$InsecureSkipTlsVerify,
-    [switch]$NoOllama
+    [switch]$NoOllama,
+    [switch]$UseLlamaCppViaDocker,
+    [string]$UseOllamaModelsPath = "",
+    [string]$LlamaCppDockerImage = "",
+    [int]$LlamaCppBasePort = 0
 )
 
 $ErrorActionPreference = "Stop"
@@ -54,6 +58,14 @@ if ([string]::IsNullOrWhiteSpace($Token)) {
     finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($tokenPointer) }
 }
 if ([string]::IsNullOrWhiteSpace($Token)) { throw "Token is required." }
+
+if ($UseLlamaCppViaDocker -and [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) {
+    $UseOllamaModelsPath = Read-Host "Ollama models path (e.g. C:\Users\user\.ollama\models)"
+}
+if ($UseLlamaCppViaDocker -and [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) {
+    throw "Ollama models path is required with -UseLlamaCppViaDocker."
+}
+
 if ($ServiceName -notmatch '^[A-Za-z0-9_.-]+$') { throw "ServiceName contains unsupported characters." }
 
 $scriptDir = $PSScriptRoot
@@ -149,6 +161,18 @@ $serviceEnvironment = @(
     "DOTNET_CLI_TELEMETRY_OPTOUT=1",
     "DOTNET_NOLOGO=1"
 )
+if ($UseLlamaCppViaDocker) {
+    $serviceEnvironment += "NGINO_USE_LLAMA_CPP_VIA_DOCKER=true"
+    if (-not [string]::IsNullOrWhiteSpace($UseOllamaModelsPath)) {
+        $serviceEnvironment += "NGINO_USE_OLLAMA_MODELS_PATH=$UseOllamaModelsPath"
+    }
+    if (-not [string]::IsNullOrWhiteSpace($LlamaCppDockerImage)) {
+        $serviceEnvironment += "NGINO_LLAMA_CPP_DOCKER_IMAGE=$LlamaCppDockerImage"
+    }
+    if ($LlamaCppBasePort -gt 0) {
+        $serviceEnvironment += "NGINO_LLAMA_CPP_BASE_PORT=$LlamaCppBasePort"
+    }
+}
 if ($InsecureSkipTlsVerify) {
     Write-Warn "Server TLS certificate validation is disabled for $ServiceName."
 }
@@ -169,6 +193,9 @@ Write-Host "  Client ID:   $ClientId"
 Write-Host "  Upstream:    $Upstream"
 Write-Host "  Service:     $ServiceName"
 Write-Host "  Install dir: $InstallDir"
+if ($UseLlamaCppViaDocker) {
+    Write-Host "  llama.cpp:   enabled (models: $UseOllamaModelsPath)"
+}
 Write-Host ""
 Write-Host "  Manage:  Get-Service $ServiceName | Start-Service/Stop-Service/Restart-Service"
 Write-Host "  Logs:    Get-WinEvent -LogName Application | Where-Object ProviderName -eq NginoClient"

deploy/install-client.sh

@@ -16,6 +16,10 @@ TOKEN=""
 CLIENT_ID="$(hostname -s 2>/dev/null || echo "linux-client")"
 UPSTREAM="$DEFAULT_UPSTREAM"
 SKIP_OLLAMA=false
+USE_LLAMA_CPP_VIA_DOCKER=false
+USE_OLLAMA_MODELS_PATH=""
+LLAMA_CPP_DOCKER_IMAGE=""
+LLAMA_CPP_BASE_PORT=""
 
 # ── Colors ────────────────────────────────────────────────────────────────────
 RED='\033[0;31m'
@@ -45,11 +49,21 @@ Optional:
   --install-dir <dir>  Install directory; defaults to $DEFAULT_INSTALL_DIR
   --service-name <n>   systemd service name; defaults to $DEFAULT_SERVICE_NAME
   --no-ollama          Skip Ollama installation and status check
+  --use-llama-cpp-via-docker
+                       Use llama.cpp via Docker for inference instead of Ollama
+  --use-ollama-models-path <dir>
+                       Path to Ollama models directory (manifests/blobs); required with --use-llama-cpp-via-docker
+  --llama-cpp-docker-image <img>
+                       llama.cpp Docker image; defaults to auto-detected (rocm/cuda/cpu)
+  --llama-cpp-base-port <num>
+                       Base port for llama.cpp containers; defaults to 8081
   -h, --help           Show this help message
 
 Examples:
   $0 --server http://gpu-server:5050 --token "my-secret"
   $0 --server http://gpu-server:5050 --token "my-secret" --no-ollama
+  $0 --server http://gpu-server:5050 --token "my-secret" \\
+     --use-llama-cpp-via-docker --use-ollama-models-path /usr/share/ollama/.ollama/models
 EOF
     exit 0
 }
@@ -57,15 +71,19 @@ EOF
 # ── Argument parsing ──────────────────────────────────────────────────────────
 while [[ $# -gt 0 ]]; do
     case "$1" in
-        --server)       SERVER_URL="$2"; shift 2 ;;
-        --token)        TOKEN="$2"; shift 2 ;;
-        --client-id)    CLIENT_ID="$2"; shift 2 ;;
-        --upstream)     UPSTREAM="$2"; shift 2 ;;
-        --install-dir)  INSTALL_DIR="$2"; shift 2 ;;
-        --service-name) SERVICE_NAME="$2"; shift 2 ;;
-        --no-ollama)    SKIP_OLLAMA=true; shift ;;
-        -h|--help)      usage ;;
-        *)              die "Unknown option: $1" ;;
+        --server)                   SERVER_URL="$2"; shift 2 ;;
+        --token)                    TOKEN="$2"; shift 2 ;;
+        --client-id)                CLIENT_ID="$2"; shift 2 ;;
+        --upstream)                 UPSTREAM="$2"; shift 2 ;;
+        --install-dir)              INSTALL_DIR="$2"; shift 2 ;;
+        --service-name)             SERVICE_NAME="$2"; shift 2 ;;
+        --no-ollama)                SKIP_OLLAMA=true; shift ;;
+        --use-llama-cpp-via-docker) USE_LLAMA_CPP_VIA_DOCKER=true; shift ;;
+        --use-ollama-models-path)   USE_OLLAMA_MODELS_PATH="$2"; shift 2 ;;
+        --llama-cpp-docker-image)   LLAMA_CPP_DOCKER_IMAGE="$2"; shift 2 ;;
+        --llama-cpp-base-port)      LLAMA_CPP_BASE_PORT="$2"; shift 2 ;;
+        -h|--help)                  usage ;;
+        *)                          die "Unknown option: $1" ;;
     esac
 done
 
@@ -85,6 +103,13 @@ if [[ -z "$TOKEN" ]]; then
     die "Token is required."
 fi
 
+if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" && -z "$USE_OLLAMA_MODELS_PATH" ]]; then
+    read -rp "Ollama models path (e.g. /usr/share/ollama/.ollama/models): " USE_OLLAMA_MODELS_PATH
+fi
+if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" && -z "$USE_OLLAMA_MODELS_PATH" ]]; then
+    die "Ollama models path is required with --use-llama-cpp-via-docker."
+fi
+
 # ── Root check ────────────────────────────────────────────────────────────────
 if [[ $EUID -ne 0 ]]; then
     die "This script must be run as root (or with sudo)."
@@ -265,7 +290,21 @@ info "Client installed to $INSTALL_DIR."
 # ── Write environment file (avoids shell injection in unit file) ─────────────
 ENV_DIR="/etc/ngino-client"
 mkdir -p "$ENV_DIR"
-printf 'NGINO_TOKEN=%s\n' "$TOKEN" > "$ENV_DIR/env"
+{
+    printf 'NGINO_TOKEN=%s\n' "$TOKEN"
+    if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" ]]; then
+        printf 'NGINO_USE_LLAMA_CPP_VIA_DOCKER=true\n'
+        if [[ -n "$USE_OLLAMA_MODELS_PATH" ]]; then
+            printf 'NGINO_USE_OLLAMA_MODELS_PATH=%s\n' "$USE_OLLAMA_MODELS_PATH"
+        fi
+        if [[ -n "$LLAMA_CPP_DOCKER_IMAGE" ]]; then
+            printf 'NGINO_LLAMA_CPP_DOCKER_IMAGE=%s\n' "$LLAMA_CPP_DOCKER_IMAGE"
+        fi
+        if [[ -n "$LLAMA_CPP_BASE_PORT" ]]; then
+            printf 'NGINO_LLAMA_CPP_BASE_PORT=%s\n' "$LLAMA_CPP_BASE_PORT"
+        fi
+    fi
+} > "$ENV_DIR/env"
 chmod 600 "$ENV_DIR/env"
 info "Environment file written to $ENV_DIR/env (mode 0600)."
 
@@ -280,8 +319,8 @@ fi
 cat > "$SERVICE_FILE" <<EOF
 [Unit]
 Description=Ngino Tunnel Client
-After=network-online.target
-Wants=network-online.target
+After=network-online.target docker.service
+Wants=network-online.target docker.service
 $([ "$SKIP_OLLAMA" = "false" ] && echo "After=ollama.service")
 $([ "$SKIP_OLLAMA" = "false" ] && echo "Wants=ollama.service")
 
@@ -318,6 +357,9 @@ echo "  Client ID:   $CLIENT_ID"
 echo "  Upstream:    $UPSTREAM"
 echo "  Service:     $SERVICE_NAME"
 echo "  Install dir: $INSTALL_DIR"
+if [[ "$USE_LLAMA_CPP_VIA_DOCKER" == "true" ]]; then
+    echo "  llama.cpp:   enabled (models: $USE_OLLAMA_MODELS_PATH)"
+fi
 echo
 echo "  Manage:  systemctl {start|stop|restart|status} $SERVICE_NAME"
 echo "  Logs:    journalctl -u $SERVICE_NAME -f"