diff --git a/README.md b/README.md index d111c87..f451842 100644 --- a/README.md +++ b/README.md @@ -133,11 +133,11 @@ sudo bash deploy/install-client.sh --server http://your-server:5050 --token "cha ``` or using ollama models with llama.cpp backend using ROCm: ```bash -sudo bash deploy/install-client.sh --server https://ai.domain.tld --token "change-me" --use-llama-cpp-via-docker --use-ollama-models-path /usr/share/ollama/.ollama/models --llama-cpp-docker-image ghcr.io/ggml-org/llama.cpp:server-rocm --llama-cpp-base-port 8081 +sudo bash deploy/install-client.sh --server https://ai.domain.tld --token "change-me" --use-llama-cpp-via-docker --use-ollama-models-path /usr/share/ollama/.ollama/models --llama-cpp-docker-image ghcr.io/ggml-org/llama.cpp:server-rocm --llama-cpp-base-port 8081 --llama-cpp-parallel 128 ``` or using ollama models with llama.cpp backend using CUDA: ```bash -sudo bash deploy/install-client.sh --server https://ai.domain.tld --token "change-me" --use-llama-cpp-via-docker --use-ollama-models-path /usr/share/ollama/.ollama/models --llama-cpp-docker-image ghcr.io/ggml-org/llama.cpp:server-cuda --llama-cpp-base-port 8081 +sudo bash deploy/install-client.sh --server https://ai.domain.tld --token "change-me" --use-llama-cpp-via-docker --use-ollama-models-path /usr/share/ollama/.ollama/models --llama-cpp-docker-image ghcr.io/ggml-org/llama.cpp:server-cuda --llama-cpp-base-port 8081 --llama-cpp-parallel 128 ``` Options: `--server`, `--token` (required); `--client-id`, `--upstream`, `--install-dir`, `--service-name`, `--no-ollama` (optional). Missing required values are prompted interactively. @@ -150,6 +150,15 @@ llama.cpp via Docker options (replaces Ollama for inferencing): | `--use-ollama-models-path ` | Path to Ollama models directory (`manifests/blobs`); required with the flag above | | `--llama-cpp-docker-image ` | Docker image; defaults to auto-detected (rocm/cuda/cpu) | | `--llama-cpp-base-port ` | Base port for containers; defaults to `8081` | +| `--llama-cpp-parallel ` | llama.cpp parallel slots per container; if unset, llama.cpp's own default is used (which is `1`) | +| `--llama-cpp-fallback-cooldown ` | Seconds before llama.cpp is retried after a failed container start; defaults to `180` | +| `--log-dir ` | Directory for log files; defaults to `/Logs` | + +### llama.cpp fallback to Ollama + +Models are served via llama.cpp Docker containers. If a container cannot be started for a model (for example, the model's GGUF blob is incompatible with the llama.cpp build), the client falls back to the Ollama upstream for that model. Transient start failures are remembered for `--llama-cpp-fallback-cooldown` seconds (default 3 minutes) and then retried; a container that starts but exits before becoming ready marks the model as falling back until it is unloaded. `load`/`unload` model commands and on-demand request routing are all covered; a failed container start is detected quickly by watching the container state, and the container log tail is written to the client log to aid debugging. + +Note: some hybrid SSM/attention models (e.g. `qwen3.5-coder-next`) are converted by Ollama into a GGUF tensor layout that stock llama.cpp cannot load (`missing tensor 'blk.0.ssm_dt.bias'` and similar). Such models are served via the Ollama fallback above. If you want them to run on llama.cpp instead, use a Hugging Face-converted GGUF (e.g. `unsloth/Qwen3-Coder-Next-GGUF`) rather than the Ollama blob. The script ensures .NET 10 and Ollama are installed, builds the client self-contained, installs it to `/opt/Ngino-client`, and creates a systemd service (`Ngino-client`). Logs: `journalctl -u Ngino-client -f`. diff --git a/deploy/install-client.sh b/deploy/install-client.sh index dd5d7ec..5b36772 100755 --- a/deploy/install-client.sh +++ b/deploy/install-client.sh @@ -20,6 +20,9 @@ USE_LLAMA_CPP_VIA_DOCKER=false USE_OLLAMA_MODELS_PATH="" LLAMA_CPP_DOCKER_IMAGE="" LLAMA_CPP_BASE_PORT="" +LLAMA_CPP_PARALLEL="" +LLAMA_CPP_FALLBACK_COOLDOWN="" +LOG_DIR="" # ── Colors ──────────────────────────────────────────────────────────────────── RED='\033[0;31m' @@ -57,13 +60,19 @@ Optional: llama.cpp Docker image; defaults to auto-detected (rocm/cuda/cpu) --llama-cpp-base-port Base port for llama.cpp containers; defaults to 8081 + --llama-cpp-parallel + llama.cpp parallel slots per container; if unset, llama.cpp's own default is used + --llama-cpp-fallback-cooldown + Seconds before llama.cpp is retried after a failed container start; defaults to 180 + --log-dir Directory for log files; defaults to /Logs -h, --help Show this help message Examples: $0 --server http://gpu-server:5050 --token "my-secret" $0 --server http://gpu-server:5050 --token "my-secret" --no-ollama $0 --server http://gpu-server:5050 --token "my-secret" \\ - --use-llama-cpp-via-docker --use-ollama-models-path /usr/share/ollama/.ollama/models + --use-llama-cpp-via-docker --use-ollama-models-path /usr/share/ollama/.ollama/models \\ + --llama-cpp-parallel 128 EOF exit 0 } @@ -82,6 +91,9 @@ while [[ $# -gt 0 ]]; do --use-ollama-models-path) USE_OLLAMA_MODELS_PATH="$2"; shift 2 ;; --llama-cpp-docker-image) LLAMA_CPP_DOCKER_IMAGE="$2"; shift 2 ;; --llama-cpp-base-port) LLAMA_CPP_BASE_PORT="$2"; shift 2 ;; + --llama-cpp-parallel) LLAMA_CPP_PARALLEL="$2"; shift 2 ;; + --llama-cpp-fallback-cooldown) LLAMA_CPP_FALLBACK_COOLDOWN="$2"; shift 2 ;; + --log-dir) LOG_DIR="$2"; shift 2 ;; -h|--help) usage ;; *) die "Unknown option: $1" ;; esac @@ -311,6 +323,15 @@ mkdir -p "$ENV_DIR" if [[ -n "$LLAMA_CPP_BASE_PORT" ]]; then printf 'NGINO_LLAMA_CPP_BASE_PORT=%s\n' "$LLAMA_CPP_BASE_PORT" fi + if [[ -n "$LLAMA_CPP_PARALLEL" ]]; then + printf 'NGINO_LLAMA_CPP_PARALLEL=%s\n' "$LLAMA_CPP_PARALLEL" + fi + if [[ -n "$LLAMA_CPP_FALLBACK_COOLDOWN" ]]; then + printf 'NGINO_LLAMA_CPP_FALLBACK_COOLDOWN_SECONDS=%s\n' "$LLAMA_CPP_FALLBACK_COOLDOWN" + fi + fi + if [[ -n "$LOG_DIR" ]]; then + printf 'NGINO_LOG_DIR=%s\n' "$LOG_DIR" fi } > "$ENV_DIR/env" chmod 600 "$ENV_DIR/env" diff --git a/src/Ngino.Client/ClientOptions.cs b/src/Ngino.Client/ClientOptions.cs index 4a3256c..cab7b65 100644 --- a/src/Ngino.Client/ClientOptions.cs +++ b/src/Ngino.Client/ClientOptions.cs @@ -28,6 +28,12 @@ internal sealed class ClientOptions public int LlamaCppBasePort { get; init; } = 8081; + public int? LlamaCppParallel { get; init; } + + public TimeSpan LlamaCppFallbackCooldown { get; init; } = TimeSpan.FromMinutes(3); + + public string? LogDirectory { get; init; } + public Uri TunnelUri { get @@ -68,7 +74,10 @@ internal sealed class ClientOptions UseLlamaCppViaDocker = ReadBool(values, false, "use-llama-cpp-via-docker", "NGINO_USE_LLAMA_CPP_VIA_DOCKER"), UseOllamaModelsPath = NormalizeDirectoryPath(Read(values, "use-ollama-models-path", "NGINO_USE_OLLAMA_MODELS_PATH")), LlamaCppDockerImage = Read(values, "llama-cpp-docker-image", "NGINO_LLAMA_CPP_DOCKER_IMAGE"), - LlamaCppBasePort = ReadInt(values, 8081, "llama-cpp-base-port", "NGINO_LLAMA_CPP_BASE_PORT") + LlamaCppBasePort = ReadInt(values, 8081, "llama-cpp-base-port", "NGINO_LLAMA_CPP_BASE_PORT"), + LlamaCppParallel = ReadOptionalInt(values, "llama-cpp-parallel", "NGINO_LLAMA_CPP_PARALLEL"), + LlamaCppFallbackCooldown = TimeSpan.FromSeconds(ReadInt(values, 180, "llama-cpp-fallback-cooldown", "NGINO_LLAMA_CPP_FALLBACK_COOLDOWN_SECONDS")), + LogDirectory = NormalizeDirectoryPath(Read(values, "log-dir", "NGINO_LOG_DIR")) }; } @@ -87,6 +96,10 @@ internal sealed class ClientOptions --use-ollama-models-path Path to Ollama models directory (manifests/blobs), required with --use-llama-cpp-via-docker --llama-cpp-docker-image llama.cpp Docker image; defaults to auto-detected (rocm/cuda/cpu) --llama-cpp-base-port Base port for llama.cpp containers; defaults to 8081 + --llama-cpp-parallel llama.cpp parallel slots per container; if unset, llama.cpp's own default is used + --llama-cpp-fallback-cooldown + Seconds before llama.cpp is retried after a failed container start; defaults to 180 + --log-dir Directory for log files; defaults to /Logs """; private static Dictionary ParseArgs(string[] args) @@ -160,6 +173,12 @@ internal sealed class ClientOptions return int.TryParse(value, out var parsed) && parsed > 0 ? parsed : fallback; } + private static int? ReadOptionalInt(Dictionary values, params string[] keys) + { + var value = Read(values, keys); + return int.TryParse(value, out var parsed) && parsed > 0 ? parsed : null; + } + private static bool ReadBool(Dictionary values, bool fallback, params string[] keys) { var value = Read(values, keys); diff --git a/src/Ngino.Client/FileLogger.cs b/src/Ngino.Client/FileLogger.cs new file mode 100644 index 0000000..9d42045 --- /dev/null +++ b/src/Ngino.Client/FileLogger.cs @@ -0,0 +1,113 @@ +using System.Text; +using Microsoft.Extensions.Logging; + +namespace Ngino.Client; + +internal sealed class FileLoggerProvider : ILoggerProvider +{ + private const long DefaultMaxFileSizeBytes = 5L * 1024 * 1024; + private const string LogFileName = "ngino-client.log"; + private const string RotatedLogFileName = "ngino-client.log.1"; + + private readonly string _directory; + private readonly long _maxFileSizeBytes; + private readonly object _lock = new(); + private StreamWriter _writer = null!; + private string _currentFile = null!; + + public FileLoggerProvider(string directory, long maxFileSizeBytes = DefaultMaxFileSizeBytes) + { + _directory = directory; + _maxFileSizeBytes = maxFileSizeBytes; + Directory.CreateDirectory(directory); + OpenFile(); + } + + public string LogDirectory => _directory; + + public ILogger CreateLogger(string categoryName) => new FileLogger(this, categoryName); + + public void WriteLog(DateTime timestamp, LogLevel level, string category, string message) + { + lock (_lock) + { + var line = $"{timestamp:yyyy-MM-dd HH:mm:ss.fff} [{level}] {category}: {message}"; + if (_writer.BaseStream.Length + line.Length + 2 > _maxFileSizeBytes) + { + RotateFile(); + } + + _writer.WriteLine(line); + } + } + + public void Dispose() + { + lock (_lock) + { + _writer.Dispose(); + } + } + + private void OpenFile() + { + _currentFile = Path.Combine(_directory, LogFileName); + _writer = new StreamWriter( + new FileStream(_currentFile, FileMode.Append, FileAccess.Write, FileShare.ReadWrite), + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)) + { + AutoFlush = true + }; + } + + private void RotateFile() + { + _writer.Dispose(); + + var rotatedFile = Path.Combine(_directory, RotatedLogFileName); + try + { + File.Delete(rotatedFile); + if (File.Exists(_currentFile)) + { + File.Move(_currentFile, rotatedFile); + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + + OpenFile(); + } +} + +internal sealed class FileLogger(FileLoggerProvider provider, string category) : ILogger +{ + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Trace; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (!IsEnabled(logLevel)) + { + return; + } + + var message = formatter(state, exception); + if (exception is not null) + { + message += Environment.NewLine + exception; + } + + provider.WriteLog(DateTime.Now, logLevel, category, message); + } +} diff --git a/src/Ngino.Client/LlamaCppManager.cs b/src/Ngino.Client/LlamaCppManager.cs index eccdbe3..d116ef4 100644 --- a/src/Ngino.Client/LlamaCppManager.cs +++ b/src/Ngino.Client/LlamaCppManager.cs @@ -16,24 +16,35 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable private const string NginoContainerLabel = "ngino-llamacpp"; private static readonly TimeSpan DockerTimeout = TimeSpan.FromSeconds(60); private static readonly TimeSpan ContainerStartTimeout = TimeSpan.FromMinutes(5); + private static readonly TimeSpan DefaultFallbackCooldown = TimeSpan.FromMinutes(3); private readonly string _blobsPath; private readonly string _manifestsPath; private readonly string _dockerImage; private readonly int _basePort; + private readonly int? _parallel; + private readonly TimeSpan _fallbackCooldown; private readonly ILogger _logger; private readonly ConcurrentDictionary _modelPorts = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _reservedPorts = new(); + private readonly ConcurrentDictionary _fallbackModels = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _modelStartLocks = new(StringComparer.OrdinalIgnoreCase); + private readonly object _portAllocationLock = new(); public LlamaCppManager( string ollamaModelsPath, string? dockerImage, int? basePort, - ILogger? logger = null) + ILogger? logger = null, + TimeSpan fallbackCooldown = default, + int? parallel = null) { _manifestsPath = Path.Combine(ollamaModelsPath, "manifests"); _blobsPath = Path.Combine(ollamaModelsPath, "blobs"); _dockerImage = dockerImage ?? GetDefaultDockerImage(); _basePort = basePort ?? DefaultBasePort; + _fallbackCooldown = fallbackCooldown > TimeSpan.Zero ? fallbackCooldown : DefaultFallbackCooldown; + _parallel = parallel is > 0 ? parallel : null; _logger = logger ?? NullLogger.Instance; } @@ -78,6 +89,45 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable return _modelPorts.ContainsKey(ollamaModelName); } + public void MarkModelAsFallback(string ollamaModelName) + { + _fallbackModels[ollamaModelName] = DateTime.UtcNow.Add(_fallbackCooldown); + _logger.LogWarning( + "Model {Model} will fall back to the Ollama upstream for {Cooldown} before llama.cpp is retried.", + ollamaModelName, _fallbackCooldown); + } + + public void MarkModelAsPermanentFallback(string ollamaModelName) + { + _fallbackModels[ollamaModelName] = DateTime.MaxValue; + _logger.LogError( + "Model {Model} exited its llama.cpp container before becoming ready. It will fall back to the Ollama upstream until it is unloaded.", + ollamaModelName); + } + + public bool IsModelOnFallback(string ollamaModelName) + { + if (_fallbackModels.TryGetValue(ollamaModelName, out var expiresAt)) + { + if (expiresAt > DateTime.UtcNow) + { + return true; + } + + _fallbackModels.TryRemove(ollamaModelName, out _); + } + + return false; + } + + public void ClearModelFallback(string ollamaModelName) + { + if (_fallbackModels.TryRemove(ollamaModelName, out _)) + { + _logger.LogInformation("Cleared llama.cpp fallback marker for model {Model}.", ollamaModelName); + } + } + public async Task IsContainerRunningAsync(string ollamaModelName) { if (!_modelPorts.ContainsKey(ollamaModelName)) @@ -96,7 +146,7 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable _logger.LogWarning( "llama.cpp container {ContainerName} is no longer running. Invalidating cached port for {Model}.", containerName, ollamaModelName); - _modelPorts.TryRemove(ollamaModelName, out _); + RemoveModelPort(ollamaModelName); } return running; @@ -104,11 +154,11 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable public bool RemoveModelMapping(string ollamaModelName) { - if (_modelPorts.TryRemove(ollamaModelName, out var port)) + if (RemoveModelPort(ollamaModelName)) { _logger.LogWarning( - "Removed stale llama.cpp port mapping for {Model} (port {Port}).", - ollamaModelName, port); + "Removed stale llama.cpp port mapping for {Model}.", + ollamaModelName); return true; } @@ -134,45 +184,75 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable return false; } + var startLock = _modelStartLocks.GetOrAdd(ollamaName, static _ => new SemaphoreSlim(1, 1)); + await startLock.WaitAsync(cancellationToken); + try + { + return await StartModelContainerCoreAsync(model, cancellationToken); + } + finally + { + startLock.Release(); + } + } + + private async Task StartModelContainerCoreAsync(LlamaCppModel model, CancellationToken cancellationToken) + { + var ollamaName = model.OllamaName; + if (string.IsNullOrWhiteSpace(ollamaName)) + { + return false; + } + if (_modelPorts.ContainsKey(ollamaName)) { _logger.LogInformation("Model {Model} already has a running container", ollamaName); return true; } + if (IsModelOnFallback(ollamaName)) + { + _logger.LogInformation( + "Model {Model} previously failed to load via llama.cpp. Skipping container start.", + ollamaName); + return false; + } + if (!File.Exists(model.BlobPath)) { _logger.LogError("Model blob not found: {BlobPath}", model.BlobPath); + MarkModelAsFallback(ollamaName); return false; } var port = FindAvailablePort(); var containerName = SanitizeContainerName($"ngino-llamacpp-{ollamaName}"); - var existingPort = await FindExistingContainerPortAsync(containerName); - if (existingPort.HasValue) - { - _logger.LogInformation( - "Reusing existing container for {Model} on port {Port}", ollamaName, existingPort.Value); - _modelPorts[ollamaName] = existingPort.Value; - return true; - } - - await RunDockerAsync(["rm", "-f", containerName], CancellationToken.None); - - var args = BuildDockerRunArgs(containerName, model, port); - _logger.LogInformation( - "Starting llama.cpp container for {Model} on port {Port}: docker {Args}", - ollamaName, port, string.Join(" ", args)); - try { + var existingPort = await FindExistingContainerPortAsync(containerName); + if (existingPort.HasValue) + { + _logger.LogInformation( + "Reusing existing container for {Model} on port {Port}", ollamaName, existingPort.Value); + _modelPorts[ollamaName] = existingPort.Value; + return true; + } + + await RunDockerAsync(["rm", "-f", containerName], CancellationToken.None); + + var args = BuildDockerRunArgs(containerName, model, port); + _logger.LogInformation( + "Starting llama.cpp container for {Model} on port {Port}: docker {Args}", + ollamaName, port, string.Join(" ", args)); + var (exitCode, output) = await RunDockerWithOutputAsync(args, cancellationToken); if (exitCode != 0) { _logger.LogError( "Failed to start llama.cpp container for {Model}, exit code: {ExitCode}, output: {Output}", ollamaName, exitCode, output); + MarkModelAsFallback(ollamaName); return false; } @@ -180,12 +260,21 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable "llama.cpp container for {Model} started on port {Port}. Waiting for it to become ready...", ollamaName, port); - var ready = await WaitForServerReadyAsync("localhost", port, cancellationToken); - if (!ready) + var result = await WaitForServerReadyAsync("localhost", port, containerName, cancellationToken); + if (result != ContainerStartResult.Ready) { + if (result == ContainerStartResult.ContainerExited) + { + MarkModelAsPermanentFallback(ollamaName); + } + else + { + MarkModelAsFallback(ollamaName); + } + _logger.LogError( - "llama.cpp container for {Model} did not become ready on port {Port} within {Timeout}. Stopping it.", - ollamaName, port, ContainerStartTimeout); + "llama.cpp container for {Model} did not become ready on port {Port} within {Timeout} ({Result}). Stopping it.", + ollamaName, port, ContainerStartTimeout, result); try { @@ -207,13 +296,18 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable catch (Exception ex) { _logger.LogError(ex, "Failed to start llama.cpp container for {Model}", ollamaName); + MarkModelAsFallback(ollamaName); return false; } + finally + { + ReleaseReservedPort(port); + } } public async Task StopModelContainerAsync(string ollamaModelName, CancellationToken cancellationToken) { - if (!_modelPorts.TryRemove(ollamaModelName, out _)) + if (!RemoveModelPort(ollamaModelName)) { _logger.LogWarning("No running container found for model {Model}", ollamaModelName); return false; @@ -227,6 +321,7 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable { await RunDockerAsync(["stop", "--time", "10", containerName], cancellationToken); await RunDockerAsync(["rm", "-f", containerName], cancellationToken); + ClearModelFallback(ollamaModelName); return true; } catch (Exception ex) @@ -262,6 +357,8 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable } _modelPorts.Clear(); + _reservedPorts.Clear(); + _fallbackModels.Clear(); } public async Task TestDockerAsync() @@ -389,7 +486,6 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable { "run", "-d", - "--rm", "--label", $"{NginoContainerLabel}=true", "--name", containerName, "-p", $"{port}:{port}", @@ -413,9 +509,12 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable args.Add("-m"); args.Add($"/models/blobs/{blobFile}"); args.Add("-ngl"); - args.Add("999"); - args.Add("--parallel"); - args.Add("4"); + args.Add("auto"); + if (_parallel.HasValue) + { + args.Add("--parallel"); + args.Add(_parallel.Value.ToString()); + } args.Add("--host"); args.Add("0.0.0.0"); args.Add("--port"); @@ -476,25 +575,51 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable private int FindAvailablePort() { - var usedPorts = new HashSet(_modelPorts.Values); - var port = _basePort; - - while (usedPorts.Contains(port)) + lock (_portAllocationLock) { - port++; - } + var usedPorts = new HashSet(_modelPorts.Values); + foreach (var reservedPort in _reservedPorts.Keys) + { + usedPorts.Add(reservedPort); + } - return port; + var port = _basePort; + + while (usedPorts.Contains(port)) + { + port++; + } + + _reservedPorts[port] = 0; + return port; + } } - private static async Task WaitForServerReadyAsync( - string host, int port, CancellationToken cancellationToken) + private void ReleaseReservedPort(int port) + { + _reservedPorts.TryRemove(port, out _); + } + + private bool RemoveModelPort(string ollamaModelName) + { + if (_modelPorts.TryRemove(ollamaModelName, out var port)) + { + ReleaseReservedPort(port); + return true; + } + + return false; + } + + private async Task WaitForServerReadyAsync( + string host, int port, string containerName, CancellationToken cancellationToken) { var deadline = DateTime.UtcNow + ContainerStartTimeout; - if (!await WaitForTcpPortAsync(host, port, deadline, cancellationToken)) + var tcpResult = await WaitForTcpPortAsync(host, port, containerName, deadline, cancellationToken); + if (tcpResult != ContainerStartResult.Ready) { - return false; + return tcpResult; } using var handler = new SocketsHttpHandler @@ -508,41 +633,57 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable { cancellationToken.ThrowIfCancellationRequested(); + if (!await IsDockerContainerRunningAsync(containerName)) + { + await LogContainerOutputAsync(containerName); + return ContainerStartResult.ContainerExited; + } + try { using var response = await httpClient.GetAsync( $"http://{host}:{port}/health", cancellationToken); if (response.IsSuccessStatusCode) { - return true; + return ContainerStartResult.Ready; } } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { } - catch (Exception) + catch (HttpRequestException ex) { - return false; + _logger.LogDebug(ex, "Health probe of llama.cpp container {ContainerName} failed; retrying.", containerName); + } + catch (IOException ex) + { + _logger.LogDebug(ex, "Health probe of llama.cpp container {ContainerName} failed; retrying.", containerName); } await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); } - return false; + return ContainerStartResult.TimedOut; } - private static async Task WaitForTcpPortAsync( - string host, int port, DateTime deadline, CancellationToken cancellationToken) + private async Task WaitForTcpPortAsync( + string host, int port, string containerName, DateTime deadline, CancellationToken cancellationToken) { while (DateTime.UtcNow < deadline) { cancellationToken.ThrowIfCancellationRequested(); + if (!await IsDockerContainerRunningAsync(containerName)) + { + await LogContainerOutputAsync(containerName); + return ContainerStartResult.ContainerExited; + } + try { using var client = new TcpClient(); await client.ConnectAsync(host, port, cancellationToken); - return true; + return ContainerStartResult.Ready; } catch (OperationCanceledException) { @@ -555,7 +696,49 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); } - return false; + return ContainerStartResult.TimedOut; + } + + private async Task IsDockerContainerRunningAsync(string containerName) + { + try + { + var (exitCode, output) = await RunDockerWithOutputAsync( + ["inspect", "-f", "{{.State.Running}}", containerName], + CancellationToken.None); + return exitCode == 0 && string.Equals(output.Trim(), "true", StringComparison.OrdinalIgnoreCase); + } + catch + { + return true; + } + } + + private async Task LogContainerOutputAsync(string containerName) + { + try + { + var (_, output) = await RunDockerWithOutputAsync( + ["logs", "--tail", "100", containerName], + CancellationToken.None); + + if (!string.IsNullOrWhiteSpace(output)) + { + _logger.LogError( + "llama.cpp container {ContainerName} exited before becoming ready. Last output:\n{Output}", + containerName, output); + } + else + { + _logger.LogError( + "llama.cpp container {ContainerName} exited before becoming ready, but produced no output.", + containerName); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to read logs of container {ContainerName}", containerName); + } } private static string SanitizeContainerName(string name) @@ -655,6 +838,13 @@ internal sealed partial class LlamaCppManager : IAsyncDisposable [GeneratedRegex(@"[^a-zA-Z0-9_.-]")] private static partial Regex InvalidContainerNameChars(); + + private enum ContainerStartResult + { + Ready, + ContainerExited, + TimedOut + } } internal sealed record LlamaCppModel diff --git a/src/Ngino.Client/Program.cs b/src/Ngino.Client/Program.cs index b9617f4..5c508a6 100644 --- a/src/Ngino.Client/Program.cs +++ b/src/Ngino.Client/Program.cs @@ -22,8 +22,15 @@ try Console.WriteLine($" ollama models path: {options.UseOllamaModelsPath ?? "(not set)"}"); Console.WriteLine($" llama.cpp docker image: {options.LlamaCppDockerImage ?? "(auto)"}"); Console.WriteLine($" llama.cpp base port: {options.LlamaCppBasePort}"); + Console.WriteLine( + options.LlamaCppParallel.HasValue + ? $" llama.cpp parallel slots: {options.LlamaCppParallel.Value}" + : " llama.cpp parallel slots: (llama.cpp default)"); } + var logDirectory = options.LogDirectory ?? Path.Combine(AppContext.BaseDirectory, "Logs"); + Console.WriteLine($" log directory: {logDirectory}"); + // Args are parsed by ClientOptions; keep them away from the host configuration. var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings { Args = [] }); builder.Services.AddSingleton(options); @@ -32,6 +39,7 @@ try builder.Services.AddWindowsService(service => service.ServiceName = "NginoClient"); // The EventLog provider defaults to Warning; connection state is worth seeing there. builder.Logging.AddFilter("Ngino.Client", LogLevel.Information); + builder.Logging.AddProvider(new FileLoggerProvider(logDirectory)); await builder.Build().RunAsync(); return 0; diff --git a/src/Ngino.Client/TunnelClient.cs b/src/Ngino.Client/TunnelClient.cs index 498401d..9f49667 100644 --- a/src/Ngino.Client/TunnelClient.cs +++ b/src/Ngino.Client/TunnelClient.cs @@ -47,7 +47,9 @@ internal sealed class TunnelClient _options.UseOllamaModelsPath, _options.LlamaCppDockerImage, _options.LlamaCppBasePort, - _logger); + _logger, + _options.LlamaCppFallbackCooldown, + _options.LlamaCppParallel); } } @@ -477,13 +479,16 @@ internal sealed class TunnelClient var started = await _llamaCppManager.StartModelContainerAsync(model, cancellationToken); if (!started) { - return new TunnelMessage - { - Type = TunnelMessageTypes.ModelCommandResult, - RequestId = message.RequestId, - StatusCode = 500, - Error = $"Unable to load model '{modelName}'." - }; + _logger.LogWarning( + "Unable to load model '{Model}' via llama.cpp. Falling back to Ollama upstream.", + modelName); + + using var fallbackRequest = BuildModelCommandRequest(_options.Upstream, "load", modelName); + using var fallbackResponse = await _httpClient.SendAsync( + fallbackRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + var fallbackBody = await fallbackResponse.Content.ReadAsByteArrayAsync(cancellationToken); + + return BuildModelCommandResult(message.RequestId, fallbackResponse, fallbackBody); } return BuildModelCommandResult(message.RequestId, 200, "OK", []); @@ -494,13 +499,18 @@ internal sealed class TunnelClient var stopped = await _llamaCppManager!.StopModelContainerAsync(modelName!, cancellationToken); if (!stopped) { - return new TunnelMessage - { - Type = TunnelMessageTypes.ModelCommandResult, - RequestId = message.RequestId, - StatusCode = 404, - Error = $"No running llama.cpp container for model '{modelName}'." - }; + _logger.LogWarning( + "No running llama.cpp container for model '{Model}'. Falling back to Ollama upstream to unload it.", + modelName); + + _llamaCppManager.ClearModelFallback(modelName!); + + using var fallbackRequest = BuildModelCommandRequest(_options.Upstream, "unload", modelName); + using var fallbackResponse = await _httpClient.SendAsync( + fallbackRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + var fallbackBody = await fallbackResponse.Content.ReadAsByteArrayAsync(cancellationToken); + + return BuildModelCommandResult(message.RequestId, fallbackResponse, fallbackBody); } return BuildModelCommandResult(message.RequestId, 200, "OK", []); @@ -713,19 +723,29 @@ internal sealed class TunnelClient if (model is not null) { - _logger.LogInformation( - "Request for model '{Model}' but no llama.cpp container is running. Starting one on demand...", - modelName); - - var started = await _llamaCppManager.StartModelContainerAsync(model, cancellationToken); - if (started) + if (_llamaCppManager.IsModelOnFallback(modelName)) { - effectiveUpstream = _llamaCppManager.GetUpstream(modelName); + _logger.LogInformation( + "Model '{Model}' previously failed to load via llama.cpp. Routing directly to the Ollama upstream.", + modelName); } else { - await SendModelLoadErrorAsync(socket, message, modelName, cancellationToken); - return; + _logger.LogInformation( + "Request for model '{Model}' but no llama.cpp container is running. Starting one on demand...", + modelName); + + var started = await _llamaCppManager.StartModelContainerAsync(model, cancellationToken); + if (started) + { + effectiveUpstream = _llamaCppManager.GetUpstream(modelName); + } + else + { + _logger.LogWarning( + "Unable to load model '{Model}' via llama.cpp. Falling back to the Ollama upstream.", + modelName); + } } } else @@ -830,50 +850,6 @@ internal sealed class TunnelClient } } - private async Task SendModelLoadErrorAsync( - ClientWebSocket socket, TunnelMessage message, string modelName, CancellationToken cancellationToken) - { - _logger.LogWarning( - "Unable to load model '{Model}' via llama.cpp. Notifying caller.", modelName); - - var body = JsonSerializer.SerializeToUtf8Bytes( - new { error = $"Unable to load model '{modelName}'" }, - JsonOptions); - - _pendingRequestBodies.TryRemove(message.RequestId, out _); - - await SendAsync( - socket, - new TunnelMessage - { - Type = TunnelMessageTypes.HttpResponseHeaders, - RequestId = message.RequestId, - StatusCode = 500, - ReasonPhrase = "Internal Server Error", - Headers = [new HeaderPair("Content-Type", "application/json")] - }, - CancellationToken.None); - - await SendAsync( - socket, - new TunnelMessage - { - Type = TunnelMessageTypes.HttpResponseBody, - RequestId = message.RequestId, - Body = body - }, - CancellationToken.None); - - await SendAsync( - socket, - new TunnelMessage - { - Type = TunnelMessageTypes.HttpResponseComplete, - RequestId = message.RequestId - }, - CancellationToken.None); - } - private void CancelAllActiveRequests() { foreach (var pair in _activeRequests.ToArray())