Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
24fd8562cf | ||
|
|
c1c2d08915 | ||
|
|
2f1a3687f4 | ||
|
|
02c0b4243c | ||
|
|
c54c897703 | ||
|
|
9176807135 | ||
|
|
d61592c239 | ||
|
|
e3a538a1fb | ||
|
|
a787c54217 | ||
|
|
f1e60ecb75 | ||
|
|
2357957eb1 | ||
|
|
477ef3e548 | ||
|
|
a01bbd368a | ||
|
|
6b18cbca0e | ||
|
|
19c5b9bce1 | ||
|
|
d69769fb28 | ||
|
|
642ad1d9e4 | ||
|
|
8eeb65b6e6 | ||
|
|
4a84ff7316 |
+2
-1
@@ -8,4 +8,5 @@
|
||||
*.sqlite-wal
|
||||
.vs/
|
||||
**appsettings.Development.json
|
||||
!**/appsettings.Example.json
|
||||
!**/appsettings.Example.json
|
||||
debug/**
|
||||
@@ -1,10 +1,43 @@
|
||||
# ReverseLlama
|
||||
|
||||
(Eigenentwicklung; weitesgehend vibecoded)
|
||||
ReverseLlama is a small outbound HTTP tunnel for running Ollama (or vLLM, etc.) on GPU workstations while exposing the API from a server that cannot reach those workstations directly.
|
||||
|
||||
ReverseLlama is a small outbound HTTP tunnel for testing Ollama on GPU workstations while exposing the API from a server that cannot reach those workstations directly.
|
||||
The client opens and maintains a WebSocket connection to the server. The server accepts normal HTTP requests and forwards them through that WebSocket to the client. The client then calls a local upstream such as `http://localhost:11434` and streams the response back.
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td><img src="docs/images/README_architecture.png" width="1400" alt="client-server architecture visualized using arrows"></td>
|
||||
<td>
|
||||
|
||||
|
||||
|
||||
The server provides
|
||||
- An API with
|
||||
- Authentication via API keys
|
||||
- Authorization (planned)
|
||||
- Load balancing (Scale your AI strategy horizontally!)
|
||||
- (Ollama-only) Model management (install, remove, load, unload models)
|
||||
- Client monitoring
|
||||
- Who is active
|
||||
- What models are running
|
||||
- How many requests is each client processing
|
||||
- How many requests has each client processed
|
||||
- Group management (planned)
|
||||
- Who can access which models
|
||||
- What clients are mapped to which groups
|
||||
- Billing (planned)
|
||||
- (planned) Price per model per thousand tokens
|
||||
- (planned) Usage per API user
|
||||
- (planned) Rate limiting
|
||||
|
||||
The client provides a persistent outbound connection to the server and forwards requests to the local Ollama (or vLLM, etc.) instance. Responses stream back through the tunnel with minimal overhead.
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
[Screenshots of the app can be seen here](docs/Screenshots.md)
|
||||
|
||||
The client opens a WebSocket connection to the server. The server accepts normal HTTP requests and forwards them through that WebSocket to the client. The client then calls a local upstream such as `http://localhost:11434` and streams the response back.
|
||||
|
||||
## Projects
|
||||
|
||||
|
||||
+39
-14
@@ -90,6 +90,30 @@ if [[ $EUID -ne 0 ]]; then
|
||||
die "This script must be run as root (or with sudo)."
|
||||
fi
|
||||
|
||||
# ── Find dotnet binary ──────────────────────────────────────────────────────
|
||||
find_dotnet() {
|
||||
local candidates=(
|
||||
"$(command -v dotnet 2>/dev/null || true)"
|
||||
/usr/share/dotnet/dotnet
|
||||
/usr/bin/dotnet
|
||||
/usr/local/bin/dotnet
|
||||
/usr/local/share/dotnet/dotnet
|
||||
"$HOME/.dotnet/dotnet"
|
||||
)
|
||||
if [[ -n "${SUDO_USER:-}" ]]; then
|
||||
local invoke_home
|
||||
invoke_home="$(eval echo "~$SUDO_USER")"
|
||||
candidates+=("$invoke_home/.dotnet/dotnet")
|
||||
fi
|
||||
for c in "${candidates[@]}"; do
|
||||
if [[ -n "$c" && -x "$c" ]]; then
|
||||
echo "$c"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Check / install .NET 10 SDK ──────────────────────────────────────────────
|
||||
install_dotnet() {
|
||||
info "Installing .NET 10 SDK..."
|
||||
@@ -120,7 +144,7 @@ install_dotnet() {
|
||||
;;
|
||||
esac
|
||||
else
|
||||
die "Unsupported distro. Install .NET 10 SDK manually: https://dotnet.microsoft.com/download"
|
||||
die "Unsupported distro. Install .NET 10 SDK manually: https://dotnet.microsoft.com/download/dotnet/10.0"
|
||||
fi
|
||||
|
||||
# Add Microsoft package repository GPG key and repo
|
||||
@@ -149,19 +173,13 @@ REPO
|
||||
$PKG_MGR install -y dotnet-sdk-10.0
|
||||
|
||||
elif command -v pacman &>/dev/null; then
|
||||
warn "Arch Linux detected. .NET 10 may need to be installed from the AUR or manually."
|
||||
warn "See: https://dotnet.microsoft.com/download"
|
||||
die "Cannot auto-install .NET SDK on Arch. Please install dotnet-sdk-10 manually."
|
||||
die "Arch Linux: install dotnet-sdk from the AUR or manually: https://dotnet.microsoft.com/download/dotnet/10.0"
|
||||
|
||||
else
|
||||
die "Unsupported package manager. Install .NET 10 SDK manually: https://dotnet.microsoft.com/download"
|
||||
die "Unsupported package manager. Install .NET 10 SDK manually: https://dotnet.microsoft.com/download/dotnet/10.0"
|
||||
fi
|
||||
|
||||
DOTNET_CMD="$(command -v dotnet 2>/dev/null || echo /usr/share/dotnet/dotnet)"
|
||||
if [[ ! -x "$DOTNET_CMD" ]]; then
|
||||
DOTNET_CMD="/usr/bin/dotnet"
|
||||
fi
|
||||
if [[ ! -x "$DOTNET_CMD" ]]; then
|
||||
if ! DOTNET_CMD="$(find_dotnet)"; then
|
||||
die ".NET SDK installation succeeded but dotnet binary not found. Please add it to PATH."
|
||||
fi
|
||||
|
||||
@@ -170,8 +188,7 @@ REPO
|
||||
}
|
||||
|
||||
DOTNET_CMD=""
|
||||
if command -v dotnet &>/dev/null; then
|
||||
DOTNET_CMD="$(command -v dotnet)"
|
||||
if DOTNET_CMD="$(find_dotnet)"; then
|
||||
DOTNET_VER="$("$DOTNET_CMD" --version 2>/dev/null || true)"
|
||||
if [[ "$DOTNET_VER" == 10.* ]]; then
|
||||
info "dotnet 10 is already installed: $DOTNET_CMD ($DOTNET_VER)"
|
||||
@@ -213,13 +230,21 @@ if [[ ! -d "$CLIENT_SRC" ]]; then
|
||||
die "Client source not found at $CLIENT_SRC. Run this script from the repository or pass --install-dir."
|
||||
fi
|
||||
|
||||
info "Building ReverseLlama client (self-contained, linux-x64)..."
|
||||
ARCH="$(uname -m)"
|
||||
case "$ARCH" in
|
||||
x86_64) DOTNET_RID="linux-x64" ;;
|
||||
aarch64) DOTNET_RID="linux-arm64" ;;
|
||||
armv7l) DOTNET_RID="linux-arm" ;;
|
||||
*) die "Unsupported architecture: $ARCH" ;;
|
||||
esac
|
||||
|
||||
info "Building ReverseLlama client (self-contained, $DOTNET_RID)..."
|
||||
BUILD_DIR="$(mktemp -d /tmp/reversellama-build.XXXXXX)"
|
||||
trap 'rm -rf "$BUILD_DIR"' EXIT
|
||||
|
||||
"$DOTNET_CMD" publish "$CLIENT_SRC/ReverseLlama.Client.csproj" \
|
||||
-c Release \
|
||||
-r linux-x64 \
|
||||
-r "$DOTNET_RID" \
|
||||
--self-contained true \
|
||||
-o "$BUILD_DIR"
|
||||
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Clients view
|
||||

|
||||
# Models view
|
||||

|
||||
# API Keys view
|
||||

|
||||
Binary file not shown.
|
After Width: | Height: | Size: 147 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 105 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 159 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 164 KiB |
@@ -73,7 +73,7 @@ internal static class AdminEndpoints
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return Results.Problem(exception.Message, statusCode: StatusCodes.Status400BadRequest);
|
||||
return Results.BadRequest(new { error = exception.Message });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -86,7 +86,7 @@ internal static class AdminEndpoints
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return Results.Problem(exception.Message, statusCode: StatusCodes.Status400BadRequest);
|
||||
return Results.BadRequest(new { error = exception.Message });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -176,7 +176,7 @@ internal static class AdminEndpoints
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return Results.Problem(exception.Message, statusCode: StatusCodes.Status400BadRequest);
|
||||
return Results.BadRequest(new { error = exception.Message });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -185,6 +185,261 @@ internal static class AdminEndpoints
|
||||
? Results.NoContent()
|
||||
: Results.NotFound(new { error = $"API key '{id}' was not found." }));
|
||||
|
||||
api.MapGet("/groups", (ManagementStore store) =>
|
||||
Results.Json(store.ListGroups()));
|
||||
|
||||
api.MapPost("/groups", (CreateGroupRequest request, ManagementStore store) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
return Results.Json(store.CreateGroup(request.Name));
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return Results.BadRequest(new { error = exception.Message });
|
||||
}
|
||||
});
|
||||
|
||||
api.MapGet("/groups/{id}", (string id, ManagementStore store) =>
|
||||
{
|
||||
var group = store.GetGroup(id);
|
||||
return group is not null
|
||||
? Results.Json(group)
|
||||
: Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
});
|
||||
|
||||
api.MapPut("/groups/{id}", (string id, UpdateGroupRequest request, ManagementStore store) =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request.Name))
|
||||
{
|
||||
return Results.BadRequest(new { error = "Name is required." });
|
||||
}
|
||||
|
||||
return store.UpdateGroup(id, request.Name)
|
||||
? Results.Ok(store.GetGroup(id))
|
||||
: Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
});
|
||||
|
||||
api.MapDelete("/groups/{id}", (string id, ManagementStore store) =>
|
||||
store.DeleteGroup(id)
|
||||
? Results.NoContent()
|
||||
: Results.NotFound(new { error = $"Group '{id}' was not found." }));
|
||||
|
||||
api.MapGet("/groups/{id}/clients", (string id, ManagementStore store) =>
|
||||
{
|
||||
var group = store.GetGroup(id);
|
||||
if (group is null)
|
||||
{
|
||||
return Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
}
|
||||
|
||||
return Results.Json(store.ListGroupClients(id));
|
||||
});
|
||||
|
||||
api.MapPost("/groups/{id}/clients", (string id, AddGroupClientRequest request, ManagementStore store) =>
|
||||
{
|
||||
var group = store.GetGroup(id);
|
||||
if (group is null)
|
||||
{
|
||||
return Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var member = store.AddGroupClient(id, request.ClientId, request.Model, request.ClientPattern);
|
||||
return Results.Json(member);
|
||||
}
|
||||
catch (ArgumentException exception)
|
||||
{
|
||||
return Results.BadRequest(new { error = exception.Message });
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return Results.BadRequest(new { error = $"Failed to add member: {exception.Message}" });
|
||||
}
|
||||
});
|
||||
|
||||
api.MapDelete("/groups/{groupId}/clients/{clientId:long}", (string groupId, long clientId, ManagementStore store) =>
|
||||
{
|
||||
var group = store.GetGroup(groupId);
|
||||
if (group is null)
|
||||
{
|
||||
return Results.NotFound(new { error = $"Group '{groupId}' was not found." });
|
||||
}
|
||||
|
||||
return store.RemoveGroupClient(clientId)
|
||||
? Results.NoContent()
|
||||
: Results.NotFound(new { error = $"Client '{clientId}' was not found." });
|
||||
});
|
||||
|
||||
api.MapGet("/api-keys/groups", (ManagementStore store) =>
|
||||
Results.Json(store.ListApiKeyGroups()));
|
||||
|
||||
api.MapPut("/api-keys/{id}/groups", (string id, SetApiKeyGroupsRequest request, ManagementStore store) =>
|
||||
{
|
||||
var keys = store.ListApiKeys();
|
||||
if (!keys.Any(k => k.Id == id))
|
||||
{
|
||||
return Results.NotFound(new { error = $"API key '{id}' was not found." });
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
store.SetApiKeyGroups(id, request.GroupIds ?? []);
|
||||
return Results.Ok(new { apiKeyId = id, groupIds = store.GetApiKeyGroupIds(id) });
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return Results.BadRequest(new { error = exception.Message });
|
||||
}
|
||||
});
|
||||
|
||||
api.MapGet("/groups/{id}/billing", (string id, ManagementStore store) =>
|
||||
{
|
||||
var group = store.GetGroup(id);
|
||||
if (group is null)
|
||||
{
|
||||
return Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
}
|
||||
|
||||
var billing = store.GetGroupBilling(id);
|
||||
return billing is not null
|
||||
? Results.Json(billing)
|
||||
: Results.Json(new GroupBillingInfo(id, "EUR", 0, 0, false, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow));
|
||||
});
|
||||
|
||||
api.MapPut("/groups/{id}/billing", (string id, UpdateBillingRequest request, ManagementStore store) =>
|
||||
{
|
||||
var group = store.GetGroup(id);
|
||||
if (group is null)
|
||||
{
|
||||
return Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var billing = store.UpsertGroupBilling(
|
||||
id,
|
||||
request.Currency ?? "EUR",
|
||||
request.DefaultRatePer1k,
|
||||
request.RefuseBelowBalance,
|
||||
request.Enabled);
|
||||
return Results.Ok(billing);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return Results.BadRequest(new { error = exception.Message });
|
||||
}
|
||||
});
|
||||
|
||||
api.MapGet("/groups/{id}/billing/rules", (string id, ManagementStore store) =>
|
||||
{
|
||||
var group = store.GetGroup(id);
|
||||
if (group is null)
|
||||
{
|
||||
return Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
}
|
||||
|
||||
return Results.Json(store.ListGroupBillingRules(id));
|
||||
});
|
||||
|
||||
api.MapPost("/groups/{id}/billing/rules", (string id, AddBillingRuleRequest request, ManagementStore store) =>
|
||||
{
|
||||
var group = store.GetGroup(id);
|
||||
if (group is null)
|
||||
{
|
||||
return Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var rule = store.AddBillingRule(id, request.ModelRegex, request.RatePer1k);
|
||||
return Results.Json(rule);
|
||||
}
|
||||
catch (ArgumentException exception)
|
||||
{
|
||||
return Results.BadRequest(new { error = exception.Message });
|
||||
}
|
||||
});
|
||||
|
||||
api.MapPut("/groups/{id}/billing/rules/{ruleId:long}", (string id, long ruleId, UpdateBillingRuleRequest request, ManagementStore store) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
return store.UpdateBillingRule(ruleId, request.ModelRegex, request.RatePer1k)
|
||||
? Results.Ok(new { id = ruleId })
|
||||
: Results.NotFound(new { error = $"Rule '{ruleId}' was not found." });
|
||||
}
|
||||
catch (ArgumentException exception)
|
||||
{
|
||||
return Results.BadRequest(new { error = exception.Message });
|
||||
}
|
||||
});
|
||||
|
||||
api.MapDelete("/groups/{id}/billing/rules/{ruleId:long}", (string id, long ruleId, ManagementStore store) =>
|
||||
store.DeleteBillingRule(ruleId)
|
||||
? Results.NoContent()
|
||||
: Results.NotFound(new { error = $"Rule '{ruleId}' was not found." }));
|
||||
|
||||
api.MapGet("/groups/{id}/billing/payments", (string id, ManagementStore store) =>
|
||||
{
|
||||
var group = store.GetGroup(id);
|
||||
if (group is null)
|
||||
{
|
||||
return Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
}
|
||||
|
||||
return Results.Json(store.ListGroupPayments(id));
|
||||
});
|
||||
|
||||
api.MapPost("/groups/{id}/billing/payments", (string id, AddPaymentRequest request, ManagementStore store, HttpContext context) =>
|
||||
{
|
||||
var group = store.GetGroup(id);
|
||||
if (group is null)
|
||||
{
|
||||
return Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var userName = GetUserName(context.User);
|
||||
var payment = store.AddPayment(id, request.Amount, request.Description, userName);
|
||||
return Results.Json(payment);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return Results.BadRequest(new { error = exception.Message });
|
||||
}
|
||||
});
|
||||
|
||||
api.MapDelete("/groups/{id}/billing/payments/{paymentId:long}", (string id, long paymentId, ManagementStore store) =>
|
||||
store.DeletePayment(paymentId)
|
||||
? Results.NoContent()
|
||||
: Results.NotFound(new { error = $"Payment '{paymentId}' was not found." }));
|
||||
|
||||
api.MapGet("/groups/{id}/billing/balance", (string id, ManagementStore store) =>
|
||||
{
|
||||
var group = store.GetGroup(id);
|
||||
if (group is null)
|
||||
{
|
||||
return Results.NotFound(new { error = $"Group '{id}' was not found." });
|
||||
}
|
||||
|
||||
return Results.Json(store.GetGroupBalance(id));
|
||||
});
|
||||
|
||||
api.MapGet("/usage/tokens", (ManagementStore store) =>
|
||||
Results.Json(new
|
||||
{
|
||||
byModel = store.GetTokenStatsByModel(),
|
||||
byClient = store.GetTokenStatsByClient(),
|
||||
byApiKey = store.GetTokenStatsByApiKey(),
|
||||
byGroup = store.GetTokenStatsByGroup()
|
||||
}));
|
||||
|
||||
api.MapGet("/usage/revenue", (ManagementStore store) =>
|
||||
Results.Json(store.GetClientRevenue()));
|
||||
|
||||
var adminHome = app.MapGet("/admin", (IWebHostEnvironment environment) =>
|
||||
ServeAdminAsset(environment, null));
|
||||
var adminAssets = app.MapGet("/admin/{**assetPath}", (IWebHostEnvironment environment, string? assetPath) =>
|
||||
@@ -224,7 +479,11 @@ internal static class AdminEndpoints
|
||||
},
|
||||
clients = BuildClientSummaries(hub, store),
|
||||
models = BuildModelSummaries(hub, store),
|
||||
apiKeys = store.ListApiKeys()
|
||||
apiKeys = store.ListApiKeys(),
|
||||
groups = store.ListGroups(),
|
||||
apiKeyGroups = store.ListApiKeyGroups(),
|
||||
clientGroups = store.ResolveClientGroups(
|
||||
hub.ClientSnapshots.Select(c => c.Id).ToList())
|
||||
};
|
||||
|
||||
private static IReadOnlyList<ClientSummary> BuildClientSummaries(TunnelHub hub, ManagementStore store)
|
||||
@@ -470,6 +729,35 @@ internal sealed record ModelActionRequest(
|
||||
|
||||
internal sealed record CreateApiKeyRequest(string? Name);
|
||||
|
||||
internal sealed record CreateGroupRequest(string? Name);
|
||||
|
||||
internal sealed record UpdateGroupRequest(string Name);
|
||||
|
||||
internal sealed record AddGroupClientRequest(
|
||||
string? ClientId,
|
||||
string? Model,
|
||||
string? ClientPattern);
|
||||
|
||||
internal sealed record SetApiKeyGroupsRequest(IReadOnlyList<string>? GroupIds);
|
||||
|
||||
internal sealed record UpdateBillingRequest(
|
||||
string? Currency,
|
||||
double DefaultRatePer1k,
|
||||
double RefuseBelowBalance,
|
||||
bool Enabled);
|
||||
|
||||
internal sealed record AddBillingRuleRequest(
|
||||
string ModelRegex,
|
||||
double RatePer1k);
|
||||
|
||||
internal sealed record UpdateBillingRuleRequest(
|
||||
string ModelRegex,
|
||||
double RatePer1k);
|
||||
|
||||
internal sealed record AddPaymentRequest(
|
||||
double Amount,
|
||||
string? Description);
|
||||
|
||||
internal sealed record ClientSummary(
|
||||
string Id,
|
||||
bool Connected,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -74,6 +74,10 @@ builder.Services.AddAuthorization();
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
var managementStore = app.Services.GetRequiredService<ManagementStore>();
|
||||
var tunnelHub = app.Services.GetRequiredService<TunnelHub>();
|
||||
managementStore.SetConnectedClientProvider(() => tunnelHub.ClientSnapshots.Select(c => c.Id));
|
||||
|
||||
if (settings.Keycloak.IsConfigured)
|
||||
{
|
||||
app.UseAuthentication();
|
||||
@@ -82,6 +86,21 @@ if (settings.Keycloak.IsConfigured)
|
||||
|
||||
app.UseElmah();
|
||||
|
||||
app.Use(async (context, next) =>
|
||||
{
|
||||
context.Response.Headers.AccessControlAllowOrigin = "*";
|
||||
context.Response.Headers.AccessControlAllowMethods = "GET, POST, PUT, DELETE, PATCH, OPTIONS";
|
||||
context.Response.Headers.AccessControlAllowHeaders = "Content-Type, Authorization";
|
||||
|
||||
if (HttpMethods.IsOptions(context.Request.Method))
|
||||
{
|
||||
context.Response.StatusCode = StatusCodes.Status204NoContent;
|
||||
return;
|
||||
}
|
||||
|
||||
await next();
|
||||
});
|
||||
|
||||
app.UseWebSockets(new WebSocketOptions
|
||||
{
|
||||
KeepAliveInterval = TimeSpan.FromSeconds(30)
|
||||
@@ -89,14 +108,7 @@ app.UseWebSockets(new WebSocketOptions
|
||||
|
||||
app.MapAdminEndpoints(settings);
|
||||
|
||||
app.MapGet("/", (TunnelHub hub) =>
|
||||
Results.Json(new
|
||||
{
|
||||
status = "ok",
|
||||
connected = hub.HasClient,
|
||||
pendingRequests = hub.PendingRequestCount,
|
||||
clients = hub.ClientsSnapshot.Count
|
||||
}));
|
||||
app.MapGet("/", () => Results.Redirect("/admin"));
|
||||
|
||||
app.MapGet(settings.StatusPath, (HttpContext context, TunnelHub hub, ServerSettings serverSettings, EmbeddingCache embeddingCache, ManagementStore managementStore) =>
|
||||
{
|
||||
|
||||
@@ -21,15 +21,16 @@ internal sealed class ResponseTokenCounter
|
||||
_buffer.Write(chunk[..length]);
|
||||
}
|
||||
|
||||
public int CountTokens()
|
||||
public TokenCounts CountTokens()
|
||||
{
|
||||
if (_buffer.Length == 0)
|
||||
{
|
||||
return 0;
|
||||
return new TokenCounts(0, 0, 0);
|
||||
}
|
||||
|
||||
var payload = Encoding.UTF8.GetString(_buffer.ToArray());
|
||||
var total = 0;
|
||||
var totalPrompt = 0;
|
||||
var totalCompletion = 0;
|
||||
var parsedLines = false;
|
||||
|
||||
foreach (var rawLine in payload.Split('\n'))
|
||||
@@ -50,30 +51,37 @@ internal sealed class ResponseTokenCounter
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryExtractFromJson(line, out var lineTokens))
|
||||
if (TryExtractTokenCountsFromJson(line, out var prompt, out var completion))
|
||||
{
|
||||
parsedLines = true;
|
||||
total += lineTokens;
|
||||
totalPrompt += prompt;
|
||||
totalCompletion += completion;
|
||||
}
|
||||
}
|
||||
|
||||
if (parsedLines)
|
||||
{
|
||||
return total;
|
||||
return new TokenCounts(totalPrompt, totalCompletion, totalPrompt + totalCompletion);
|
||||
}
|
||||
|
||||
return TryExtractFromJson(payload, out var tokens) ? tokens : 0;
|
||||
if (TryExtractTokenCountsFromJson(payload, out var promptFallback, out var completionFallback))
|
||||
{
|
||||
return new TokenCounts(promptFallback, completionFallback, promptFallback + completionFallback);
|
||||
}
|
||||
|
||||
return new TokenCounts(0, 0, 0);
|
||||
}
|
||||
|
||||
private static bool TryExtractFromJson(string json, out int tokens)
|
||||
private static bool TryExtractTokenCountsFromJson(string json, out int promptTokens, out int completionTokens)
|
||||
{
|
||||
tokens = 0;
|
||||
promptTokens = 0;
|
||||
completionTokens = 0;
|
||||
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(json);
|
||||
tokens = ExtractTokens(document.RootElement);
|
||||
return tokens > 0;
|
||||
ExtractTokenCounts(document.RootElement, out promptTokens, out completionTokens);
|
||||
return promptTokens > 0 || completionTokens > 0;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
@@ -81,70 +89,68 @@ internal sealed class ResponseTokenCounter
|
||||
}
|
||||
}
|
||||
|
||||
private static int ExtractTokens(JsonElement element)
|
||||
private static void ExtractTokenCounts(JsonElement element, out int promptTokens, out int completionTokens)
|
||||
{
|
||||
promptTokens = 0;
|
||||
completionTokens = 0;
|
||||
|
||||
if (element.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
var total = 0;
|
||||
foreach (var item in element.EnumerateArray())
|
||||
{
|
||||
total += ExtractTokens(item);
|
||||
ExtractTokenCounts(item, out var itemPrompt, out var itemCompletion);
|
||||
promptTokens += itemPrompt;
|
||||
completionTokens += itemCompletion;
|
||||
}
|
||||
|
||||
return total;
|
||||
return;
|
||||
}
|
||||
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return 0;
|
||||
return;
|
||||
}
|
||||
|
||||
if (element.TryGetProperty("usage", out var usage) && usage.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
if (TryGetInt(usage, "total_tokens", out var totalTokens))
|
||||
if (TryGetInt(usage, "prompt_tokens", out var pt))
|
||||
{
|
||||
return totalTokens;
|
||||
promptTokens += pt;
|
||||
}
|
||||
|
||||
var usageTotal = 0;
|
||||
if (TryGetInt(usage, "prompt_tokens", out var promptTokens))
|
||||
if (TryGetInt(usage, "completion_tokens", out var ct))
|
||||
{
|
||||
usageTotal += promptTokens;
|
||||
completionTokens += ct;
|
||||
}
|
||||
|
||||
if (TryGetInt(usage, "completion_tokens", out var completionTokens))
|
||||
if (promptTokens == 0 && completionTokens == 0)
|
||||
{
|
||||
usageTotal += completionTokens;
|
||||
if (TryGetInt(usage, "input_tokens", out var it))
|
||||
{
|
||||
promptTokens += it;
|
||||
}
|
||||
|
||||
if (TryGetInt(usage, "output_tokens", out var ot))
|
||||
{
|
||||
completionTokens += ot;
|
||||
}
|
||||
}
|
||||
|
||||
if (TryGetInt(usage, "input_tokens", out var inputTokens))
|
||||
if (promptTokens > 0 || completionTokens > 0)
|
||||
{
|
||||
usageTotal += inputTokens;
|
||||
}
|
||||
|
||||
if (TryGetInt(usage, "output_tokens", out var outputTokens))
|
||||
{
|
||||
usageTotal += outputTokens;
|
||||
}
|
||||
|
||||
if (usageTotal > 0)
|
||||
{
|
||||
return usageTotal;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
var ollamaTotal = 0;
|
||||
if (TryGetInt(element, "prompt_eval_count", out var promptEvalCount))
|
||||
if (TryGetInt(element, "prompt_eval_count", out var promptEval))
|
||||
{
|
||||
ollamaTotal += promptEvalCount;
|
||||
promptTokens += promptEval;
|
||||
}
|
||||
|
||||
if (TryGetInt(element, "eval_count", out var evalCount))
|
||||
if (TryGetInt(element, "eval_count", out var eval))
|
||||
{
|
||||
ollamaTotal += evalCount;
|
||||
completionTokens += eval;
|
||||
}
|
||||
|
||||
return ollamaTotal;
|
||||
}
|
||||
|
||||
private static bool TryGetInt(JsonElement element, string propertyName, out int value)
|
||||
@@ -155,3 +161,5 @@ internal sealed class ResponseTokenCounter
|
||||
&& property.TryGetInt32(out value);
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record TokenCounts(int PromptTokens, int CompletionTokens, int TotalTokens);
|
||||
|
||||
@@ -35,13 +35,30 @@ internal static class ReverseProxyEndpoint
|
||||
EmbeddingCache embeddingCache,
|
||||
ManagementStore managementStore)
|
||||
{
|
||||
if (!TokenAuthentication.IsAuthorized(context.Request, settings, managementStore, allowQueryToken: false, allowPathToken: true))
|
||||
var auth = TokenAuthentication.Authorize(context.Request, settings, managementStore, allowQueryToken: false, allowPathToken: true);
|
||||
if (!auth.IsAuthorized)
|
||||
{
|
||||
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
|
||||
await context.Response.WriteAsync(UnauthorizedMessage, context.RequestAborted);
|
||||
return;
|
||||
}
|
||||
|
||||
var billingCheck = managementStore.CheckBalanceForApiKey(auth.ApiKeyId);
|
||||
if (!billingCheck.Allowed)
|
||||
{
|
||||
context.Response.StatusCode = StatusCodes.Status402PaymentRequired;
|
||||
await context.Response.WriteAsJsonAsync(new
|
||||
{
|
||||
error = "Insufficient balance.",
|
||||
balance = billingCheck.Balance,
|
||||
currency = billingCheck.Currency,
|
||||
threshold = billingCheck.Threshold
|
||||
}, context.RequestAborted);
|
||||
return;
|
||||
}
|
||||
|
||||
var groupAccess = ResolveGroupAccess(auth.ApiKeyId, managementStore);
|
||||
|
||||
var pathTokenRemoved = TokenAuthentication.TryRemovePathToken(context.Request.Path, settings, managementStore, out var proxyPath);
|
||||
if (!pathTokenRemoved)
|
||||
{
|
||||
@@ -56,6 +73,13 @@ internal static class ReverseProxyEndpoint
|
||||
|
||||
if (TryGetClientAddress(proxyPath, out var pathClientId, out var clientPath))
|
||||
{
|
||||
if (!groupAccess.IsClientAllowed(pathClientId))
|
||||
{
|
||||
context.Response.StatusCode = StatusCodes.Status403Forbidden;
|
||||
await context.Response.WriteAsync($"Access to client '{pathClientId}' is not permitted.", context.RequestAborted);
|
||||
return;
|
||||
}
|
||||
|
||||
await ForwardToClientAsync(
|
||||
context,
|
||||
pathClientId,
|
||||
@@ -65,7 +89,15 @@ internal static class ReverseProxyEndpoint
|
||||
settings,
|
||||
loggerFactory,
|
||||
embeddingCache,
|
||||
managementStore);
|
||||
managementStore,
|
||||
groupAccess,
|
||||
auth.ApiKeyId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (IsTagsRequest(context.Request, proxyPath))
|
||||
{
|
||||
await HandleTagsAsync(context, hub, managementStore, groupAccess);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -79,7 +111,10 @@ internal static class ReverseProxyEndpoint
|
||||
var requestedModel = embeddingRequest?.Model ?? await GetRequestedModelAsync(context.Request, proxyPath);
|
||||
var connection = hub.SelectBest(
|
||||
requestedModel,
|
||||
clientId => !managementStore.GetClientAccess(clientId).IsDisabled);
|
||||
clientId => !managementStore.GetClientAccess(clientId).IsDisabled
|
||||
&& (requestedModel is null
|
||||
? groupAccess.IsClientAllowed(clientId)
|
||||
: groupAccess.IsClientModelAllowed(clientId, requestedModel)));
|
||||
if (connection is null)
|
||||
{
|
||||
if (!hub.HasClient)
|
||||
@@ -104,7 +139,8 @@ internal static class ReverseProxyEndpoint
|
||||
loggerFactory,
|
||||
embeddingCache,
|
||||
embeddingRequest,
|
||||
managementStore);
|
||||
managementStore,
|
||||
auth.ApiKeyId);
|
||||
}
|
||||
|
||||
public static async Task HandleClientAsync(
|
||||
@@ -117,13 +153,36 @@ internal static class ReverseProxyEndpoint
|
||||
EmbeddingCache embeddingCache,
|
||||
ManagementStore managementStore)
|
||||
{
|
||||
if (!TokenAuthentication.IsAuthorized(context.Request, settings, managementStore, allowQueryToken: false, allowPathToken: true))
|
||||
var auth = TokenAuthentication.Authorize(context.Request, settings, managementStore, allowQueryToken: false, allowPathToken: true);
|
||||
if (!auth.IsAuthorized)
|
||||
{
|
||||
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
|
||||
await context.Response.WriteAsync(UnauthorizedMessage, context.RequestAborted);
|
||||
return;
|
||||
}
|
||||
|
||||
var billingCheck = managementStore.CheckBalanceForApiKey(auth.ApiKeyId);
|
||||
if (!billingCheck.Allowed)
|
||||
{
|
||||
context.Response.StatusCode = StatusCodes.Status402PaymentRequired;
|
||||
await context.Response.WriteAsJsonAsync(new
|
||||
{
|
||||
error = "Insufficient balance.",
|
||||
balance = billingCheck.Balance,
|
||||
currency = billingCheck.Currency,
|
||||
threshold = billingCheck.Threshold
|
||||
}, context.RequestAborted);
|
||||
return;
|
||||
}
|
||||
|
||||
var groupAccess = ResolveGroupAccess(auth.ApiKeyId, managementStore);
|
||||
if (!groupAccess.IsClientAllowed(clientId))
|
||||
{
|
||||
context.Response.StatusCode = StatusCodes.Status403Forbidden;
|
||||
await context.Response.WriteAsync($"Access to client '{clientId}' is not permitted.", context.RequestAborted);
|
||||
return;
|
||||
}
|
||||
|
||||
var pathAndQuery = $"/{path}{context.Request.QueryString}";
|
||||
var clientPath = new PathString($"/{path}");
|
||||
await ForwardToClientAsync(
|
||||
@@ -135,7 +194,9 @@ internal static class ReverseProxyEndpoint
|
||||
settings,
|
||||
loggerFactory,
|
||||
embeddingCache,
|
||||
managementStore);
|
||||
managementStore,
|
||||
groupAccess,
|
||||
auth.ApiKeyId);
|
||||
}
|
||||
|
||||
private static async Task ForwardToClientAsync(
|
||||
@@ -147,7 +208,9 @@ internal static class ReverseProxyEndpoint
|
||||
ServerSettings settings,
|
||||
ILoggerFactory loggerFactory,
|
||||
EmbeddingCache embeddingCache,
|
||||
ManagementStore managementStore)
|
||||
ManagementStore managementStore,
|
||||
GroupAccess? groupAccess = null,
|
||||
string? apiKeyId = null)
|
||||
{
|
||||
var clientAccess = managementStore.GetClientAccess(clientId);
|
||||
if (clientAccess.IsDisabled)
|
||||
@@ -173,6 +236,13 @@ internal static class ReverseProxyEndpoint
|
||||
}
|
||||
|
||||
var requestedModel = embeddingRequest?.Model ?? await GetRequestedModelAsync(context.Request, clientPath);
|
||||
if (requestedModel is not null && groupAccess is not null && !groupAccess.IsClientModelAllowed(clientId, requestedModel))
|
||||
{
|
||||
context.Response.StatusCode = StatusCodes.Status403Forbidden;
|
||||
await context.Response.WriteAsync($"Access to model '{requestedModel}' on client '{clientId}' is not permitted.", context.RequestAborted);
|
||||
return;
|
||||
}
|
||||
|
||||
await ForwardAsync(
|
||||
context,
|
||||
connection,
|
||||
@@ -182,7 +252,8 @@ internal static class ReverseProxyEndpoint
|
||||
loggerFactory,
|
||||
embeddingCache,
|
||||
embeddingRequest,
|
||||
managementStore);
|
||||
managementStore,
|
||||
apiKeyId);
|
||||
}
|
||||
|
||||
private static bool IsRootPath(PathString path) =>
|
||||
@@ -231,6 +302,69 @@ internal static class ReverseProxyEndpoint
|
||||
return true;
|
||||
}
|
||||
|
||||
private static GroupAccess ResolveGroupAccess(string? apiKeyId, ManagementStore managementStore)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(apiKeyId))
|
||||
{
|
||||
return GroupAccess.Unrestricted;
|
||||
}
|
||||
|
||||
return managementStore.ResolveGroupAccess(apiKeyId);
|
||||
}
|
||||
|
||||
private static bool IsTagsRequest(HttpRequest request, PathString proxyPath) =>
|
||||
HttpMethods.IsGet(request.Method)
|
||||
&& string.Equals(proxyPath.Value, "/api/tags", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static async Task HandleTagsAsync(
|
||||
HttpContext context,
|
||||
TunnelHub hub,
|
||||
ManagementStore managementStore,
|
||||
GroupAccess groupAccess)
|
||||
{
|
||||
var models = new Dictionary<string, SortedSet<string>>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var client in hub.ClientSnapshots)
|
||||
{
|
||||
foreach (var model in client.Models)
|
||||
{
|
||||
if (!models.TryGetValue(model, out var clients))
|
||||
{
|
||||
clients = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
models[model] = clients;
|
||||
}
|
||||
|
||||
clients.Add(client.Id);
|
||||
}
|
||||
}
|
||||
|
||||
var filteredModels = new List<object>();
|
||||
foreach (var (model, clients) in models.OrderBy(kvp => kvp.Key, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var accessibleClients = clients
|
||||
.Where(clientId => groupAccess.IsClientModelAllowed(clientId, model))
|
||||
.ToList();
|
||||
|
||||
if (accessibleClients.Count > 0)
|
||||
{
|
||||
filteredModels.Add(new
|
||||
{
|
||||
name = model,
|
||||
model,
|
||||
modified_at = DateTimeOffset.UtcNow,
|
||||
size = 0,
|
||||
digest = "",
|
||||
details = new { }
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var response = new { models = filteredModels };
|
||||
context.Response.StatusCode = StatusCodes.Status200OK;
|
||||
context.Response.ContentType = "application/json";
|
||||
await context.Response.WriteAsJsonAsync(response, context.RequestAborted);
|
||||
}
|
||||
|
||||
private static string GetNoRouteMessage(string? requestedModel) =>
|
||||
string.IsNullOrWhiteSpace(requestedModel)
|
||||
? "No tunnel client is available for this request."
|
||||
@@ -363,7 +497,8 @@ internal static class ReverseProxyEndpoint
|
||||
ILoggerFactory loggerFactory,
|
||||
EmbeddingCache embeddingCache,
|
||||
EmbeddingCacheRequest? embeddingRequest,
|
||||
ManagementStore managementStore)
|
||||
ManagementStore managementStore,
|
||||
string? apiKeyId = null)
|
||||
{
|
||||
var logger = loggerFactory.CreateLogger("ReverseLlama.Server.ReverseProxy");
|
||||
var requestId = Guid.NewGuid().ToString("n");
|
||||
@@ -448,13 +583,29 @@ internal static class ReverseProxyEndpoint
|
||||
{
|
||||
connection.RemovePending(requestId);
|
||||
var completedAt = DateTimeOffset.UtcNow;
|
||||
var tokenCounts = tokenCounter.CountTokens();
|
||||
|
||||
var cost = 0.0;
|
||||
if (!string.IsNullOrWhiteSpace(apiKeyId) && tokenCounts.TotalTokens > 0)
|
||||
{
|
||||
var billing = managementStore.ResolveBillingForApiKey(apiKeyId);
|
||||
if (billing is not null)
|
||||
{
|
||||
cost = managementStore.CalculateCost(billing.GroupId, requestedModel, tokenCounts.TotalTokens);
|
||||
}
|
||||
}
|
||||
|
||||
managementStore.RecordRequest(new RequestMetric(
|
||||
connection.ClientId,
|
||||
requestedModel,
|
||||
context.Request.Method,
|
||||
pathAndQuery,
|
||||
statusCode ?? (context.Response.HasStarted ? context.Response.StatusCode : null),
|
||||
tokenCounter.CountTokens(),
|
||||
tokenCounts.PromptTokens,
|
||||
tokenCounts.CompletionTokens,
|
||||
tokenCounts.TotalTokens,
|
||||
apiKeyId,
|
||||
cost,
|
||||
startedAt,
|
||||
completedAt,
|
||||
completedAt - startedAt));
|
||||
|
||||
@@ -6,7 +6,7 @@ internal static class TokenAuthentication
|
||||
{
|
||||
private static readonly PathString PathTokenPrefix = new("/token");
|
||||
|
||||
public static bool IsAuthorized(
|
||||
public static AuthResult Authorize(
|
||||
HttpRequest request,
|
||||
ServerSettings settings,
|
||||
ManagementStore managementStore,
|
||||
@@ -15,36 +15,70 @@ internal static class TokenAuthentication
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(settings.Token) && !managementStore.HasApiKeys)
|
||||
{
|
||||
return true;
|
||||
return AuthResult.Success(null);
|
||||
}
|
||||
|
||||
if (request.Headers.TryGetValue(ProtocolConstants.TokenHeader, out var headerValues)
|
||||
&& headerValues.Any(value => IsTokenAuthorized(value, settings, managementStore, updateApiKeyLastUsed: true)))
|
||||
if (request.Headers.TryGetValue(ProtocolConstants.TokenHeader, out var headerValues))
|
||||
{
|
||||
return true;
|
||||
foreach (var value in headerValues)
|
||||
{
|
||||
var result = AuthorizeToken(value, settings, managementStore, updateApiKeyLastUsed: true);
|
||||
if (result.IsAuthorized)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bearer form for OpenAI-compatible clients (e.g. n8n's OpenAI nodes pointed
|
||||
// at /clients/{id}/v1) that can send an API key but no custom headers.
|
||||
if (request.Headers.TryGetValue("Authorization", out var authorizationValues)
|
||||
&& authorizationValues.Any(value => TryGetBearerToken(value, out var bearerToken)
|
||||
&& IsTokenAuthorized(bearerToken, settings, managementStore, updateApiKeyLastUsed: true)))
|
||||
if (request.Headers.TryGetValue("Authorization", out var authorizationValues))
|
||||
{
|
||||
return true;
|
||||
foreach (var value in authorizationValues)
|
||||
{
|
||||
if (TryGetBearerToken(value, out var bearerToken))
|
||||
{
|
||||
var result = AuthorizeToken(bearerToken, settings, managementStore, updateApiKeyLastUsed: true);
|
||||
if (result.IsAuthorized)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (allowPathToken
|
||||
&& TryGetPathToken(request.Path, out var pathToken, out _)
|
||||
&& IsTokenAuthorized(pathToken, settings, managementStore, updateApiKeyLastUsed: true))
|
||||
&& TryGetPathToken(request.Path, out var pathToken, out _))
|
||||
{
|
||||
return true;
|
||||
var result = AuthorizeToken(pathToken, settings, managementStore, updateApiKeyLastUsed: true);
|
||||
if (result.IsAuthorized)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
return allowQueryToken
|
||||
&& request.Query.TryGetValue("token", out var queryValues)
|
||||
&& queryValues.Any(value => IsTokenAuthorized(value, settings, managementStore, updateApiKeyLastUsed: true));
|
||||
if (allowQueryToken
|
||||
&& request.Query.TryGetValue("token", out var queryValues))
|
||||
{
|
||||
foreach (var value in queryValues)
|
||||
{
|
||||
var result = AuthorizeToken(value, settings, managementStore, updateApiKeyLastUsed: true);
|
||||
if (result.IsAuthorized)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return AuthResult.Failure;
|
||||
}
|
||||
|
||||
public static bool IsAuthorized(
|
||||
HttpRequest request,
|
||||
ServerSettings settings,
|
||||
ManagementStore managementStore,
|
||||
bool allowQueryToken,
|
||||
bool allowPathToken = false) =>
|
||||
Authorize(request, settings, managementStore, allowQueryToken, allowPathToken).IsAuthorized;
|
||||
|
||||
public static bool TryRemovePathToken(
|
||||
PathString path,
|
||||
ServerSettings settings,
|
||||
@@ -69,7 +103,7 @@ internal static class TokenAuthentication
|
||||
TryGetBearerToken(value, out var token)
|
||||
&& IsTokenAuthorized(token, settings, managementStore, updateApiKeyLastUsed: false);
|
||||
|
||||
public static bool IsTokenAuthorized(
|
||||
private static AuthResult AuthorizeToken(
|
||||
string? token,
|
||||
ServerSettings settings,
|
||||
ManagementStore managementStore,
|
||||
@@ -77,14 +111,32 @@ internal static class TokenAuthentication
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
return false;
|
||||
return AuthResult.Failure;
|
||||
}
|
||||
|
||||
return !string.IsNullOrWhiteSpace(settings.Token)
|
||||
&& string.Equals(token, settings.Token, StringComparison.Ordinal)
|
||||
|| managementStore.IsApiKeyValid(token, updateApiKeyLastUsed);
|
||||
if (!string.IsNullOrWhiteSpace(settings.Token)
|
||||
&& string.Equals(token, settings.Token, StringComparison.Ordinal))
|
||||
{
|
||||
return AuthResult.Success(null);
|
||||
}
|
||||
|
||||
var apiKeyId = managementStore.GetApiKeyId(token);
|
||||
if (apiKeyId is not null)
|
||||
{
|
||||
managementStore.IsApiKeyValid(token, updateApiKeyLastUsed);
|
||||
return AuthResult.Success(apiKeyId);
|
||||
}
|
||||
|
||||
return AuthResult.Failure;
|
||||
}
|
||||
|
||||
public static bool IsTokenAuthorized(
|
||||
string? token,
|
||||
ServerSettings settings,
|
||||
ManagementStore managementStore,
|
||||
bool updateApiKeyLastUsed) =>
|
||||
AuthorizeToken(token, settings, managementStore, updateApiKeyLastUsed).IsAuthorized;
|
||||
|
||||
private static bool TryGetBearerToken(string? authorization, out string token)
|
||||
{
|
||||
token = "";
|
||||
@@ -131,3 +183,20 @@ internal static class TokenAuthentication
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class AuthResult
|
||||
{
|
||||
public static AuthResult Failure { get; } = new(false, null);
|
||||
|
||||
public static AuthResult Success(string? apiKeyId) => new(true, apiKeyId);
|
||||
|
||||
public bool IsAuthorized { get; }
|
||||
|
||||
public string? ApiKeyId { get; }
|
||||
|
||||
private AuthResult(bool isAuthorized, string? apiKeyId)
|
||||
{
|
||||
IsAuthorized = isAuthorized;
|
||||
ApiKeyId = apiKeyId;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net.WebSockets;
|
||||
using System.Threading;
|
||||
using ReverseLlama.Protocol;
|
||||
|
||||
namespace ReverseLlama.Server;
|
||||
@@ -9,6 +10,7 @@ internal sealed class TunnelHub
|
||||
private readonly ConcurrentDictionary<string, TunnelConnection> _connections = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ILogger<TunnelHub> _logger;
|
||||
private readonly ILoggerFactory _loggerFactory;
|
||||
private long _roundRobinCounter;
|
||||
|
||||
public TunnelHub(ILogger<TunnelHub> logger, ILoggerFactory loggerFactory)
|
||||
{
|
||||
@@ -40,17 +42,23 @@ internal sealed class TunnelHub
|
||||
var withModel = allOpen.Where(connection => connection.HasModel(model)).ToList();
|
||||
if (withModel.Count > 0)
|
||||
{
|
||||
return withModel
|
||||
.OrderBy(connection => connection.PendingRequestCount)
|
||||
.ThenBy(connection => connection.ClientId, StringComparer.OrdinalIgnoreCase)
|
||||
.First();
|
||||
return PickBest(withModel);
|
||||
}
|
||||
}
|
||||
|
||||
return allOpen
|
||||
.OrderBy(connection => connection.PendingRequestCount)
|
||||
.ThenBy(connection => connection.ClientId, StringComparer.OrdinalIgnoreCase)
|
||||
.First();
|
||||
return PickBest(allOpen);
|
||||
}
|
||||
|
||||
private TunnelConnection PickBest(List<TunnelConnection> candidates)
|
||||
{
|
||||
var tick = (int)(Interlocked.Increment(ref _roundRobinCounter) & 0x7FFFFFFF);
|
||||
|
||||
return candidates
|
||||
.Select((connection, index) => (connection, index))
|
||||
.OrderBy(x => x.connection.PendingRequestCount)
|
||||
.ThenBy(x => (tick + x.index) % candidates.Count)
|
||||
.First()
|
||||
.connection;
|
||||
}
|
||||
|
||||
/// <summary>The only open connection, or null when zero or more than one client is connected.</summary>
|
||||
|
||||
@@ -281,7 +281,8 @@ textarea {
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
min-width: 760px;
|
||||
table-layout: fixed;
|
||||
min-width: 900px;
|
||||
}
|
||||
|
||||
th,
|
||||
@@ -292,6 +293,11 @@ td {
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
.col-models {
|
||||
width: 40%;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
th {
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
@@ -338,7 +344,7 @@ tr:last-child td {
|
||||
color: var(--text);
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
overflow-wrap: anywhere;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.badge.good {
|
||||
@@ -359,6 +365,17 @@ tr:last-child td {
|
||||
color: var(--amber);
|
||||
}
|
||||
|
||||
.badge[href] {
|
||||
text-decoration: none;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.badge[href]:hover {
|
||||
border-color: var(--blue);
|
||||
background: #edf4ff;
|
||||
color: var(--blue);
|
||||
}
|
||||
|
||||
.metric-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(120px, 1fr));
|
||||
@@ -440,6 +457,65 @@ tr:last-child td {
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.group-link {
|
||||
color: var(--blue);
|
||||
text-decoration: none;
|
||||
font-weight: 800;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.group-link:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
.code-inline {
|
||||
font-family: "Cascadia Mono", Consolas, monospace;
|
||||
font-size: 12px;
|
||||
background: var(--panel-alt);
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 4px;
|
||||
padding: 2px 6px;
|
||||
display: inline-block;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.balance-display {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 24px;
|
||||
}
|
||||
|
||||
.balance-current strong {
|
||||
display: block;
|
||||
font-size: 28px;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
.balance-current span {
|
||||
display: block;
|
||||
margin-top: 4px;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.balance-detail {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.billing-toggle {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.billing-toggle input[type="checkbox"] {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
@media (max-width: 860px) {
|
||||
.app-shell {
|
||||
grid-template-columns: 1fr;
|
||||
@@ -455,7 +531,7 @@ tr:last-child td {
|
||||
}
|
||||
|
||||
.nav {
|
||||
grid-template-columns: repeat(3, 1fr);
|
||||
grid-template-columns: repeat(5, 1fr);
|
||||
}
|
||||
|
||||
.nav a {
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
const state = {
|
||||
summary: null,
|
||||
detail: null,
|
||||
groupDetail: null,
|
||||
newKey: null,
|
||||
loading: false
|
||||
loading: false,
|
||||
usageData: null
|
||||
};
|
||||
|
||||
const content = document.getElementById("content");
|
||||
@@ -20,7 +22,7 @@ content.addEventListener("click", async (event) => {
|
||||
return;
|
||||
}
|
||||
|
||||
const { action, clientId, model, keyId } = button.dataset;
|
||||
const { action, clientId, model, keyId, groupId, memberId } = button.dataset;
|
||||
|
||||
try {
|
||||
setBusy(button, true);
|
||||
@@ -67,6 +69,53 @@ content.addEventListener("click", async (event) => {
|
||||
await navigator.clipboard.writeText(button.dataset.key);
|
||||
setNotice("API key copied.");
|
||||
}
|
||||
|
||||
if (action === "delete-group") {
|
||||
if (!confirm("Delete this group and all its clients and key assignments?")) {
|
||||
return;
|
||||
}
|
||||
|
||||
await api(`/groups/${encodeURIComponent(groupId)}`, { method: "DELETE" });
|
||||
setNotice("Group deleted.");
|
||||
window.location.hash = "#groups";
|
||||
await refresh();
|
||||
}
|
||||
|
||||
if (action === "remove-client") {
|
||||
if (!confirm("Remove this client from the group?")) {
|
||||
return;
|
||||
}
|
||||
|
||||
await api(`/groups/${encodeURIComponent(groupId)}/clients/${memberId}`, { method: "DELETE" });
|
||||
setNotice("Client removed.");
|
||||
await loadGroupDetail(groupId);
|
||||
}
|
||||
|
||||
if (action === "toggle-key-assignment") {
|
||||
await toggleApiKeyAssignment(groupId, keyId, button.dataset.assigned === "true");
|
||||
}
|
||||
|
||||
if (action === "delete-billing-rule") {
|
||||
const ruleId = button.dataset.ruleId;
|
||||
if (!confirm("Delete this billing rule?")) {
|
||||
return;
|
||||
}
|
||||
|
||||
await api(`/groups/${encodeURIComponent(groupId)}/billing/rules/${ruleId}`, { method: "DELETE" });
|
||||
setNotice("Billing rule deleted.");
|
||||
await loadGroupDetail(groupId);
|
||||
}
|
||||
|
||||
if (action === "delete-payment") {
|
||||
const paymentId = button.dataset.paymentId;
|
||||
if (!confirm("Delete this payment record?")) {
|
||||
return;
|
||||
}
|
||||
|
||||
await api(`/groups/${encodeURIComponent(groupId)}/billing/payments/${paymentId}`, { method: "DELETE" });
|
||||
setNotice("Payment deleted.");
|
||||
await loadGroupDetail(groupId);
|
||||
}
|
||||
} catch (error) {
|
||||
setNotice(error.message, true);
|
||||
} finally {
|
||||
@@ -104,6 +153,97 @@ content.addEventListener("submit", async (event) => {
|
||||
setNotice("API key created.");
|
||||
await refresh();
|
||||
}
|
||||
|
||||
if (form.dataset.form === "create-group") {
|
||||
const result = await api("/groups", {
|
||||
method: "POST",
|
||||
body: { name: data.name }
|
||||
});
|
||||
setNotice("Group created.");
|
||||
form.reset();
|
||||
await refresh();
|
||||
window.location.hash = `#groups/${encodeURIComponent(result.id)}`;
|
||||
}
|
||||
|
||||
if (form.dataset.form === "edit-group-name") {
|
||||
const groupId = form.dataset.groupId;
|
||||
await api(`/groups/${encodeURIComponent(groupId)}`, {
|
||||
method: "PUT",
|
||||
body: { name: data.name }
|
||||
});
|
||||
setNotice("Group name updated.");
|
||||
await refresh();
|
||||
await loadGroupDetail(groupId);
|
||||
}
|
||||
|
||||
if (form.dataset.form === "add-client") {
|
||||
const groupId = form.dataset.groupId;
|
||||
const body = {};
|
||||
if (data.clientId && data.clientId.trim()) {
|
||||
body.clientId = data.clientId.trim();
|
||||
}
|
||||
if (data.model && data.model.trim()) {
|
||||
body.model = data.model.trim();
|
||||
}
|
||||
if (data.clientPattern && data.clientPattern.trim()) {
|
||||
body.clientPattern = data.clientPattern.trim();
|
||||
}
|
||||
|
||||
if (!body.clientId && !body.clientPattern) {
|
||||
throw new Error("Either Client ID or Client pattern is required.");
|
||||
}
|
||||
|
||||
await api(`/groups/${encodeURIComponent(groupId)}/clients`, {
|
||||
method: "POST",
|
||||
body
|
||||
});
|
||||
setNotice("Client added.");
|
||||
form.reset();
|
||||
await loadGroupDetail(groupId);
|
||||
}
|
||||
|
||||
if (form.dataset.form === "billing-config") {
|
||||
const groupId = form.dataset.groupId;
|
||||
await api(`/groups/${encodeURIComponent(groupId)}/billing`, {
|
||||
method: "PUT",
|
||||
body: {
|
||||
currency: data.currency,
|
||||
defaultRatePer1k: parseFloat(data.defaultRatePer1k) || 0,
|
||||
refuseBelowBalance: parseFloat(data.refuseBelowBalance) || 0,
|
||||
enabled: data.enabled === "on"
|
||||
}
|
||||
});
|
||||
setNotice("Billing configuration saved.");
|
||||
await loadGroupDetail(groupId);
|
||||
}
|
||||
|
||||
if (form.dataset.form === "add-billing-rule") {
|
||||
const groupId = form.dataset.groupId;
|
||||
await api(`/groups/${encodeURIComponent(groupId)}/billing/rules`, {
|
||||
method: "POST",
|
||||
body: {
|
||||
modelRegex: data.modelRegex,
|
||||
ratePer1k: parseFloat(data.ratePer1k) || 0
|
||||
}
|
||||
});
|
||||
setNotice("Billing rule added.");
|
||||
form.reset();
|
||||
await loadGroupDetail(groupId);
|
||||
}
|
||||
|
||||
if (form.dataset.form === "add-payment") {
|
||||
const groupId = form.dataset.groupId;
|
||||
await api(`/groups/${encodeURIComponent(groupId)}/billing/payments`, {
|
||||
method: "POST",
|
||||
body: {
|
||||
amount: parseFloat(data.amount) || 0,
|
||||
description: data.description || null
|
||||
}
|
||||
});
|
||||
setNotice("Payment recorded.");
|
||||
form.reset();
|
||||
await loadGroupDetail(groupId);
|
||||
}
|
||||
} catch (error) {
|
||||
setNotice(error.message, true);
|
||||
} finally {
|
||||
@@ -166,19 +306,28 @@ async function api(path, options = {}) {
|
||||
|
||||
async function renderRoute() {
|
||||
const hash = (window.location.hash || "#clients").slice(1);
|
||||
const [view, encodedModel] = hash.split("/");
|
||||
const parts = hash.split("/");
|
||||
const view = parts[0];
|
||||
const encodedParam = parts[1];
|
||||
|
||||
document.querySelectorAll("[data-nav]").forEach((link) => {
|
||||
link.classList.toggle("active", link.dataset.nav === view);
|
||||
});
|
||||
|
||||
if (view === "models" && encodedModel) {
|
||||
const model = decodeURIComponent(encodedModel);
|
||||
if (view === "models" && encodedParam) {
|
||||
const model = decodeURIComponent(encodedParam);
|
||||
await loadModelDetail(model);
|
||||
return;
|
||||
}
|
||||
|
||||
if (view === "groups" && encodedParam) {
|
||||
const groupId = decodeURIComponent(encodedParam);
|
||||
await loadGroupDetail(groupId);
|
||||
return;
|
||||
}
|
||||
|
||||
state.detail = null;
|
||||
state.groupDetail = null;
|
||||
|
||||
if (view === "models") {
|
||||
renderModels();
|
||||
@@ -190,6 +339,16 @@ async function renderRoute() {
|
||||
return;
|
||||
}
|
||||
|
||||
if (view === "groups") {
|
||||
renderGroups();
|
||||
return;
|
||||
}
|
||||
|
||||
if (view === "usage") {
|
||||
await loadUsage();
|
||||
return;
|
||||
}
|
||||
|
||||
renderClients();
|
||||
}
|
||||
|
||||
@@ -203,6 +362,7 @@ function updateShell() {
|
||||
<div>${escapeHtml(summary.user?.name || "Signed in")}</div>
|
||||
<div>${summary.clients.length} clients</div>
|
||||
<div>${summary.models.length} models</div>
|
||||
<div>${(summary.groups || []).length} groups</div>
|
||||
<div>${formatDate(summary.generatedAtUtc)}</div>
|
||||
`;
|
||||
}
|
||||
@@ -226,7 +386,10 @@ function renderClients() {
|
||||
}
|
||||
|
||||
function clientsTable(clients) {
|
||||
const rows = clients.map((client) => `
|
||||
const clientGroups = state.summary?.clientGroups || {};
|
||||
const rows = clients.map((client) => {
|
||||
const groups = clientGroups[client.id] || [];
|
||||
return `
|
||||
<tr>
|
||||
<td>
|
||||
<div class="cell-main">${escapeHtml(client.id)}</div>
|
||||
@@ -244,8 +407,13 @@ function clientsTable(clients) {
|
||||
<div class="cell-main">${number(client.requestStats.total)}</div>
|
||||
<div class="cell-sub">${number(client.requestStats.last10Minutes)} in 10m, ${number(client.requestStats.lastHour)} in 1h</div>
|
||||
</td>
|
||||
<td>${modelBadges(client.models)}</td>
|
||||
<td class="col-models">${modelBadges(client.models)}</td>
|
||||
<td>${modelBadges(client.activeModels)}</td>
|
||||
<td>
|
||||
<div class="badge-row">
|
||||
${groups.length ? groups.map((g) => `<a class="badge" href="#groups/${encodeURIComponent(g)}">${escapeHtml(g)}</a>`).join("") : `<span class="cell-sub">None</span>`}
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="actions">
|
||||
<button class="button warning" data-action="disable-hour" data-client-id="${escapeAttr(client.id)}" ${client.disabled ? "disabled" : ""}>Disable 1h</button>
|
||||
@@ -254,7 +422,7 @@ function clientsTable(clients) {
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
`).join("");
|
||||
`}).join("");
|
||||
|
||||
return `
|
||||
<table>
|
||||
@@ -264,8 +432,9 @@ function clientsTable(clients) {
|
||||
<th>Status</th>
|
||||
<th>Pending</th>
|
||||
<th>Requests</th>
|
||||
<th>Listed models</th>
|
||||
<th class="col-models">Listed models</th>
|
||||
<th>Active models</th>
|
||||
<th>Groups</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
@@ -565,6 +734,606 @@ function apiKeysTable(keys) {
|
||||
`;
|
||||
}
|
||||
|
||||
function renderGroups() {
|
||||
const groups = state.summary?.groups || [];
|
||||
pageTitle.textContent = "Groups";
|
||||
pageSubtitle.textContent = "Manage access groups that control which clients and models API keys can reach.";
|
||||
|
||||
content.innerHTML = `
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Create group</h2>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<form class="form-row" data-form="create-group">
|
||||
<div class="field">
|
||||
<label for="groupName">Name</label>
|
||||
<input class="input" id="groupName" name="name" placeholder="e.g. embeddings" required>
|
||||
</div>
|
||||
<button class="button" type="submit">Create</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Groups</h2>
|
||||
<span class="badge">${groups.length} total</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
${groups.length ? groupsTable(groups) : emptyState("No groups have been created.")}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function groupsTable(groups) {
|
||||
const rows = groups.map((group) => `
|
||||
<tr>
|
||||
<td>
|
||||
<a class="cell-main group-link" href="#groups/${encodeURIComponent(group.id)}">${escapeHtml(group.name)}</a>
|
||||
</td>
|
||||
<td>${formatDate(group.createdAtUtc)}</td>
|
||||
<td>
|
||||
<div class="actions">
|
||||
<a class="button secondary" href="#groups/${encodeURIComponent(group.id)}">Edit</a>
|
||||
<button class="button danger" data-action="delete-group" data-group-id="${escapeAttr(group.id)}">Delete</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
`).join("");
|
||||
|
||||
return `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Created</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
async function loadGroupDetail(groupId) {
|
||||
pageTitle.textContent = "Group Detail";
|
||||
pageSubtitle.textContent = groupId;
|
||||
content.innerHTML = `<div class="panel"><div class="empty">Loading group...</div></div>`;
|
||||
|
||||
try {
|
||||
const [group, clients, apiKeyGroups, billing, rules, payments, balance] = await Promise.all([
|
||||
api(`/groups/${encodeURIComponent(groupId)}`),
|
||||
api(`/groups/${encodeURIComponent(groupId)}/clients`),
|
||||
api("/api-keys/groups"),
|
||||
api(`/groups/${encodeURIComponent(groupId)}/billing`),
|
||||
api(`/groups/${encodeURIComponent(groupId)}/billing/rules`),
|
||||
api(`/groups/${encodeURIComponent(groupId)}/billing/payments`),
|
||||
api(`/groups/${encodeURIComponent(groupId)}/billing/balance`)
|
||||
]);
|
||||
|
||||
state.groupDetail = { group, clients, apiKeyGroups, billing, rules, payments, balance };
|
||||
renderGroupDetail();
|
||||
} catch (error) {
|
||||
content.innerHTML = `<div class="panel"><div class="empty">${escapeHtml(error.message)}</div></div>`;
|
||||
}
|
||||
}
|
||||
|
||||
function renderGroupDetail() {
|
||||
const { group, clients, apiKeyGroups, billing, rules, payments, balance } = state.groupDetail;
|
||||
const allApiKeys = state.summary?.apiKeys || [];
|
||||
const assignedKeyIds = new Set(
|
||||
apiKeyGroups
|
||||
.filter((akg) => akg.apiKeyId && (akg.groupIds || []).includes(group.id))
|
||||
.map((akg) => akg.apiKeyId)
|
||||
);
|
||||
|
||||
pageTitle.textContent = "Group Detail";
|
||||
pageSubtitle.textContent = group.name;
|
||||
|
||||
content.innerHTML = `
|
||||
<div class="toolbar">
|
||||
<a class="button secondary" href="#groups">Back to groups</a>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Group name</h2>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<form class="form-row" data-form="edit-group-name" data-group-id="${escapeAttr(group.id)}">
|
||||
<div class="field">
|
||||
<label for="editGroupName">Name</label>
|
||||
<input class="input" id="editGroupName" name="name" value="${escapeAttr(group.name)}" required>
|
||||
</div>
|
||||
<button class="button" type="submit">Save</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Add client</h2>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<form class="form-row" data-form="add-client" data-group-id="${escapeAttr(group.id)}">
|
||||
<div class="field">
|
||||
<label for="addClientClientId">Client ID</label>
|
||||
<input class="input" id="addClientClientId" name="clientId" placeholder="Client_1">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="addClientModel">Model (optional)</label>
|
||||
<input class="input" id="addClientModel" name="model" placeholder="bge-m3">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="addClientPattern">Client pattern (regex, optional)</label>
|
||||
<input class="input" id="addClientPattern" name="clientPattern" placeholder="GPU_[0-9]*">
|
||||
</div>
|
||||
<button class="button" type="submit">Add</button>
|
||||
</form>
|
||||
<div class="cell-sub" style="margin-top:8px">Provide either a Client ID (for explicit client access) or a Client pattern (for regex-based matching). Model is optional to restrict to a specific model.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Clients</h2>
|
||||
<span class="badge">${clients.length} total</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
${clients.length ? groupClientsTable(clients, group.id) : emptyState("No clients in this group.")}
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>API key assignments</h2>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
${allApiKeys.length ? apiKeyAssignmentTable(allApiKeys, group.id, assignedKeyIds) : emptyState("No API keys have been created.")}
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Billing</h2>
|
||||
${billing.enabled ? badge("Enabled", "good") : badge("Disabled", "")}
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<form class="form-row" data-form="billing-config" data-group-id="${escapeAttr(group.id)}">
|
||||
<div class="field">
|
||||
<label for="billingCurrency">Currency</label>
|
||||
<select class="select" id="billingCurrency" name="currency">
|
||||
${["EUR", "USD", "GBP", "INR", "JPY", "CAD", "AUD", "CHF"].map(c => `<option value="${c}" ${billing.currency === c ? "selected" : ""}>${c}</option>`).join("")}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="billingDefaultRate">Default rate / 1k tokens</label>
|
||||
<input class="input" id="billingDefaultRate" name="defaultRatePer1k" type="number" step="0.0001" min="0" value="${billing.defaultRatePer1k || 0}">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="billingRefuseBelow">Refuse below balance</label>
|
||||
<input class="input" id="billingRefuseBelow" name="refuseBelowBalance" type="number" step="0.01" value="${billing.refuseBelowBalance || 0}">
|
||||
</div>
|
||||
<div class="field billing-toggle">
|
||||
<label for="billingEnabled">Enabled</label>
|
||||
<input type="checkbox" id="billingEnabled" name="enabled" ${billing.enabled ? "checked" : ""}>
|
||||
</div>
|
||||
<button class="button" type="submit">Save</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Balance</h2>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div class="balance-display">
|
||||
<div class="balance-current">
|
||||
<strong>${formatCurrency(balance.balance, balance.currency)}</strong>
|
||||
<span>Current balance</span>
|
||||
</div>
|
||||
<div class="balance-detail">
|
||||
<div>${formatCurrency(balance.totalPayments, balance.currency)} payments</div>
|
||||
<div>${formatCurrency(balance.totalCosts, balance.currency)} costs</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Billing rules</h2>
|
||||
<span class="badge">${rules.length} total</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
${rules.length ? billingRulesTable(rules, group.id) : emptyState("No billing rules. The default rate applies to all models.")}
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<form class="form-row" data-form="add-billing-rule" data-group-id="${escapeAttr(group.id)}">
|
||||
<div class="field">
|
||||
<label for="ruleModelRegex">Model regex</label>
|
||||
<input class="input" id="ruleModelRegex" name="modelRegex" placeholder="llama.*" required>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="ruleRate">Rate / 1k tokens</label>
|
||||
<input class="input" id="ruleRate" name="ratePer1k" type="number" step="0.0001" min="0" required>
|
||||
</div>
|
||||
<button class="button" type="submit">Add rule</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Payments</h2>
|
||||
<span class="badge">${payments.length} total</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
${payments.length ? paymentsTable(payments, group.id) : emptyState("No payments recorded.")}
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<form class="form-row" data-form="add-payment" data-group-id="${escapeAttr(group.id)}">
|
||||
<div class="field">
|
||||
<label for="paymentAmount">Amount</label>
|
||||
<input class="input" id="paymentAmount" name="amount" type="number" step="0.01" min="0" required>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="paymentDescription">Description</label>
|
||||
<input class="input" id="paymentDescription" name="description" placeholder="Invoice #1234">
|
||||
</div>
|
||||
<button class="button" type="submit">Record payment</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function groupClientsTable(clients, groupId) {
|
||||
const rows = clients.map((client) => `
|
||||
<tr>
|
||||
<td>
|
||||
${client.clientId
|
||||
? `<div class="cell-main">${escapeHtml(client.clientId)}</div>`
|
||||
: `<div class="cell-sub">-</div>`
|
||||
}
|
||||
</td>
|
||||
<td>
|
||||
${client.model
|
||||
? `<div class="cell-main">${escapeHtml(client.model)}</div>`
|
||||
: `<div class="cell-sub">All models</div>`
|
||||
}
|
||||
</td>
|
||||
<td>
|
||||
${client.clientPattern
|
||||
? `<div class="cell-main code-inline">${escapeHtml(client.clientPattern)}</div>`
|
||||
: `<div class="cell-sub">-</div>`
|
||||
}
|
||||
</td>
|
||||
<td>
|
||||
<button class="button danger" data-action="remove-client" data-group-id="${escapeAttr(groupId)}" data-member-id="${client.id}">Remove</button>
|
||||
</td>
|
||||
</tr>
|
||||
`).join("");
|
||||
|
||||
return `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Client ID</th>
|
||||
<th>Model</th>
|
||||
<th>Client pattern</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
function apiKeyAssignmentTable(apiKeys, groupId, assignedKeyIds) {
|
||||
const rows = apiKeys.map((key) => {
|
||||
const isAssigned = assignedKeyIds.has(key.id);
|
||||
return `
|
||||
<tr>
|
||||
<td>
|
||||
<div class="cell-main">${escapeHtml(key.name)}</div>
|
||||
<div class="cell-sub">${escapeHtml(key.keyPrefix)}...</div>
|
||||
</td>
|
||||
<td>
|
||||
<button class="button ${isAssigned ? "danger" : "secondary"}"
|
||||
data-action="toggle-key-assignment"
|
||||
data-group-id="${escapeAttr(groupId)}"
|
||||
data-key-id="${escapeAttr(key.id)}"
|
||||
data-assigned="${isAssigned}">${isAssigned ? "Remove" : "Assign"}</button>
|
||||
</td>
|
||||
</tr>
|
||||
`;
|
||||
}).join("");
|
||||
|
||||
return `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>API Key</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
async function toggleApiKeyAssignment(groupId, keyId, currentlyAssigned) {
|
||||
const apiKeyGroups = state.groupDetail?.apiKeyGroups || [];
|
||||
const allApiKeys = state.summary?.apiKeys || [];
|
||||
|
||||
const keyGroups = apiKeyGroups.find((akg) => akg.apiKeyId === keyId);
|
||||
const currentGroupIds = keyGroups ? [...keyGroups.groupIds] : [];
|
||||
|
||||
let newGroupIds;
|
||||
if (currentlyAssigned) {
|
||||
newGroupIds = currentGroupIds.filter((id) => id !== groupId);
|
||||
} else {
|
||||
newGroupIds = [...currentGroupIds, groupId];
|
||||
}
|
||||
|
||||
await api(`/api-keys/${encodeURIComponent(keyId)}/groups`, {
|
||||
method: "PUT",
|
||||
body: { groupIds: newGroupIds }
|
||||
});
|
||||
|
||||
setNotice(currentlyAssigned ? "API key unassigned from group." : "API key assigned to group.");
|
||||
await loadGroupDetail(groupId);
|
||||
}
|
||||
|
||||
function billingRulesTable(rules, groupId) {
|
||||
const rows = rules.map((rule) => `
|
||||
<tr>
|
||||
<td><div class="code-inline">${escapeHtml(rule.modelRegex)}</div></td>
|
||||
<td>${rule.ratePer1k}</td>
|
||||
<td>
|
||||
<button class="button danger" data-action="delete-billing-rule" data-group-id="${escapeAttr(groupId)}" data-rule-id="${rule.id}">Delete</button>
|
||||
</td>
|
||||
</tr>
|
||||
`).join("");
|
||||
|
||||
return `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Model regex</th>
|
||||
<th>Rate / 1k tokens</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
function paymentsTable(payments, groupId) {
|
||||
const rows = payments.map((payment) => `
|
||||
<tr>
|
||||
<td>${formatCurrency(payment.amount, "")}</td>
|
||||
<td>${payment.description ? escapeHtml(payment.description) : `<span class="cell-sub">-</span>`}</td>
|
||||
<td>${payment.createdBy ? escapeHtml(payment.createdBy) : `<span class="cell-sub">-</span>`}</td>
|
||||
<td>${formatDate(payment.createdAtUtc)}</td>
|
||||
<td>
|
||||
<button class="button danger" data-action="delete-payment" data-group-id="${escapeAttr(groupId)}" data-payment-id="${payment.id}">Delete</button>
|
||||
</td>
|
||||
</tr>
|
||||
`).join("");
|
||||
|
||||
return `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Amount</th>
|
||||
<th>Description</th>
|
||||
<th>Created by</th>
|
||||
<th>Date</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
function formatCurrency(value, currency) {
|
||||
const num = typeof value === "number" ? value : parseFloat(value) || 0;
|
||||
const formatted = new Intl.NumberFormat(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 4 }).format(num);
|
||||
if (currency && typeof currency === "string" && currency.length > 0) {
|
||||
return `${formatted} ${currency}`;
|
||||
}
|
||||
return formatted;
|
||||
}
|
||||
|
||||
async function loadUsage() {
|
||||
pageTitle.textContent = "Usage";
|
||||
pageSubtitle.textContent = "Token usage and revenue statistics.";
|
||||
content.innerHTML = `<div class="panel"><div class="empty">Loading usage data...</div></div>`;
|
||||
|
||||
try {
|
||||
const [usage, revenue] = await Promise.all([
|
||||
api("/usage/tokens"),
|
||||
api("/usage/revenue")
|
||||
]);
|
||||
|
||||
state.usageData = { usage, revenue };
|
||||
renderUsage();
|
||||
} catch (error) {
|
||||
content.innerHTML = `<div class="panel"><div class="empty">${escapeHtml(error.message)}</div></div>`;
|
||||
}
|
||||
}
|
||||
|
||||
function renderUsage() {
|
||||
const { usage, revenue } = state.usageData;
|
||||
const hasAnyBilling = (state.summary?.groups || []).some((g) => {
|
||||
const billing = state.groupDetail?.billing;
|
||||
return billing?.enabled;
|
||||
});
|
||||
|
||||
const byModel = usage.byModel || [];
|
||||
const byClient = usage.byClient || [];
|
||||
const byApiKey = usage.byApiKey || [];
|
||||
const byGroup = usage.byGroup || [];
|
||||
const clientRevenue = revenue || [];
|
||||
|
||||
const revenueMap = {};
|
||||
for (const r of clientRevenue) {
|
||||
revenueMap[r.clientId] = r;
|
||||
}
|
||||
|
||||
content.innerHTML = `
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Tokens by model</h2>
|
||||
<span class="badge">${byModel.length} models</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
${byModel.length ? tokenStatsModelTable(byModel) : emptyState("No token data yet.")}
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Tokens by machine</h2>
|
||||
<span class="badge">${byClient.length} machines</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
${byClient.length ? tokenStatsClientTable(byClient, revenueMap) : emptyState("No token data yet.")}
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Tokens by API key</h2>
|
||||
<span class="badge">${byApiKey.length} keys</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
${byApiKey.length ? tokenStatsApiKeyTable(byApiKey) : emptyState("No token data yet.")}
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2>Tokens by group</h2>
|
||||
<span class="badge">${byGroup.length} groups</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
${byGroup.length ? tokenStatsGroupTable(byGroup) : emptyState("No token data yet.")}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function tokenStatsModelTable(stats) {
|
||||
const rows = stats.map((s) => `
|
||||
<tr>
|
||||
<td><div class="cell-main">${escapeHtml(s.model)}</div></td>
|
||||
<td>${number(s.promptTokens)}</td>
|
||||
<td>${number(s.completionTokens)}</td>
|
||||
<td>${number(s.totalTokens)}</td>
|
||||
<td>${number(s.requests)}</td>
|
||||
</tr>
|
||||
`).join("");
|
||||
|
||||
return `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Model</th>
|
||||
<th>Prompt tokens</th>
|
||||
<th>Completion tokens</th>
|
||||
<th>Total tokens</th>
|
||||
<th>Requests</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
function tokenStatsClientTable(stats, revenueMap) {
|
||||
const rows = stats.map((s) => {
|
||||
const rev = revenueMap[s.clientId];
|
||||
return `
|
||||
<tr>
|
||||
<td><div class="cell-main">${escapeHtml(s.clientId)}</div></td>
|
||||
<td>${number(s.promptTokens)}</td>
|
||||
<td>${number(s.completionTokens)}</td>
|
||||
<td>${number(s.totalTokens)}</td>
|
||||
<td>${number(s.requests)}</td>
|
||||
${rev ? `<td>${formatCurrency(rev.revenue, rev.currency)}</td>` : `<td><span class="cell-sub">-</span></td>`}
|
||||
</tr>
|
||||
`}).join("");
|
||||
|
||||
return `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Machine</th>
|
||||
<th>Prompt tokens</th>
|
||||
<th>Completion tokens</th>
|
||||
<th>Total tokens</th>
|
||||
<th>Requests</th>
|
||||
<th>Revenue</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
function tokenStatsApiKeyTable(stats) {
|
||||
const rows = stats.map((s) => `
|
||||
<tr>
|
||||
<td>
|
||||
<div class="cell-main">${escapeHtml(s.apiKeyName)}</div>
|
||||
<div class="cell-sub">${escapeHtml(s.apiKeyPrefix)}...</div>
|
||||
</td>
|
||||
<td>${number(s.promptTokens)}</td>
|
||||
<td>${number(s.completionTokens)}</td>
|
||||
<td>${number(s.totalTokens)}</td>
|
||||
<td>${number(s.requests)}</td>
|
||||
</tr>
|
||||
`).join("");
|
||||
|
||||
return `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>API key</th>
|
||||
<th>Prompt tokens</th>
|
||||
<th>Completion tokens</th>
|
||||
<th>Total tokens</th>
|
||||
<th>Requests</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
function tokenStatsGroupTable(stats) {
|
||||
const rows = stats.map((s) => `
|
||||
<tr>
|
||||
<td><div class="cell-main">${escapeHtml(s.groupName)}</div></td>
|
||||
<td>${number(s.promptTokens)}</td>
|
||||
<td>${number(s.completionTokens)}</td>
|
||||
<td>${number(s.totalTokens)}</td>
|
||||
<td>${number(s.requests)}</td>
|
||||
</tr>
|
||||
`).join("");
|
||||
|
||||
return `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Group</th>
|
||||
<th>Prompt tokens</th>
|
||||
<th>Completion tokens</th>
|
||||
<th>Total tokens</th>
|
||||
<th>Requests</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
function connectedClients() {
|
||||
return (state.summary?.clients || []).filter((client) => client.connected);
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@
|
||||
<a href="#clients" data-nav="clients">Clients</a>
|
||||
<a href="#models" data-nav="models">Models</a>
|
||||
<a href="#api-keys" data-nav="api-keys">API keys</a>
|
||||
<a href="#groups" data-nav="groups">Groups</a>
|
||||
<a href="#usage" data-nav="usage">Usage</a>
|
||||
</nav>
|
||||
<div class="sidebar-meta" id="sidebarMeta">Loading</div>
|
||||
</aside>
|
||||
|
||||
Reference in New Issue
Block a user