CI / test (push) Successful in 5m35s
A tool's answer is read by the caller's policies as the prompt it becomes, so an instruction planted in a ticket is withheld and personal data redacted; its arguments are read before they leave. ctl mcp add --rate bounds each caller's calls a minute to a server (migration 0050), and five failures in a row take a server out for a minute, its tools off the list. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
486 lines
16 KiB
Go
486 lines
16 KiB
Go
package router
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"maps"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mcpclient"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/secrets"
|
|
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// The company's own MCP servers behind llmgw's /mcp. An administrator
|
|
// registers each once (llmgw ctl mcp add); /mcp then lists its tools as
|
|
// server.tool beside llmgw's own, to a caller whose key is granted them, and
|
|
// calls them with the server's credentials, which never leave the control
|
|
// plane. Every call is in the audit log.
|
|
|
|
const (
|
|
// mcpServersFresh is how long the server list read from the store is
|
|
// used before it is read again: a server added on one control plane is
|
|
// served by every one within it.
|
|
mcpServersFresh = 30 * time.Second
|
|
// mcpToolsFresh is how long a server's tool list is used. A server that
|
|
// cannot be asked keeps the last list it gave.
|
|
mcpToolsFresh = 5 * time.Minute
|
|
// mcpCallTimeout is a call's bound when the server's own is not set.
|
|
mcpCallTimeout = 60 * time.Second
|
|
// mcpAnswerBytes bounds one tool's answer: it becomes a prompt.
|
|
mcpAnswerBytes = 4 << 20
|
|
// mcpBreakAfter failures in a row take a server out for mcpBreakFor:
|
|
// its tools leave the list, and a call to one is answered at once.
|
|
mcpBreakAfter = 5
|
|
mcpBreakFor = time.Minute
|
|
)
|
|
|
|
// mcpInspect has the guardrails read a tool's text: what goes to the company
|
|
// system, and what comes back to the agent as its prompt. It answers the
|
|
// text to use (redacted, if the policy redacts), or why it is refused.
|
|
type mcpInspect func(req *http.Request, path, text string) (out, refused string, err error)
|
|
|
|
type mcpStore interface {
|
|
MCPServers(ctx context.Context) ([]storepkg.MCPServer, error)
|
|
WriteAudit(ctx context.Context, e storepkg.AuditEvent) error
|
|
}
|
|
|
|
// mcpGateway is the company servers as /mcp serves them.
|
|
type mcpGateway struct {
|
|
st mcpStore
|
|
log *slog.Logger
|
|
http *http.Client
|
|
now func() time.Time
|
|
|
|
calls *prometheus.CounterVec
|
|
duration *prometheus.HistogramVec
|
|
|
|
mu sync.Mutex
|
|
readAt time.Time
|
|
upstream map[string]*mcpUpstream
|
|
// local is this cluster's own MCP server, the admin API's, which a
|
|
// server registered as llmgw:admin is answered by in this process.
|
|
local http.Handler
|
|
// inspect is the guardrails; nil reads nothing.
|
|
inspect mcpInspect
|
|
// calledAt is each caller's recent calls to each server, for its rate.
|
|
calledAt map[string][]time.Time
|
|
}
|
|
|
|
// SetAdminMCP is the cluster's own read-only MCP server, served to keys
|
|
// granted it once an administrator registers it as llmgw:admin.
|
|
func (r *Router) SetAdminMCP(h http.Handler) {
|
|
if r.mcpgw == nil {
|
|
return
|
|
}
|
|
r.mcpgw.mu.Lock()
|
|
defer r.mcpgw.mu.Unlock()
|
|
r.mcpgw.local = h
|
|
r.mcpgw.readAt = time.Time{}
|
|
}
|
|
|
|
// mcpUpstream is one server: its settings, its client, its tools.
|
|
type mcpUpstream struct {
|
|
cfg storepkg.MCPServer
|
|
client *mcpclient.Client
|
|
|
|
mu sync.Mutex
|
|
tools []mcpclient.Tool
|
|
toolsAt time.Time
|
|
toolErr error
|
|
// fails is the calls that failed in a row, and brokenUntil when a
|
|
// server taken out for them is tried again.
|
|
fails int
|
|
brokenUntil time.Time
|
|
}
|
|
|
|
// broken is a server taken out for failing.
|
|
func (u *mcpUpstream) broken(now time.Time) bool {
|
|
u.mu.Lock()
|
|
defer u.mu.Unlock()
|
|
return now.Before(u.brokenUntil)
|
|
}
|
|
|
|
// result counts a call that reached the server or did not.
|
|
func (u *mcpUpstream) result(ok bool, now time.Time) (justBroke bool) {
|
|
u.mu.Lock()
|
|
defer u.mu.Unlock()
|
|
if ok {
|
|
u.fails = 0
|
|
return false
|
|
}
|
|
u.fails++
|
|
if u.fails >= mcpBreakAfter && !now.Before(u.brokenUntil) {
|
|
u.brokenUntil, u.fails = now.Add(mcpBreakFor), 0
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// allow is whether a caller may call a server's tools once more this minute.
|
|
func (g *mcpGateway) allow(caller string, u *mcpUpstream) bool {
|
|
if u.cfg.RatePerMinute <= 0 {
|
|
return true
|
|
}
|
|
g.mu.Lock()
|
|
defer g.mu.Unlock()
|
|
if g.calledAt == nil {
|
|
g.calledAt = map[string][]time.Time{}
|
|
}
|
|
key, now := caller+"|"+u.cfg.Name, g.now()
|
|
recent := g.calledAt[key][:0]
|
|
for _, t := range g.calledAt[key] {
|
|
if now.Sub(t) < time.Minute {
|
|
recent = append(recent, t)
|
|
}
|
|
}
|
|
if len(recent) >= u.cfg.RatePerMinute {
|
|
g.calledAt[key] = recent
|
|
return false
|
|
}
|
|
g.calledAt[key] = append(recent, now)
|
|
return true
|
|
}
|
|
|
|
func newMCPGateway(st mcpStore, log *slog.Logger) *mcpGateway {
|
|
return &mcpGateway{
|
|
st: st, log: log, http: &http.Client{}, now: time.Now, upstream: map[string]*mcpUpstream{},
|
|
calls: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_mcp_tool_calls_total",
|
|
Help: "Calls to the company MCP servers' tools through /mcp, by server, tool and outcome (ok, tool_error, error, denied).",
|
|
}, []string{"server", "tool", "outcome"}),
|
|
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_mcp_tool_call_seconds",
|
|
Help: "How long a company MCP server took to answer a tool call.",
|
|
Buckets: prometheus.ExponentialBuckets(0.05, 2, 12),
|
|
}, []string{"server", "tool"}),
|
|
}
|
|
}
|
|
|
|
func (g *mcpGateway) collectors() []prometheus.Collector {
|
|
return []prometheus.Collector{g.calls, g.duration}
|
|
}
|
|
|
|
// servers are the enabled servers, read from the store when the list is
|
|
// stale; a store that does not answer leaves the last list.
|
|
func (g *mcpGateway) servers(ctx context.Context) []*mcpUpstream {
|
|
g.mu.Lock()
|
|
defer g.mu.Unlock()
|
|
if g.now().Sub(g.readAt) >= mcpServersFresh {
|
|
list, err := g.st.MCPServers(ctx)
|
|
if err != nil {
|
|
g.log.Warn("could not read the MCP servers; serving the last list", "err", err)
|
|
} else {
|
|
g.readAt = g.now()
|
|
next := map[string]*mcpUpstream{}
|
|
for _, s := range list {
|
|
if old, ok := g.upstream[s.Name]; ok && sameMCPServer(old.cfg, s) {
|
|
old.cfg = s
|
|
next[s.Name] = old
|
|
continue
|
|
}
|
|
next[s.Name] = g.upstreamFor(s)
|
|
}
|
|
g.upstream = next
|
|
}
|
|
}
|
|
var out []*mcpUpstream
|
|
for _, name := range slices.Sorted(maps.Keys(g.upstream)) {
|
|
if u := g.upstream[name]; u.cfg.Enabled {
|
|
out = append(out, u)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// sameMCPServer is whether a client made for a can serve b: the same address
|
|
// and headers. A changed description or switch keeps the session.
|
|
func sameMCPServer(a, b storepkg.MCPServer) bool {
|
|
return a.URL == b.URL && maps.Equal(a.Headers, b.Headers) && maps.Equal(a.HeaderRefs, b.HeaderRefs)
|
|
}
|
|
|
|
func (g *mcpGateway) upstreamFor(s storepkg.MCPServer) *mcpUpstream {
|
|
cfg := s
|
|
url, client := s.URL, g.http
|
|
if s.URL == adminapi.AdminMCPAddress && g.local != nil {
|
|
url, client = adminapi.InProcessURL, &http.Client{Transport: adminapi.InProcess(g.local)}
|
|
}
|
|
return &mcpUpstream{cfg: s, client: &mcpclient.Client{
|
|
URL: url, HTTP: client, MaxBytes: mcpAnswerBytes,
|
|
// Read at each request, so a rotated credential is used at once.
|
|
Header: func(ctx context.Context) (http.Header, error) {
|
|
h := http.Header{}
|
|
for k, v := range cfg.Headers {
|
|
h.Set(k, v)
|
|
}
|
|
for k, ref := range cfg.HeaderRefs {
|
|
v, err := secrets.ReadString(ctx, ref)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the %s header of MCP server %s: %w", k, cfg.Name, err)
|
|
}
|
|
h.Set(k, v)
|
|
}
|
|
return h, nil
|
|
},
|
|
}}
|
|
}
|
|
|
|
// toolsOf is a server's tools, asked for again when stale.
|
|
func (u *mcpUpstream) toolsOf(ctx context.Context, now time.Time, log *slog.Logger) []mcpclient.Tool {
|
|
u.mu.Lock()
|
|
defer u.mu.Unlock()
|
|
if now.Sub(u.toolsAt) < mcpToolsFresh && (u.tools != nil || u.toolErr != nil) {
|
|
return u.tools
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
tools, err := u.client.Tools(ctx)
|
|
u.toolsAt, u.toolErr = now, err
|
|
if err != nil {
|
|
log.Warn("could not list an MCP server's tools; serving the last list", "server", u.cfg.Name, "err", err)
|
|
return u.tools
|
|
}
|
|
u.tools = tools
|
|
return tools
|
|
}
|
|
|
|
// list is the company tools this caller may use, as /mcp lists them. No
|
|
// caller (authentication off) is granted none: they are granted, not assumed.
|
|
func (g *mcpGateway) list(req *http.Request) []map[string]any {
|
|
p, ok := auth.FromContext(req.Context())
|
|
if g == nil || !ok || len(p.Limits.ToolGrants) == 0 {
|
|
return nil
|
|
}
|
|
var out []map[string]any
|
|
for _, u := range g.servers(req.Context()) {
|
|
if u.broken(g.now()) {
|
|
continue
|
|
}
|
|
for _, t := range u.toolsOf(req.Context(), g.now(), g.log) {
|
|
name := u.cfg.Name + "." + t.Name
|
|
if !p.Limits.AllowsTool(name) {
|
|
continue
|
|
}
|
|
desc := t.Description
|
|
if u.cfg.Description != "" {
|
|
desc = "[" + u.cfg.Description + "] " + desc
|
|
}
|
|
tool := map[string]any{"name": name, "description": desc}
|
|
if len(t.InputSchema) > 0 {
|
|
tool["inputSchema"] = t.InputSchema
|
|
} else {
|
|
tool["inputSchema"] = map[string]any{"type": "object"}
|
|
}
|
|
if t.Title != "" {
|
|
tool["title"] = t.Title
|
|
}
|
|
if len(t.Annotations) > 0 {
|
|
tool["annotations"] = t.Annotations
|
|
}
|
|
out = append(out, tool)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// isCompanyTool is a tool name with a server's prefix: llmgw's own have no dot.
|
|
func isCompanyTool(name string) bool { return strings.Contains(name, ".") }
|
|
|
|
// call calls a company tool for the caller and gives the tool's result as
|
|
// /mcp answers it: the server's own, or a tool error saying what went wrong.
|
|
func (g *mcpGateway) call(req *http.Request, name string, args json.RawMessage, progress func(json.RawMessage)) any {
|
|
server, tool, _ := strings.Cut(name, ".")
|
|
p, ok := auth.FromContext(req.Context())
|
|
if g == nil || !ok || !p.Limits.AllowsTool(name) {
|
|
if g != nil {
|
|
g.calls.WithLabelValues(server, tool, "denied").Inc()
|
|
g.audit(req.Context(), p, ok, name, server, args, 0, http.StatusForbidden, "not granted")
|
|
}
|
|
return toolText(name+" is not a tool this key may call", true)
|
|
}
|
|
var u *mcpUpstream
|
|
for _, s := range g.servers(req.Context()) {
|
|
if s.cfg.Name == server {
|
|
u = s
|
|
}
|
|
}
|
|
if u == nil {
|
|
return toolText("no MCP server "+server+" is registered and on", true)
|
|
}
|
|
if u.broken(g.now()) {
|
|
g.calls.WithLabelValues(server, tool, "broken").Inc()
|
|
return toolText(fmt.Sprintf("the MCP server %s failed %d calls in a row and is tried again within %s",
|
|
server, mcpBreakAfter, mcpBreakFor), true)
|
|
}
|
|
if !g.allow(kbActor(p), u) {
|
|
g.calls.WithLabelValues(server, tool, "rate_limited").Inc()
|
|
g.audit(req.Context(), p, true, name, server, args, 0, http.StatusTooManyRequests, "rate limited")
|
|
return toolText(fmt.Sprintf("more than %d calls a minute to %s: try again shortly", u.cfg.RatePerMinute, server), true)
|
|
}
|
|
// What goes to the company system first: a secret pasted into an
|
|
// argument does not leave.
|
|
if g.inspect != nil {
|
|
out, refused, err := g.inspect(req, "/mcp/tool-arguments/"+name, string(args))
|
|
switch {
|
|
case err != nil:
|
|
g.calls.WithLabelValues(server, tool, "denied").Inc()
|
|
return toolText("the call to "+name+" could not be checked against this gateway's policies; try again", true)
|
|
case refused != "":
|
|
g.calls.WithLabelValues(server, tool, "denied").Inc()
|
|
g.audit(req.Context(), p, true, name, server, args, 0, http.StatusBadRequest, "arguments refused by policy")
|
|
return toolText("the call to "+name+" was refused: "+refused, true)
|
|
case out != string(args) && json.Valid([]byte(out)):
|
|
args = json.RawMessage(out)
|
|
}
|
|
}
|
|
timeout := u.cfg.Timeout
|
|
if timeout <= 0 {
|
|
timeout = mcpCallTimeout
|
|
}
|
|
ctx, cancel := context.WithTimeout(req.Context(), timeout)
|
|
defer cancel()
|
|
// Who is calling, for the server's own log; its credentials are llmgw's.
|
|
who := http.Header{"X-Llmgw-Key": {p.KeyID}}
|
|
if p.UserID != "" {
|
|
who.Set("X-Llmgw-User", p.UserID)
|
|
}
|
|
if p.ProjectID != "" {
|
|
who.Set("X-Llmgw-Project", p.ProjectID)
|
|
}
|
|
start := g.now()
|
|
raw, err := u.client.Call(ctx, tool, args, who, progress)
|
|
took := g.now().Sub(start)
|
|
g.duration.WithLabelValues(server, tool).Observe(took.Seconds())
|
|
var rpcErr *mcpclient.Error
|
|
if u.result(err == nil || errors.As(err, &rpcErr), g.now()) {
|
|
g.log.Warn("an MCP server failed calls in a row; its tools are out for a while",
|
|
"server", server, "failures", mcpBreakAfter, "for", mcpBreakFor)
|
|
}
|
|
switch {
|
|
case err != nil:
|
|
g.calls.WithLabelValues(server, tool, "error").Inc()
|
|
g.audit(req.Context(), p, true, name, server, args, took, http.StatusBadGateway, err.Error())
|
|
msg := err.Error()
|
|
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
|
msg = fmt.Sprintf("the MCP server %s did not answer within %s", server, timeout)
|
|
}
|
|
return toolText(msg, true)
|
|
}
|
|
var result map[string]any
|
|
if json.Unmarshal(raw, &result) != nil {
|
|
g.calls.WithLabelValues(server, tool, "error").Inc()
|
|
g.audit(req.Context(), p, true, name, server, args, took, http.StatusBadGateway, "the answer was not a tool result")
|
|
return toolText("the MCP server "+server+" answered something that is not a tool result", true)
|
|
}
|
|
// The answer becomes the agent's prompt: read as one, so an
|
|
// instruction planted in a ticket is caught as a pasted one would be.
|
|
if g.inspect != nil {
|
|
if why := g.inspectResult(req, name, result); why != "" {
|
|
g.calls.WithLabelValues(server, tool, "withheld").Inc()
|
|
g.audit(req.Context(), p, true, name, server, args, took, http.StatusBadGateway, "answer withheld by policy")
|
|
return toolText("the answer of "+name+" was withheld: "+why, true)
|
|
}
|
|
}
|
|
outcome, status := "ok", http.StatusOK
|
|
if isErr, _ := result["isError"].(bool); isErr {
|
|
outcome, status = "tool_error", http.StatusUnprocessableEntity
|
|
}
|
|
g.calls.WithLabelValues(server, tool, outcome).Inc()
|
|
g.audit(req.Context(), p, true, name, server, args, took, status, "")
|
|
return result
|
|
}
|
|
|
|
// audit records a call: who, which tool, the arguments' size and a digest of
|
|
// them, how long, and how it ended. Never the arguments themselves: they are
|
|
// the company's data.
|
|
func (g *mcpGateway) audit(ctx context.Context, p auth.Principal, authed bool, tool, server string,
|
|
args json.RawMessage, took time.Duration, status int, why string,
|
|
) {
|
|
actor := "anonymous"
|
|
if authed {
|
|
actor = kbActor(p)
|
|
}
|
|
sum := sha256.Sum256(args)
|
|
detail := fmt.Sprintf("arguments %d bytes, sha256 %s, %d ms", len(args), hex.EncodeToString(sum[:6]), took.Milliseconds())
|
|
if why != "" {
|
|
detail += ": " + why
|
|
}
|
|
if err := g.st.WriteAudit(context.WithoutCancel(ctx), storepkg.AuditEvent{Actor: actor, Source: "mcp",
|
|
Action: "call " + tool, Target: "mcp:" + server, Detail: detail, Status: status}); err != nil {
|
|
g.log.Warn("could not record an MCP tool call in the audit log", "tool", tool, "err", err)
|
|
}
|
|
}
|
|
|
|
// inspectResult has the guardrails read each text of a tool's answer,
|
|
// redacting it in place; it says why when the answer is not to be given.
|
|
func (g *mcpGateway) inspectResult(req *http.Request, name string, result map[string]any) string {
|
|
content, _ := result["content"].([]any)
|
|
for _, c := range content {
|
|
item, _ := c.(map[string]any)
|
|
text, _ := item["text"].(string)
|
|
if item["type"] != "text" || text == "" {
|
|
continue
|
|
}
|
|
out, refused, err := g.inspect(req, "/mcp/tool-answer/"+name, text)
|
|
switch {
|
|
case err != nil:
|
|
return "it could not be checked against this gateway's policies"
|
|
case refused != "":
|
|
return refused
|
|
}
|
|
item["text"] = out
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// inspectToolText is the guardrails reading a tool's text as a prompt, under
|
|
// the policy in force for the caller and a path of its own, so a policy can
|
|
// be attached to tools alone.
|
|
func (r *Router) inspectToolText(req *http.Request, path, text string) (string, string, error) {
|
|
if r.guard == nil || strings.TrimSpace(text) == "" {
|
|
return text, "", nil
|
|
}
|
|
s := callerSubject(req)
|
|
s.Path = path
|
|
body, _ := json.Marshal(map[string]any{"messages": []any{map[string]any{"role": "user", "content": text}}})
|
|
v, err := r.guard.Input(req.Context(), s, body)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
if v.Shadow {
|
|
return text, "", nil
|
|
}
|
|
if v.Action == guard.Block {
|
|
msg := v.Message
|
|
if msg == "" {
|
|
msg = "refused by policy " + v.Policy
|
|
}
|
|
return "", msg, nil
|
|
}
|
|
if v.Body != nil {
|
|
var redacted struct {
|
|
Messages []struct {
|
|
Content string `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
if json.Unmarshal(v.Body, &redacted) == nil && len(redacted.Messages) > 0 {
|
|
return redacted.Messages[0].Content, "", nil
|
|
}
|
|
}
|
|
return text, "", nil
|
|
}
|