Files
llmgw/internal/router/mcp_gateway.go
T
Ilker ManapandClaude Opus 5.5 3a4488b41e
CI / test (push) Successful in 5m35s
The company's MCP tools under the guardrails, a rate and a breaker
A tool's answer is read by the caller's policies as the prompt it
becomes, so an instruction planted in a ticket is withheld and personal
data redacted; its arguments are read before they leave. ctl mcp add
--rate bounds each caller's calls a minute to a server (migration 0050),
and five failures in a row take a server out for a minute, its tools
off the list.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-20 23:00:22 +02:00

486 lines
16 KiB
Go

package router
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log/slog"
"maps"
"net/http"
"slices"
"strings"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/mcpclient"
"gitea.mynodes.xyz/ilker/llmgw/internal/secrets"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// The company's own MCP servers behind llmgw's /mcp. An administrator
// registers each once (llmgw ctl mcp add); /mcp then lists its tools as
// server.tool beside llmgw's own, to a caller whose key is granted them, and
// calls them with the server's credentials, which never leave the control
// plane. Every call is in the audit log.
const (
// mcpServersFresh is how long the server list read from the store is
// used before it is read again: a server added on one control plane is
// served by every one within it.
mcpServersFresh = 30 * time.Second
// mcpToolsFresh is how long a server's tool list is used. A server that
// cannot be asked keeps the last list it gave.
mcpToolsFresh = 5 * time.Minute
// mcpCallTimeout is a call's bound when the server's own is not set.
mcpCallTimeout = 60 * time.Second
// mcpAnswerBytes bounds one tool's answer: it becomes a prompt.
mcpAnswerBytes = 4 << 20
// mcpBreakAfter failures in a row take a server out for mcpBreakFor:
// its tools leave the list, and a call to one is answered at once.
mcpBreakAfter = 5
mcpBreakFor = time.Minute
)
// mcpInspect has the guardrails read a tool's text: what goes to the company
// system, and what comes back to the agent as its prompt. It answers the
// text to use (redacted, if the policy redacts), or why it is refused.
type mcpInspect func(req *http.Request, path, text string) (out, refused string, err error)
type mcpStore interface {
MCPServers(ctx context.Context) ([]storepkg.MCPServer, error)
WriteAudit(ctx context.Context, e storepkg.AuditEvent) error
}
// mcpGateway is the company servers as /mcp serves them.
type mcpGateway struct {
st mcpStore
log *slog.Logger
http *http.Client
now func() time.Time
calls *prometheus.CounterVec
duration *prometheus.HistogramVec
mu sync.Mutex
readAt time.Time
upstream map[string]*mcpUpstream
// local is this cluster's own MCP server, the admin API's, which a
// server registered as llmgw:admin is answered by in this process.
local http.Handler
// inspect is the guardrails; nil reads nothing.
inspect mcpInspect
// calledAt is each caller's recent calls to each server, for its rate.
calledAt map[string][]time.Time
}
// SetAdminMCP is the cluster's own read-only MCP server, served to keys
// granted it once an administrator registers it as llmgw:admin.
func (r *Router) SetAdminMCP(h http.Handler) {
if r.mcpgw == nil {
return
}
r.mcpgw.mu.Lock()
defer r.mcpgw.mu.Unlock()
r.mcpgw.local = h
r.mcpgw.readAt = time.Time{}
}
// mcpUpstream is one server: its settings, its client, its tools.
type mcpUpstream struct {
cfg storepkg.MCPServer
client *mcpclient.Client
mu sync.Mutex
tools []mcpclient.Tool
toolsAt time.Time
toolErr error
// fails is the calls that failed in a row, and brokenUntil when a
// server taken out for them is tried again.
fails int
brokenUntil time.Time
}
// broken is a server taken out for failing.
func (u *mcpUpstream) broken(now time.Time) bool {
u.mu.Lock()
defer u.mu.Unlock()
return now.Before(u.brokenUntil)
}
// result counts a call that reached the server or did not.
func (u *mcpUpstream) result(ok bool, now time.Time) (justBroke bool) {
u.mu.Lock()
defer u.mu.Unlock()
if ok {
u.fails = 0
return false
}
u.fails++
if u.fails >= mcpBreakAfter && !now.Before(u.brokenUntil) {
u.brokenUntil, u.fails = now.Add(mcpBreakFor), 0
return true
}
return false
}
// allow is whether a caller may call a server's tools once more this minute.
func (g *mcpGateway) allow(caller string, u *mcpUpstream) bool {
if u.cfg.RatePerMinute <= 0 {
return true
}
g.mu.Lock()
defer g.mu.Unlock()
if g.calledAt == nil {
g.calledAt = map[string][]time.Time{}
}
key, now := caller+"|"+u.cfg.Name, g.now()
recent := g.calledAt[key][:0]
for _, t := range g.calledAt[key] {
if now.Sub(t) < time.Minute {
recent = append(recent, t)
}
}
if len(recent) >= u.cfg.RatePerMinute {
g.calledAt[key] = recent
return false
}
g.calledAt[key] = append(recent, now)
return true
}
func newMCPGateway(st mcpStore, log *slog.Logger) *mcpGateway {
return &mcpGateway{
st: st, log: log, http: &http.Client{}, now: time.Now, upstream: map[string]*mcpUpstream{},
calls: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_mcp_tool_calls_total",
Help: "Calls to the company MCP servers' tools through /mcp, by server, tool and outcome (ok, tool_error, error, denied).",
}, []string{"server", "tool", "outcome"}),
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "llmgw_mcp_tool_call_seconds",
Help: "How long a company MCP server took to answer a tool call.",
Buckets: prometheus.ExponentialBuckets(0.05, 2, 12),
}, []string{"server", "tool"}),
}
}
func (g *mcpGateway) collectors() []prometheus.Collector {
return []prometheus.Collector{g.calls, g.duration}
}
// servers are the enabled servers, read from the store when the list is
// stale; a store that does not answer leaves the last list.
func (g *mcpGateway) servers(ctx context.Context) []*mcpUpstream {
g.mu.Lock()
defer g.mu.Unlock()
if g.now().Sub(g.readAt) >= mcpServersFresh {
list, err := g.st.MCPServers(ctx)
if err != nil {
g.log.Warn("could not read the MCP servers; serving the last list", "err", err)
} else {
g.readAt = g.now()
next := map[string]*mcpUpstream{}
for _, s := range list {
if old, ok := g.upstream[s.Name]; ok && sameMCPServer(old.cfg, s) {
old.cfg = s
next[s.Name] = old
continue
}
next[s.Name] = g.upstreamFor(s)
}
g.upstream = next
}
}
var out []*mcpUpstream
for _, name := range slices.Sorted(maps.Keys(g.upstream)) {
if u := g.upstream[name]; u.cfg.Enabled {
out = append(out, u)
}
}
return out
}
// sameMCPServer is whether a client made for a can serve b: the same address
// and headers. A changed description or switch keeps the session.
func sameMCPServer(a, b storepkg.MCPServer) bool {
return a.URL == b.URL && maps.Equal(a.Headers, b.Headers) && maps.Equal(a.HeaderRefs, b.HeaderRefs)
}
func (g *mcpGateway) upstreamFor(s storepkg.MCPServer) *mcpUpstream {
cfg := s
url, client := s.URL, g.http
if s.URL == adminapi.AdminMCPAddress && g.local != nil {
url, client = adminapi.InProcessURL, &http.Client{Transport: adminapi.InProcess(g.local)}
}
return &mcpUpstream{cfg: s, client: &mcpclient.Client{
URL: url, HTTP: client, MaxBytes: mcpAnswerBytes,
// Read at each request, so a rotated credential is used at once.
Header: func(ctx context.Context) (http.Header, error) {
h := http.Header{}
for k, v := range cfg.Headers {
h.Set(k, v)
}
for k, ref := range cfg.HeaderRefs {
v, err := secrets.ReadString(ctx, ref)
if err != nil {
return nil, fmt.Errorf("the %s header of MCP server %s: %w", k, cfg.Name, err)
}
h.Set(k, v)
}
return h, nil
},
}}
}
// toolsOf is a server's tools, asked for again when stale.
func (u *mcpUpstream) toolsOf(ctx context.Context, now time.Time, log *slog.Logger) []mcpclient.Tool {
u.mu.Lock()
defer u.mu.Unlock()
if now.Sub(u.toolsAt) < mcpToolsFresh && (u.tools != nil || u.toolErr != nil) {
return u.tools
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
tools, err := u.client.Tools(ctx)
u.toolsAt, u.toolErr = now, err
if err != nil {
log.Warn("could not list an MCP server's tools; serving the last list", "server", u.cfg.Name, "err", err)
return u.tools
}
u.tools = tools
return tools
}
// list is the company tools this caller may use, as /mcp lists them. No
// caller (authentication off) is granted none: they are granted, not assumed.
func (g *mcpGateway) list(req *http.Request) []map[string]any {
p, ok := auth.FromContext(req.Context())
if g == nil || !ok || len(p.Limits.ToolGrants) == 0 {
return nil
}
var out []map[string]any
for _, u := range g.servers(req.Context()) {
if u.broken(g.now()) {
continue
}
for _, t := range u.toolsOf(req.Context(), g.now(), g.log) {
name := u.cfg.Name + "." + t.Name
if !p.Limits.AllowsTool(name) {
continue
}
desc := t.Description
if u.cfg.Description != "" {
desc = "[" + u.cfg.Description + "] " + desc
}
tool := map[string]any{"name": name, "description": desc}
if len(t.InputSchema) > 0 {
tool["inputSchema"] = t.InputSchema
} else {
tool["inputSchema"] = map[string]any{"type": "object"}
}
if t.Title != "" {
tool["title"] = t.Title
}
if len(t.Annotations) > 0 {
tool["annotations"] = t.Annotations
}
out = append(out, tool)
}
}
return out
}
// isCompanyTool is a tool name with a server's prefix: llmgw's own have no dot.
func isCompanyTool(name string) bool { return strings.Contains(name, ".") }
// call calls a company tool for the caller and gives the tool's result as
// /mcp answers it: the server's own, or a tool error saying what went wrong.
func (g *mcpGateway) call(req *http.Request, name string, args json.RawMessage, progress func(json.RawMessage)) any {
server, tool, _ := strings.Cut(name, ".")
p, ok := auth.FromContext(req.Context())
if g == nil || !ok || !p.Limits.AllowsTool(name) {
if g != nil {
g.calls.WithLabelValues(server, tool, "denied").Inc()
g.audit(req.Context(), p, ok, name, server, args, 0, http.StatusForbidden, "not granted")
}
return toolText(name+" is not a tool this key may call", true)
}
var u *mcpUpstream
for _, s := range g.servers(req.Context()) {
if s.cfg.Name == server {
u = s
}
}
if u == nil {
return toolText("no MCP server "+server+" is registered and on", true)
}
if u.broken(g.now()) {
g.calls.WithLabelValues(server, tool, "broken").Inc()
return toolText(fmt.Sprintf("the MCP server %s failed %d calls in a row and is tried again within %s",
server, mcpBreakAfter, mcpBreakFor), true)
}
if !g.allow(kbActor(p), u) {
g.calls.WithLabelValues(server, tool, "rate_limited").Inc()
g.audit(req.Context(), p, true, name, server, args, 0, http.StatusTooManyRequests, "rate limited")
return toolText(fmt.Sprintf("more than %d calls a minute to %s: try again shortly", u.cfg.RatePerMinute, server), true)
}
// What goes to the company system first: a secret pasted into an
// argument does not leave.
if g.inspect != nil {
out, refused, err := g.inspect(req, "/mcp/tool-arguments/"+name, string(args))
switch {
case err != nil:
g.calls.WithLabelValues(server, tool, "denied").Inc()
return toolText("the call to "+name+" could not be checked against this gateway's policies; try again", true)
case refused != "":
g.calls.WithLabelValues(server, tool, "denied").Inc()
g.audit(req.Context(), p, true, name, server, args, 0, http.StatusBadRequest, "arguments refused by policy")
return toolText("the call to "+name+" was refused: "+refused, true)
case out != string(args) && json.Valid([]byte(out)):
args = json.RawMessage(out)
}
}
timeout := u.cfg.Timeout
if timeout <= 0 {
timeout = mcpCallTimeout
}
ctx, cancel := context.WithTimeout(req.Context(), timeout)
defer cancel()
// Who is calling, for the server's own log; its credentials are llmgw's.
who := http.Header{"X-Llmgw-Key": {p.KeyID}}
if p.UserID != "" {
who.Set("X-Llmgw-User", p.UserID)
}
if p.ProjectID != "" {
who.Set("X-Llmgw-Project", p.ProjectID)
}
start := g.now()
raw, err := u.client.Call(ctx, tool, args, who, progress)
took := g.now().Sub(start)
g.duration.WithLabelValues(server, tool).Observe(took.Seconds())
var rpcErr *mcpclient.Error
if u.result(err == nil || errors.As(err, &rpcErr), g.now()) {
g.log.Warn("an MCP server failed calls in a row; its tools are out for a while",
"server", server, "failures", mcpBreakAfter, "for", mcpBreakFor)
}
switch {
case err != nil:
g.calls.WithLabelValues(server, tool, "error").Inc()
g.audit(req.Context(), p, true, name, server, args, took, http.StatusBadGateway, err.Error())
msg := err.Error()
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
msg = fmt.Sprintf("the MCP server %s did not answer within %s", server, timeout)
}
return toolText(msg, true)
}
var result map[string]any
if json.Unmarshal(raw, &result) != nil {
g.calls.WithLabelValues(server, tool, "error").Inc()
g.audit(req.Context(), p, true, name, server, args, took, http.StatusBadGateway, "the answer was not a tool result")
return toolText("the MCP server "+server+" answered something that is not a tool result", true)
}
// The answer becomes the agent's prompt: read as one, so an
// instruction planted in a ticket is caught as a pasted one would be.
if g.inspect != nil {
if why := g.inspectResult(req, name, result); why != "" {
g.calls.WithLabelValues(server, tool, "withheld").Inc()
g.audit(req.Context(), p, true, name, server, args, took, http.StatusBadGateway, "answer withheld by policy")
return toolText("the answer of "+name+" was withheld: "+why, true)
}
}
outcome, status := "ok", http.StatusOK
if isErr, _ := result["isError"].(bool); isErr {
outcome, status = "tool_error", http.StatusUnprocessableEntity
}
g.calls.WithLabelValues(server, tool, outcome).Inc()
g.audit(req.Context(), p, true, name, server, args, took, status, "")
return result
}
// audit records a call: who, which tool, the arguments' size and a digest of
// them, how long, and how it ended. Never the arguments themselves: they are
// the company's data.
func (g *mcpGateway) audit(ctx context.Context, p auth.Principal, authed bool, tool, server string,
args json.RawMessage, took time.Duration, status int, why string,
) {
actor := "anonymous"
if authed {
actor = kbActor(p)
}
sum := sha256.Sum256(args)
detail := fmt.Sprintf("arguments %d bytes, sha256 %s, %d ms", len(args), hex.EncodeToString(sum[:6]), took.Milliseconds())
if why != "" {
detail += ": " + why
}
if err := g.st.WriteAudit(context.WithoutCancel(ctx), storepkg.AuditEvent{Actor: actor, Source: "mcp",
Action: "call " + tool, Target: "mcp:" + server, Detail: detail, Status: status}); err != nil {
g.log.Warn("could not record an MCP tool call in the audit log", "tool", tool, "err", err)
}
}
// inspectResult has the guardrails read each text of a tool's answer,
// redacting it in place; it says why when the answer is not to be given.
func (g *mcpGateway) inspectResult(req *http.Request, name string, result map[string]any) string {
content, _ := result["content"].([]any)
for _, c := range content {
item, _ := c.(map[string]any)
text, _ := item["text"].(string)
if item["type"] != "text" || text == "" {
continue
}
out, refused, err := g.inspect(req, "/mcp/tool-answer/"+name, text)
switch {
case err != nil:
return "it could not be checked against this gateway's policies"
case refused != "":
return refused
}
item["text"] = out
}
return ""
}
// inspectToolText is the guardrails reading a tool's text as a prompt, under
// the policy in force for the caller and a path of its own, so a policy can
// be attached to tools alone.
func (r *Router) inspectToolText(req *http.Request, path, text string) (string, string, error) {
if r.guard == nil || strings.TrimSpace(text) == "" {
return text, "", nil
}
s := callerSubject(req)
s.Path = path
body, _ := json.Marshal(map[string]any{"messages": []any{map[string]any{"role": "user", "content": text}}})
v, err := r.guard.Input(req.Context(), s, body)
if err != nil {
return "", "", err
}
if v.Shadow {
return text, "", nil
}
if v.Action == guard.Block {
msg := v.Message
if msg == "" {
msg = "refused by policy " + v.Policy
}
return "", msg, nil
}
if v.Body != nil {
var redacted struct {
Messages []struct {
Content string `json:"content"`
} `json:"messages"`
}
if json.Unmarshal(v.Body, &redacted) == nil && len(redacted.Messages) > 0 {
return redacted.Messages[0].Content, "", nil
}
}
return text, "", nil
}