package router import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "log/slog" "maps" "net/http" "slices" "strings" "sync" "time" "github.com/prometheus/client_golang/prometheus" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/mcpclient" "gitea.mynodes.xyz/ilker/llmgw/internal/secrets" storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // The company's own MCP servers behind llmgw's /mcp. An administrator // registers each once (llmgw ctl mcp add); /mcp then lists its tools as // server.tool beside llmgw's own, to a caller whose key is granted them, and // calls them with the server's credentials, which never leave the control // plane. Every call is in the audit log. const ( // mcpServersFresh is how long the server list read from the store is // used before it is read again: a server added on one control plane is // served by every one within it. mcpServersFresh = 30 * time.Second // mcpToolsFresh is how long a server's tool list is used. A server that // cannot be asked keeps the last list it gave. mcpToolsFresh = 5 * time.Minute // mcpCallTimeout is a call's bound when the server's own is not set. mcpCallTimeout = 60 * time.Second // mcpAnswerBytes bounds one tool's answer: it becomes a prompt. mcpAnswerBytes = 4 << 20 // mcpBreakAfter failures in a row take a server out for mcpBreakFor: // its tools leave the list, and a call to one is answered at once. mcpBreakAfter = 5 mcpBreakFor = time.Minute ) // mcpInspect has the guardrails read a tool's text: what goes to the company // system, and what comes back to the agent as its prompt. It answers the // text to use (redacted, if the policy redacts), or why it is refused. type mcpInspect func(req *http.Request, path, text string) (out, refused string, err error) type mcpStore interface { MCPServers(ctx context.Context) ([]storepkg.MCPServer, error) WriteAudit(ctx context.Context, e storepkg.AuditEvent) error } // mcpGateway is the company servers as /mcp serves them. type mcpGateway struct { st mcpStore log *slog.Logger http *http.Client now func() time.Time calls *prometheus.CounterVec duration *prometheus.HistogramVec mu sync.Mutex readAt time.Time upstream map[string]*mcpUpstream // local is this cluster's own MCP server, the admin API's, which a // server registered as llmgw:admin is answered by in this process. local http.Handler // inspect is the guardrails; nil reads nothing. inspect mcpInspect // calledAt is each caller's recent calls to each server, for its rate. calledAt map[string][]time.Time } // SetAdminMCP is the cluster's own read-only MCP server, served to keys // granted it once an administrator registers it as llmgw:admin. func (r *Router) SetAdminMCP(h http.Handler) { if r.mcpgw == nil { return } r.mcpgw.mu.Lock() defer r.mcpgw.mu.Unlock() r.mcpgw.local = h r.mcpgw.readAt = time.Time{} } // mcpUpstream is one server: its settings, its client, its tools. type mcpUpstream struct { cfg storepkg.MCPServer client *mcpclient.Client mu sync.Mutex tools []mcpclient.Tool toolsAt time.Time toolErr error // fails is the calls that failed in a row, and brokenUntil when a // server taken out for them is tried again. fails int brokenUntil time.Time } // broken is a server taken out for failing. func (u *mcpUpstream) broken(now time.Time) bool { u.mu.Lock() defer u.mu.Unlock() return now.Before(u.brokenUntil) } // result counts a call that reached the server or did not. func (u *mcpUpstream) result(ok bool, now time.Time) (justBroke bool) { u.mu.Lock() defer u.mu.Unlock() if ok { u.fails = 0 return false } u.fails++ if u.fails >= mcpBreakAfter && !now.Before(u.brokenUntil) { u.brokenUntil, u.fails = now.Add(mcpBreakFor), 0 return true } return false } // allow is whether a caller may call a server's tools once more this minute. func (g *mcpGateway) allow(caller string, u *mcpUpstream) bool { if u.cfg.RatePerMinute <= 0 { return true } g.mu.Lock() defer g.mu.Unlock() if g.calledAt == nil { g.calledAt = map[string][]time.Time{} } key, now := caller+"|"+u.cfg.Name, g.now() recent := g.calledAt[key][:0] for _, t := range g.calledAt[key] { if now.Sub(t) < time.Minute { recent = append(recent, t) } } if len(recent) >= u.cfg.RatePerMinute { g.calledAt[key] = recent return false } g.calledAt[key] = append(recent, now) return true } func newMCPGateway(st mcpStore, log *slog.Logger) *mcpGateway { return &mcpGateway{ st: st, log: log, http: &http.Client{}, now: time.Now, upstream: map[string]*mcpUpstream{}, calls: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "llmgw_mcp_tool_calls_total", Help: "Calls to the company MCP servers' tools through /mcp, by server, tool and outcome (ok, tool_error, error, denied).", }, []string{"server", "tool", "outcome"}), duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "llmgw_mcp_tool_call_seconds", Help: "How long a company MCP server took to answer a tool call.", Buckets: prometheus.ExponentialBuckets(0.05, 2, 12), }, []string{"server", "tool"}), } } func (g *mcpGateway) collectors() []prometheus.Collector { return []prometheus.Collector{g.calls, g.duration} } // servers are the enabled servers, read from the store when the list is // stale; a store that does not answer leaves the last list. func (g *mcpGateway) servers(ctx context.Context) []*mcpUpstream { g.mu.Lock() defer g.mu.Unlock() if g.now().Sub(g.readAt) >= mcpServersFresh { list, err := g.st.MCPServers(ctx) if err != nil { g.log.Warn("could not read the MCP servers; serving the last list", "err", err) } else { g.readAt = g.now() next := map[string]*mcpUpstream{} for _, s := range list { if old, ok := g.upstream[s.Name]; ok && sameMCPServer(old.cfg, s) { old.cfg = s next[s.Name] = old continue } next[s.Name] = g.upstreamFor(s) } g.upstream = next } } var out []*mcpUpstream for _, name := range slices.Sorted(maps.Keys(g.upstream)) { if u := g.upstream[name]; u.cfg.Enabled { out = append(out, u) } } return out } // sameMCPServer is whether a client made for a can serve b: the same address // and headers. A changed description or switch keeps the session. func sameMCPServer(a, b storepkg.MCPServer) bool { return a.URL == b.URL && maps.Equal(a.Headers, b.Headers) && maps.Equal(a.HeaderRefs, b.HeaderRefs) } func (g *mcpGateway) upstreamFor(s storepkg.MCPServer) *mcpUpstream { cfg := s url, client := s.URL, g.http if s.URL == adminapi.AdminMCPAddress && g.local != nil { url, client = adminapi.InProcessURL, &http.Client{Transport: adminapi.InProcess(g.local)} } return &mcpUpstream{cfg: s, client: &mcpclient.Client{ URL: url, HTTP: client, MaxBytes: mcpAnswerBytes, // Read at each request, so a rotated credential is used at once. Header: func(ctx context.Context) (http.Header, error) { h := http.Header{} for k, v := range cfg.Headers { h.Set(k, v) } for k, ref := range cfg.HeaderRefs { v, err := secrets.ReadString(ctx, ref) if err != nil { return nil, fmt.Errorf("the %s header of MCP server %s: %w", k, cfg.Name, err) } h.Set(k, v) } return h, nil }, }} } // toolsOf is a server's tools, asked for again when stale. func (u *mcpUpstream) toolsOf(ctx context.Context, now time.Time, log *slog.Logger) []mcpclient.Tool { u.mu.Lock() defer u.mu.Unlock() if now.Sub(u.toolsAt) < mcpToolsFresh && (u.tools != nil || u.toolErr != nil) { return u.tools } ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() tools, err := u.client.Tools(ctx) u.toolsAt, u.toolErr = now, err if err != nil { log.Warn("could not list an MCP server's tools; serving the last list", "server", u.cfg.Name, "err", err) return u.tools } u.tools = tools return tools } // list is the company tools this caller may use, as /mcp lists them. No // caller (authentication off) is granted none: they are granted, not assumed. func (g *mcpGateway) list(req *http.Request) []map[string]any { p, ok := auth.FromContext(req.Context()) if g == nil || !ok || len(p.Limits.ToolGrants) == 0 { return nil } var out []map[string]any for _, u := range g.servers(req.Context()) { if u.broken(g.now()) { continue } for _, t := range u.toolsOf(req.Context(), g.now(), g.log) { name := u.cfg.Name + "." + t.Name if !p.Limits.AllowsTool(name) { continue } desc := t.Description if u.cfg.Description != "" { desc = "[" + u.cfg.Description + "] " + desc } tool := map[string]any{"name": name, "description": desc} if len(t.InputSchema) > 0 { tool["inputSchema"] = t.InputSchema } else { tool["inputSchema"] = map[string]any{"type": "object"} } if t.Title != "" { tool["title"] = t.Title } if len(t.Annotations) > 0 { tool["annotations"] = t.Annotations } out = append(out, tool) } } return out } // isCompanyTool is a tool name with a server's prefix: llmgw's own have no dot. func isCompanyTool(name string) bool { return strings.Contains(name, ".") } // call calls a company tool for the caller and gives the tool's result as // /mcp answers it: the server's own, or a tool error saying what went wrong. func (g *mcpGateway) call(req *http.Request, name string, args json.RawMessage, progress func(json.RawMessage)) any { server, tool, _ := strings.Cut(name, ".") p, ok := auth.FromContext(req.Context()) if g == nil || !ok || !p.Limits.AllowsTool(name) { if g != nil { g.calls.WithLabelValues(server, tool, "denied").Inc() g.audit(req.Context(), p, ok, name, server, args, 0, http.StatusForbidden, "not granted") } return toolText(name+" is not a tool this key may call", true) } var u *mcpUpstream for _, s := range g.servers(req.Context()) { if s.cfg.Name == server { u = s } } if u == nil { return toolText("no MCP server "+server+" is registered and on", true) } if u.broken(g.now()) { g.calls.WithLabelValues(server, tool, "broken").Inc() return toolText(fmt.Sprintf("the MCP server %s failed %d calls in a row and is tried again within %s", server, mcpBreakAfter, mcpBreakFor), true) } if !g.allow(kbActor(p), u) { g.calls.WithLabelValues(server, tool, "rate_limited").Inc() g.audit(req.Context(), p, true, name, server, args, 0, http.StatusTooManyRequests, "rate limited") return toolText(fmt.Sprintf("more than %d calls a minute to %s: try again shortly", u.cfg.RatePerMinute, server), true) } // What goes to the company system first: a secret pasted into an // argument does not leave. if g.inspect != nil { out, refused, err := g.inspect(req, "/mcp/tool-arguments/"+name, string(args)) switch { case err != nil: g.calls.WithLabelValues(server, tool, "denied").Inc() return toolText("the call to "+name+" could not be checked against this gateway's policies; try again", true) case refused != "": g.calls.WithLabelValues(server, tool, "denied").Inc() g.audit(req.Context(), p, true, name, server, args, 0, http.StatusBadRequest, "arguments refused by policy") return toolText("the call to "+name+" was refused: "+refused, true) case out != string(args) && json.Valid([]byte(out)): args = json.RawMessage(out) } } timeout := u.cfg.Timeout if timeout <= 0 { timeout = mcpCallTimeout } ctx, cancel := context.WithTimeout(req.Context(), timeout) defer cancel() // Who is calling, for the server's own log; its credentials are llmgw's. who := http.Header{"X-Llmgw-Key": {p.KeyID}} if p.UserID != "" { who.Set("X-Llmgw-User", p.UserID) } if p.ProjectID != "" { who.Set("X-Llmgw-Project", p.ProjectID) } start := g.now() raw, err := u.client.Call(ctx, tool, args, who, progress) took := g.now().Sub(start) g.duration.WithLabelValues(server, tool).Observe(took.Seconds()) var rpcErr *mcpclient.Error if u.result(err == nil || errors.As(err, &rpcErr), g.now()) { g.log.Warn("an MCP server failed calls in a row; its tools are out for a while", "server", server, "failures", mcpBreakAfter, "for", mcpBreakFor) } switch { case err != nil: g.calls.WithLabelValues(server, tool, "error").Inc() g.audit(req.Context(), p, true, name, server, args, took, http.StatusBadGateway, err.Error()) msg := err.Error() if errors.Is(ctx.Err(), context.DeadlineExceeded) { msg = fmt.Sprintf("the MCP server %s did not answer within %s", server, timeout) } return toolText(msg, true) } var result map[string]any if json.Unmarshal(raw, &result) != nil { g.calls.WithLabelValues(server, tool, "error").Inc() g.audit(req.Context(), p, true, name, server, args, took, http.StatusBadGateway, "the answer was not a tool result") return toolText("the MCP server "+server+" answered something that is not a tool result", true) } // The answer becomes the agent's prompt: read as one, so an // instruction planted in a ticket is caught as a pasted one would be. if g.inspect != nil { if why := g.inspectResult(req, name, result); why != "" { g.calls.WithLabelValues(server, tool, "withheld").Inc() g.audit(req.Context(), p, true, name, server, args, took, http.StatusBadGateway, "answer withheld by policy") return toolText("the answer of "+name+" was withheld: "+why, true) } } outcome, status := "ok", http.StatusOK if isErr, _ := result["isError"].(bool); isErr { outcome, status = "tool_error", http.StatusUnprocessableEntity } g.calls.WithLabelValues(server, tool, outcome).Inc() g.audit(req.Context(), p, true, name, server, args, took, status, "") return result } // audit records a call: who, which tool, the arguments' size and a digest of // them, how long, and how it ended. Never the arguments themselves: they are // the company's data. func (g *mcpGateway) audit(ctx context.Context, p auth.Principal, authed bool, tool, server string, args json.RawMessage, took time.Duration, status int, why string, ) { actor := "anonymous" if authed { actor = kbActor(p) } sum := sha256.Sum256(args) detail := fmt.Sprintf("arguments %d bytes, sha256 %s, %d ms", len(args), hex.EncodeToString(sum[:6]), took.Milliseconds()) if why != "" { detail += ": " + why } if err := g.st.WriteAudit(context.WithoutCancel(ctx), storepkg.AuditEvent{Actor: actor, Source: "mcp", Action: "call " + tool, Target: "mcp:" + server, Detail: detail, Status: status}); err != nil { g.log.Warn("could not record an MCP tool call in the audit log", "tool", tool, "err", err) } } // inspectResult has the guardrails read each text of a tool's answer, // redacting it in place; it says why when the answer is not to be given. func (g *mcpGateway) inspectResult(req *http.Request, name string, result map[string]any) string { content, _ := result["content"].([]any) for _, c := range content { item, _ := c.(map[string]any) text, _ := item["text"].(string) if item["type"] != "text" || text == "" { continue } out, refused, err := g.inspect(req, "/mcp/tool-answer/"+name, text) switch { case err != nil: return "it could not be checked against this gateway's policies" case refused != "": return refused } item["text"] = out } return "" } // inspectToolText is the guardrails reading a tool's text as a prompt, under // the policy in force for the caller and a path of its own, so a policy can // be attached to tools alone. func (r *Router) inspectToolText(req *http.Request, path, text string) (string, string, error) { if r.guard == nil || strings.TrimSpace(text) == "" { return text, "", nil } s := callerSubject(req) s.Path = path body, _ := json.Marshal(map[string]any{"messages": []any{map[string]any{"role": "user", "content": text}}}) v, err := r.guard.Input(req.Context(), s, body) if err != nil { return "", "", err } if v.Shadow { return text, "", nil } if v.Action == guard.Block { msg := v.Message if msg == "" { msg = "refused by policy " + v.Policy } return "", msg, nil } if v.Body != nil { var redacted struct { Messages []struct { Content string `json:"content"` } `json:"messages"` } if json.Unmarshal(v.Body, &redacted) == nil && len(redacted.Messages) > 0 { return redacted.Messages[0].Content, "", nil } } return text, "", nil }