Fix an agent that could neither explain itself nor stop
Two bugs found while verifying mTLS against real machines. session cancelled its own derived context and then asked that context whether the agent was shutting down. The answer was always yes, so every stream error read as a clean close: the log said err=<nil>, the backoff reset, and an agent whose certificate the control plane refused hammered it forever at the base interval with nothing to diagnose. It now keeps the parent context to answer that question, and a refusal reaches the operator naming both the certificate and the claim. Cancelling the context also did not stop the agent. HTTP/2 watches the context while the request body is being written, and a bidirectional stream spends its life blocked on the read side instead, so the read never returned. Measured on a deployed node: `systemctl stop llmgw-agent` took 90 seconds and ended in SIGKILL, on every restart. Closing the response when the context ends is what actually unblocks it. TestAgentStopsWhenContextEnds hangs reliably without the second fix, which is the only reason to trust that it tests it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1e3cafdc86
commit
ec1b6b04df
9 files changed
+515
-27
No files matched your search
+25
-6
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -93,7 +94,7 @@ func newServerCmd() *cobra.Command {
|
||||
return err
|
||||
}
|
||||
|
||||
ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, false, log)
|
||||
ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, cfg.AgentTLS.Enabled(), log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -102,7 +103,8 @@ func newServerCmd() *cobra.Command {
|
||||
agentPath, agentHandler = svc.Handler()
|
||||
|
||||
log.Info("control plane running",
|
||||
"models", len(cfg.Models), "agent_listen", cfg.AgentListen)
|
||||
"models", len(cfg.Models), "agent_listen", cfg.AgentListen,
|
||||
"agent_mtls", cfg.AgentTLS.Enabled())
|
||||
}
|
||||
|
||||
adminMux, err := rt.AdminHandler()
|
||||
@@ -147,17 +149,34 @@ func newServerCmd() *cobra.Command {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(agentPath, agentHandler)
|
||||
agentSrv = &http.Server{
|
||||
// Agents speak plaintext HTTP/2, which Go will not do
|
||||
// without being asked. mTLS replaces this (plan §5.6);
|
||||
// until then the agent port must stay on a trusted network.
|
||||
Handler: h2c.NewHandler(mux, &http2.Server{}),
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
if !cfg.AgentTLS.Enabled() {
|
||||
// Without TLS, agents speak plaintext HTTP/2, which Go will
|
||||
// not do unless asked. Nothing then stops a node from
|
||||
// claiming to be another one, so this belongs on a trusted
|
||||
// network only.
|
||||
agentSrv.Handler = h2c.NewHandler(mux, &http2.Server{})
|
||||
}
|
||||
if agentLn, err = net.Listen("tcp", cfg.AgentListen); err != nil {
|
||||
_ = apiLn.Close()
|
||||
_ = adminLn.Close()
|
||||
return fmt.Errorf("listening on %s: %w", cfg.AgentListen, err)
|
||||
}
|
||||
if cfg.AgentTLS.Enabled() {
|
||||
tc, terr := cfg.AgentTLS.ServerConfig()
|
||||
if terr != nil {
|
||||
_ = apiLn.Close()
|
||||
_ = adminLn.Close()
|
||||
_ = agentLn.Close()
|
||||
return terr
|
||||
}
|
||||
// Serve on an already-wrapped listener rather than
|
||||
// ServeTLS: agentSrv.TLSConfig stays nil, which is exactly
|
||||
// the case where net/http sets up HTTP/2 for ALPN itself.
|
||||
agentLn = tls.NewListener(agentLn, tc)
|
||||
}
|
||||
}
|
||||
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
|
||||
Reference in new issue
Block a user