Agent stream: connect the fleet to the control plane

One bidirectional connect-go stream per node, carrying registration, heartbeats
and shard status up, and start, stop and drain down. The agent dials out: GPU
nodes sit behind firewalls far more often than a control plane does, and an
outbound connection needs no inbound rule on every node in the fleet.

- registration carries the whole hardware inventory, so there is no separate
  discovery call and a reconnecting agent refreshes what the scheduler knows.
  The fabric labels travel with it; they cannot be inferred from the wire, so
  losing them in transit would silently disable multi-node placement.
- agentsvc is both the stream handler and the reconciler's dispatcher, because
  it holds the open streams and is therefore the only thing that can deliver a
  command to a node.
- the agent validates a profile before launching anything, since an engine that
  refuses a profile does so after minutes of loading, then supervises the
  process and reports every transition with the engine's output tail on failure.
- reconnects forever with jittered backoff. A control plane restart must not
  require touching every GPU node, and a fleet must not reconnect in lockstep
  and knock it over again. Engines keep serving throughout, because the router
  talks to them directly.

llmgw agent is real now, and llmgw server runs a control plane whenever any
model declares a profile. The static endpoint list stays for router-only runs.

Verified with real binaries as separate processes: the agent registers, a
replica is placed, an engine launches through the vLLM adapter's actual argument
construction, streaming traffic is served, and killing the agent moves the node
through unreachable to lost with placement explaining why nothing can run.

config.Model and catalog.Model were the same thing described twice, so the
router now reads catalog entries directly and the validation rules live in one
place.

Still missing: mTLS. The stream is plaintext HTTP/2 and must stay on a trusted
network until that lands.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ilkerandClaude Opus 5 committed 2026-09-16 02:48:21 +02:00
1 parent c1b9ea1bc2
commit 2373726efd
24 files changed
+3645 -87

No files matched your search

+69 -8
View File
@@ -1,31 +1,92 @@
package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/signal"
"syscall"
"text/tabwriter"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
func newAgentCmd() *cobra.Command {
var configPath string
var (
server string
nodeID string
mgmtAddr string
)
cmd := &cobra.Command{
Use: "agent",
Short: "Node agent: discovery and engine supervision (phase 2)",
Long: "Runs on every GPU node. It discovers the hardware, starts and supervises\n" +
"inference engines, and reports to the control plane.\n\n" +
"The control plane connection is not built yet, so for now the useful\n" +
"subcommand is `agent inventory`.",
RunE: func(_ *cobra.Command, _ []string) error {
return errNotImplemented
Short: "Node agent: discovery and engine supervision",
Long: "Runs on every GPU node. It discovers the hardware, connects to the control\n" +
"plane, and starts, supervises and stops inference engines on demand.\n\n" +
"The agent dials out and keeps one stream open. Engines keep serving if the\n" +
"agent restarts or the control plane goes away: the router talks to them\n" +
"directly, and orphan adoption picks them up again.",
RunE: func(cmd *cobra.Command, _ []string) error {
log := logger()
cfg, err := config.LoadAgent(configPath)
if err != nil {
return err
}
if server != "" {
cfg.Server = server
}
if nodeID != "" {
cfg.NodeID = nodeID
}
if mgmtAddr != "" {
cfg.MgmtAddr = mgmtAddr
}
if cfg.NodeID == "" {
host, err := os.Hostname()
if err != nil {
return fmt.Errorf("node_id is unset and the hostname is unavailable: %w", err)
}
cfg.NodeID = host
}
if cfg.Hostname == "" {
cfg.Hostname, _ = os.Hostname()
}
cfg.Version = version.Version
a, err := agent.New(cfg, agent.HTTP2Client(30*time.Second), log)
if err != nil {
return err
}
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
log.Info("agent starting",
"node", cfg.NodeID, "server", cfg.Server,
"fabrics", len(cfg.Fabrics), "nvml", discovery.NVMLSupported())
err = a.Run(ctx)
if errors.Is(err, context.Canceled) {
return nil
}
return err
},
}
cmd.Flags().StringVar(&configPath, "config", "agent.yaml", "configuration file")
cmd.Flags().StringVar(&configPath, "config", "", "configuration file")
cmd.Flags().StringVar(&server, "server", "", "control plane address (overrides the config)")
cmd.Flags().StringVar(&nodeID, "node-id", "", "node id (defaults to the hostname)")
cmd.Flags().StringVar(&mgmtAddr, "mgmt-addr", "", "address engines are reachable on")
cmd.AddCommand(newInventoryCmd())
return cmd
}
+138 -2
View File
@@ -4,17 +4,25 @@ import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"os/signal"
"strings"
"sync"
"syscall"
"time"
"github.com/spf13/cobra"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
"golang.org/x/sync/errgroup"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
)
@@ -55,8 +63,10 @@ func newServerCmd() *cobra.Command {
if err := cfg.Validate(); err != nil {
return err
}
if len(cfg.Endpoints) == 0 {
return errors.New("no endpoints configured; use --endpoint or a config file")
controlMode := cfg.ControlPlaneMode()
if !controlMode && len(cfg.Endpoints) == 0 {
return errors.New("nothing to serve: declare models with profiles, " +
"or give --endpoint for a router-only run")
}
rt, err := router.New(cfg, log)
@@ -70,6 +80,28 @@ func newServerCmd() *cobra.Command {
rt.Start(ctx)
defer rt.Stop()
// The control plane replaces the static endpoint list: agents
// register, the reconciler places replicas, and Ready ones are
// published to the router (plan §6.4).
var agentHandler http.Handler
var agentPath string
if controlMode {
cat, err := cfg.Catalog()
if err != nil {
return err
}
ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, log)
if err != nil {
return err
}
defer ctl.Close()
agentPath, agentHandler = svc.Handler()
log.Info("control plane running",
"models", len(cfg.Models), "agent_listen", cfg.AgentListen)
}
adminHandler, err := rt.AdminHandler()
if err != nil {
return err
@@ -103,14 +135,39 @@ func newServerCmd() *cobra.Command {
"default_profile", cfg.Router.DefaultProfile,
)
var agentSrv *http.Server
var agentLn net.Listener
if agentHandler != nil {
mux := http.NewServeMux()
mux.Handle(agentPath, agentHandler)
agentSrv = &http.Server{
// Agents speak plaintext HTTP/2, which Go will not do
// without being asked. mTLS replaces this (plan §5.6);
// until then the agent port must stay on a trusted network.
Handler: h2c.NewHandler(mux, &http2.Server{}),
ReadHeaderTimeout: 10 * time.Second,
}
if agentLn, err = net.Listen("tcp", cfg.AgentListen); err != nil {
_ = apiLn.Close()
_ = adminLn.Close()
return fmt.Errorf("listening on %s: %w", cfg.AgentListen, err)
}
}
g, gctx := errgroup.WithContext(ctx)
g.Go(func() error { return serve(apiSrv, apiLn) })
g.Go(func() error { return serve(adminSrv, adminLn) })
if agentSrv != nil {
g.Go(func() error { return serve(agentSrv, agentLn) })
}
g.Go(func() error {
<-gctx.Done()
log.Info("shutting down")
sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if agentSrv != nil {
_ = agentSrv.Shutdown(sctx)
}
_ = adminSrv.Shutdown(sctx)
return apiSrv.Shutdown(sctx)
})
@@ -127,6 +184,85 @@ func newServerCmd() *cobra.Command {
return cmd
}
// startControlPlane builds the control plane and its agent service.
//
// The service is both the AgentService handler and the reconciler's Dispatcher:
// it holds the open streams, so it is the only thing that can deliver a command
// to a node.
func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Catalog,
sink control.EndpointSink, log *slog.Logger,
) (*control.Controller, *agentsvc.Service, error) {
ccfg := control.DefaultConfig()
ccfg.Reconciler = reconciler.Config{
Interval: cfg.Control.Interval,
BaseBackoff: cfg.Control.BaseBackoff,
MaxBackoff: cfg.Control.MaxBackoff,
UnreachableAfter: cfg.Control.UnreachableAfter,
LostAfter: cfg.Control.LostAfter,
MgmtInterface: cfg.Control.MgmtInterface,
}
if len(cfg.Control.PortRange) == 2 {
ccfg.Reconciler.PortRange = [2]int{cfg.Control.PortRange[0], cfg.Control.PortRange[1]}
}
// The service needs the inventory and the status sink, both of which the
// controller owns; the controller needs the service as its dispatcher. The
// cycle is broken by giving the controller a dispatcher that forwards to
// the service once it exists.
fwd := &dispatcherRef{}
ctl := control.New(ccfg, cat, fwd, sink, log)
svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), log)
fwd.set(svc)
ctl.Start(ctx)
return ctl, svc, nil
}
// dispatcherRef lets the controller be built before the service exists.
type dispatcherRef struct {
mu sync.RWMutex
d reconciler.Dispatcher
}
func (r *dispatcherRef) set(d reconciler.Dispatcher) {
r.mu.Lock()
r.d = d
r.mu.Unlock()
}
func (r *dispatcherRef) get() (reconciler.Dispatcher, error) {
r.mu.RLock()
defer r.mu.RUnlock()
if r.d == nil {
return nil, errors.New("the agent service is not ready yet")
}
return r.d, nil
}
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
d, err := r.get()
if err != nil {
return err
}
return d.StartShard(ctx, cmd)
}
func (r *dispatcherRef) StopShard(ctx context.Context, nodeID, shardID string) error {
d, err := r.get()
if err != nil {
return err
}
return d.StopShard(ctx, nodeID, shardID)
}
func (r *dispatcherRef) DrainShard(ctx context.Context, nodeID, shardID string) error {
d, err := r.get()
if err != nil {
return err
}
return d.DrainShard(ctx, nodeID, shardID)
}
func serve(s *http.Server, ln net.Listener) error {
if err := s.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err