Agent stream: connect the fleet to the control plane
One bidirectional connect-go stream per node, carrying registration, heartbeats and shard status up, and start, stop and drain down. The agent dials out: GPU nodes sit behind firewalls far more often than a control plane does, and an outbound connection needs no inbound rule on every node in the fleet. - registration carries the whole hardware inventory, so there is no separate discovery call and a reconnecting agent refreshes what the scheduler knows. The fabric labels travel with it; they cannot be inferred from the wire, so losing them in transit would silently disable multi-node placement. - agentsvc is both the stream handler and the reconciler's dispatcher, because it holds the open streams and is therefore the only thing that can deliver a command to a node. - the agent validates a profile before launching anything, since an engine that refuses a profile does so after minutes of loading, then supervises the process and reports every transition with the engine's output tail on failure. - reconnects forever with jittered backoff. A control plane restart must not require touching every GPU node, and a fleet must not reconnect in lockstep and knock it over again. Engines keep serving throughout, because the router talks to them directly. llmgw agent is real now, and llmgw server runs a control plane whenever any model declares a profile. The static endpoint list stays for router-only runs. Verified with real binaries as separate processes: the agent registers, a replica is placed, an engine launches through the vLLM adapter's actual argument construction, streaming traffic is served, and killing the agent moves the node through unreachable to lost with placement explaining why nothing can run. config.Model and catalog.Model were the same thing described twice, so the router now reads catalog entries directly and the validation rules live in one place. Still missing: mTLS. The stream is plaintext HTTP/2 and must stay on a trusted network until that lands. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c1b9ea1bc2
commit
2373726efd
24 files changed
+3645
-87
No files matched your search
+138
-2
@@ -4,17 +4,25 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/net/http2"
|
||||
"golang.org/x/net/http2/h2c"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||
)
|
||||
|
||||
@@ -55,8 +63,10 @@ func newServerCmd() *cobra.Command {
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(cfg.Endpoints) == 0 {
|
||||
return errors.New("no endpoints configured; use --endpoint or a config file")
|
||||
controlMode := cfg.ControlPlaneMode()
|
||||
if !controlMode && len(cfg.Endpoints) == 0 {
|
||||
return errors.New("nothing to serve: declare models with profiles, " +
|
||||
"or give --endpoint for a router-only run")
|
||||
}
|
||||
|
||||
rt, err := router.New(cfg, log)
|
||||
@@ -70,6 +80,28 @@ func newServerCmd() *cobra.Command {
|
||||
rt.Start(ctx)
|
||||
defer rt.Stop()
|
||||
|
||||
// The control plane replaces the static endpoint list: agents
|
||||
// register, the reconciler places replicas, and Ready ones are
|
||||
// published to the router (plan §6.4).
|
||||
var agentHandler http.Handler
|
||||
var agentPath string
|
||||
if controlMode {
|
||||
cat, err := cfg.Catalog()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ctl.Close()
|
||||
agentPath, agentHandler = svc.Handler()
|
||||
|
||||
log.Info("control plane running",
|
||||
"models", len(cfg.Models), "agent_listen", cfg.AgentListen)
|
||||
}
|
||||
|
||||
adminHandler, err := rt.AdminHandler()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -103,14 +135,39 @@ func newServerCmd() *cobra.Command {
|
||||
"default_profile", cfg.Router.DefaultProfile,
|
||||
)
|
||||
|
||||
var agentSrv *http.Server
|
||||
var agentLn net.Listener
|
||||
if agentHandler != nil {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(agentPath, agentHandler)
|
||||
agentSrv = &http.Server{
|
||||
// Agents speak plaintext HTTP/2, which Go will not do
|
||||
// without being asked. mTLS replaces this (plan §5.6);
|
||||
// until then the agent port must stay on a trusted network.
|
||||
Handler: h2c.NewHandler(mux, &http2.Server{}),
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
if agentLn, err = net.Listen("tcp", cfg.AgentListen); err != nil {
|
||||
_ = apiLn.Close()
|
||||
_ = adminLn.Close()
|
||||
return fmt.Errorf("listening on %s: %w", cfg.AgentListen, err)
|
||||
}
|
||||
}
|
||||
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
g.Go(func() error { return serve(apiSrv, apiLn) })
|
||||
g.Go(func() error { return serve(adminSrv, adminLn) })
|
||||
if agentSrv != nil {
|
||||
g.Go(func() error { return serve(agentSrv, agentLn) })
|
||||
}
|
||||
g.Go(func() error {
|
||||
<-gctx.Done()
|
||||
log.Info("shutting down")
|
||||
sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if agentSrv != nil {
|
||||
_ = agentSrv.Shutdown(sctx)
|
||||
}
|
||||
_ = adminSrv.Shutdown(sctx)
|
||||
return apiSrv.Shutdown(sctx)
|
||||
})
|
||||
@@ -127,6 +184,85 @@ func newServerCmd() *cobra.Command {
|
||||
return cmd
|
||||
}
|
||||
|
||||
// startControlPlane builds the control plane and its agent service.
|
||||
//
|
||||
// The service is both the AgentService handler and the reconciler's Dispatcher:
|
||||
// it holds the open streams, so it is the only thing that can deliver a command
|
||||
// to a node.
|
||||
func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Catalog,
|
||||
sink control.EndpointSink, log *slog.Logger,
|
||||
) (*control.Controller, *agentsvc.Service, error) {
|
||||
ccfg := control.DefaultConfig()
|
||||
ccfg.Reconciler = reconciler.Config{
|
||||
Interval: cfg.Control.Interval,
|
||||
BaseBackoff: cfg.Control.BaseBackoff,
|
||||
MaxBackoff: cfg.Control.MaxBackoff,
|
||||
UnreachableAfter: cfg.Control.UnreachableAfter,
|
||||
LostAfter: cfg.Control.LostAfter,
|
||||
MgmtInterface: cfg.Control.MgmtInterface,
|
||||
}
|
||||
if len(cfg.Control.PortRange) == 2 {
|
||||
ccfg.Reconciler.PortRange = [2]int{cfg.Control.PortRange[0], cfg.Control.PortRange[1]}
|
||||
}
|
||||
|
||||
// The service needs the inventory and the status sink, both of which the
|
||||
// controller owns; the controller needs the service as its dispatcher. The
|
||||
// cycle is broken by giving the controller a dispatcher that forwards to
|
||||
// the service once it exists.
|
||||
fwd := &dispatcherRef{}
|
||||
ctl := control.New(ccfg, cat, fwd, sink, log)
|
||||
svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), log)
|
||||
fwd.set(svc)
|
||||
|
||||
ctl.Start(ctx)
|
||||
return ctl, svc, nil
|
||||
}
|
||||
|
||||
// dispatcherRef lets the controller be built before the service exists.
|
||||
type dispatcherRef struct {
|
||||
mu sync.RWMutex
|
||||
d reconciler.Dispatcher
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) set(d reconciler.Dispatcher) {
|
||||
r.mu.Lock()
|
||||
r.d = d
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) get() (reconciler.Dispatcher, error) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if r.d == nil {
|
||||
return nil, errors.New("the agent service is not ready yet")
|
||||
}
|
||||
return r.d, nil
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.StartShard(ctx, cmd)
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) StopShard(ctx context.Context, nodeID, shardID string) error {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.StopShard(ctx, nodeID, shardID)
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) DrainShard(ctx context.Context, nodeID, shardID string) error {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.DrainShard(ctx, nodeID, shardID)
|
||||
}
|
||||
|
||||
func serve(s *http.Server, ln net.Listener) error {
|
||||
if err := s.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
|
||||
Reference in new issue
Block a user