package cli import ( "context" "errors" "fmt" "log/slog" "net" "net/http" "os/signal" "strings" "sync" "syscall" "time" "github.com/spf13/cobra" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" "golang.org/x/sync/errgroup" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/router" ) func newServerCmd() *cobra.Command { var ( configPath string listen string adminListen string endpoints []string ) cmd := &cobra.Command{ Use: "server", Short: "Run the router (phase 1; the control plane arrives in phase 3)", Long: "Serves the OpenAI API in front of a set of engine replicas.\n\n" + "In this phase the replica list is static: give it in the config file, or\n" + "pass --endpoint id=URL=model[,model] once per replica for a quick test.", RunE: func(cmd *cobra.Command, _ []string) error { log := logger() cfg, err := config.Load(configPath) if err != nil { return err } if listen != "" { cfg.Listen = listen } if adminListen != "" { cfg.AdminListen = adminListen } for _, spec := range endpoints { ep, err := parseEndpointFlag(spec) if err != nil { return err } cfg.Endpoints = append(cfg.Endpoints, ep) } if err := cfg.Validate(); err != nil { return err } controlMode := cfg.ControlPlaneMode() if !controlMode && len(cfg.Endpoints) == 0 { return errors.New("nothing to serve: declare models with profiles, " + "or give --endpoint for a router-only run") } rt, err := router.New(cfg, log) if err != nil { return err } ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM) defer stop() rt.Start(ctx) defer rt.Stop() // The control plane replaces the static endpoint list: agents // register, the reconciler places replicas, and Ready ones are // published to the router (plan §6.4). var agentHandler http.Handler var agentPath string if controlMode { cat, err := cfg.Catalog() if err != nil { return err } ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, log) if err != nil { return err } defer ctl.Close() agentPath, agentHandler = svc.Handler() log.Info("control plane running", "models", len(cfg.Models), "agent_listen", cfg.AgentListen) } adminHandler, err := rt.AdminHandler() if err != nil { return err } apiSrv := &http.Server{ Handler: rt.Handler(), ReadHeaderTimeout: 10 * time.Second, // No write timeout: responses stream for minutes and the proxy // enforces its own per-phase deadlines. } adminSrv := &http.Server{ Handler: adminHandler, ReadHeaderTimeout: 10 * time.Second, } apiLn, err := net.Listen("tcp", cfg.Listen) if err != nil { return fmt.Errorf("listening on %s: %w", cfg.Listen, err) } adminLn, err := net.Listen("tcp", cfg.AdminListen) if err != nil { _ = apiLn.Close() return fmt.Errorf("listening on %s: %w", cfg.AdminListen, err) } log.Info("router listening", "api", apiLn.Addr().String(), "admin", adminLn.Addr().String(), "endpoints", len(cfg.Endpoints), "default_profile", cfg.Router.DefaultProfile, ) var agentSrv *http.Server var agentLn net.Listener if agentHandler != nil { mux := http.NewServeMux() mux.Handle(agentPath, agentHandler) agentSrv = &http.Server{ // Agents speak plaintext HTTP/2, which Go will not do // without being asked. mTLS replaces this (plan §5.6); // until then the agent port must stay on a trusted network. Handler: h2c.NewHandler(mux, &http2.Server{}), ReadHeaderTimeout: 10 * time.Second, } if agentLn, err = net.Listen("tcp", cfg.AgentListen); err != nil { _ = apiLn.Close() _ = adminLn.Close() return fmt.Errorf("listening on %s: %w", cfg.AgentListen, err) } } g, gctx := errgroup.WithContext(ctx) g.Go(func() error { return serve(apiSrv, apiLn) }) g.Go(func() error { return serve(adminSrv, adminLn) }) if agentSrv != nil { g.Go(func() error { return serve(agentSrv, agentLn) }) } g.Go(func() error { <-gctx.Done() log.Info("shutting down") sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() if agentSrv != nil { _ = agentSrv.Shutdown(sctx) } _ = adminSrv.Shutdown(sctx) return apiSrv.Shutdown(sctx) }) return g.Wait() }, } f := cmd.Flags() f.StringVar(&configPath, "config", "", "configuration file (optional)") f.StringVar(&listen, "listen", "", "override the client-facing address") f.StringVar(&adminListen, "admin-listen", "", "override the metrics/pprof address") f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)") return cmd } // startControlPlane builds the control plane and its agent service. // // The service is both the AgentService handler and the reconciler's Dispatcher: // it holds the open streams, so it is the only thing that can deliver a command // to a node. func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Catalog, sink control.EndpointSink, log *slog.Logger, ) (*control.Controller, *agentsvc.Service, error) { ccfg := control.DefaultConfig() ccfg.Reconciler = reconciler.Config{ Interval: cfg.Control.Interval, BaseBackoff: cfg.Control.BaseBackoff, MaxBackoff: cfg.Control.MaxBackoff, UnreachableAfter: cfg.Control.UnreachableAfter, LostAfter: cfg.Control.LostAfter, MgmtInterface: cfg.Control.MgmtInterface, } if len(cfg.Control.PortRange) == 2 { ccfg.Reconciler.PortRange = [2]int{cfg.Control.PortRange[0], cfg.Control.PortRange[1]} } // The service needs the inventory and the status sink, both of which the // controller owns; the controller needs the service as its dispatcher. The // cycle is broken by giving the controller a dispatcher that forwards to // the service once it exists. fwd := &dispatcherRef{} ctl := control.New(ccfg, cat, fwd, sink, log) svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), log) fwd.set(svc) ctl.Start(ctx) return ctl, svc, nil } // dispatcherRef lets the controller be built before the service exists. type dispatcherRef struct { mu sync.RWMutex d reconciler.Dispatcher } func (r *dispatcherRef) set(d reconciler.Dispatcher) { r.mu.Lock() r.d = d r.mu.Unlock() } func (r *dispatcherRef) get() (reconciler.Dispatcher, error) { r.mu.RLock() defer r.mu.RUnlock() if r.d == nil { return nil, errors.New("the agent service is not ready yet") } return r.d, nil } func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error { d, err := r.get() if err != nil { return err } return d.StartShard(ctx, cmd) } func (r *dispatcherRef) StopShard(ctx context.Context, nodeID, shardID string) error { d, err := r.get() if err != nil { return err } return d.StopShard(ctx, nodeID, shardID) } func (r *dispatcherRef) DrainShard(ctx context.Context, nodeID, shardID string) error { d, err := r.get() if err != nil { return err } return d.DrainShard(ctx, nodeID, shardID) } func serve(s *http.Server, ln net.Listener) error { if err := s.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { return err } return nil } // parseEndpointFlag decodes id=URL=model[,model]. // // The URL contains '=' only in pathological cases, so splitting into exactly // three parts keeps the flag readable while staying unambiguous. func parseEndpointFlag(spec string) (config.Endpoint, error) { parts := strings.SplitN(spec, "=", 3) if len(parts) != 3 { return config.Endpoint{}, fmt.Errorf("--endpoint %q: expected id=URL=model[,model]", spec) } models := strings.Split(parts[2], ",") for i := range models { models[i] = strings.TrimSpace(models[i]) } return config.Endpoint{ ID: strings.TrimSpace(parts[0]), URL: strings.TrimRight(strings.TrimSpace(parts[1]), "/"), Models: models, Pool: "static", Weight: 1, }, nil }