llmgw stop, start and restart act on the whole system
Issue #3 asked for commands to stop and start the system, and 05aa21c gave
only one machine's services. Now:
- llmgw stop sets serving.stopped in the cluster's settings, live: the
reconciler wants every model at no replicas, freeing every GPU, and the
gateway answers every POST under /v1/ with 503 system_stopped. The control
planes, agents and mirror keep running, and no model's definition changes.
- llmgw start clears it; every model is placed again as it asks.
- llmgw restart has every agent restart over its stream (RestartAgent), then
the other control planes over ssh, standbys before a leader, then this
one; engines keep serving.
- llmgw status is the cluster's summary, saying when it is stopped.
One machine's services moved to llmgw service status|start|stop|restart.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
05aa21c4d4
commit
7245dbe1dd
29 files changed
+891
-254
No files matched your search
@@ -237,6 +237,8 @@ func (w *settingsWatch) check(ctx context.Context) {
|
||||
// again: what it reads at every use, or not at all.
|
||||
var liveSettings = []string{
|
||||
"guard",
|
||||
// Stopping and starting the service (llmgw stop, llmgw start).
|
||||
"serving",
|
||||
// Sent to the nodes and this machine's firewall unit as they change.
|
||||
"firewall", "firewall_networks",
|
||||
// Read by server init, node add and backups, not by the server.
|
||||
|
||||
@@ -63,8 +63,10 @@ func Root() *cobra.Command {
|
||||
newWipeCmd(),
|
||||
newDocsCmd(),
|
||||
)
|
||||
// status, start, stop and restart: this machine's llmgw services.
|
||||
root.AddCommand(newServiceCmds()...)
|
||||
// status, start, stop and restart: the whole system; service: this
|
||||
// machine's services.
|
||||
root.AddCommand(newSystemCmds()...)
|
||||
root.AddCommand(newServiceCmd())
|
||||
return root
|
||||
}
|
||||
|
||||
|
||||
@@ -186,6 +186,7 @@ func newServerCmd() *cobra.Command {
|
||||
|
||||
rt.Start(ctx)
|
||||
defer rt.Stop()
|
||||
rt.SetStopped(cfg.Serving.Stopped)
|
||||
|
||||
// Which cluster this is, to federated peers: the id in the hop
|
||||
// header that stops a request going round two clusters that
|
||||
@@ -252,6 +253,8 @@ func newServerCmd() *cobra.Command {
|
||||
}
|
||||
defer ctl.Close()
|
||||
agentSvc = svc
|
||||
// Stopped by the operators (llmgw stop): nothing placed.
|
||||
ctl.Reconciler().SetStopped(cfg.Serving.Stopped)
|
||||
// Whether nodes keep llmgw's firewall, as the settings say now.
|
||||
svc.SetFirewall(func() *llmgwv1.Firewall {
|
||||
c := current.Load()
|
||||
@@ -367,6 +370,9 @@ func newServerCmd() *cobra.Command {
|
||||
if st := rt.Store(); st != nil {
|
||||
// Removing a node revokes its certificate.
|
||||
api.SetRevoker(st.RevokeNode)
|
||||
if agentSvc != nil {
|
||||
api.SetAgentRestarter(agentSvc.RestartAgents)
|
||||
}
|
||||
api.SetAudit(st)
|
||||
// The cluster's settings, each change checked against this
|
||||
// machine's own keys before it is kept.
|
||||
@@ -676,6 +682,14 @@ func newServerCmd() *cobra.Command {
|
||||
return false
|
||||
}
|
||||
current.Store(&next)
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "serving") }) {
|
||||
rt.SetStopped(next.Serving.Stopped)
|
||||
if admin != nil {
|
||||
admin.Reconciler().SetStopped(next.Serving.Stopped)
|
||||
}
|
||||
log.Warn("the service is " + map[bool]string{true: "stopped", false: "started"}[next.Serving.Stopped] +
|
||||
" by the cluster's settings")
|
||||
}
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "firewall") }) {
|
||||
if err := writeFirewallState(next); err != nil {
|
||||
log.Warn("could not record the firewall's settings for this machine", "err", err)
|
||||
|
||||
+16
-3
@@ -78,13 +78,26 @@ func hasUnit(list []string, s string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func newServiceCmds() []*cobra.Command {
|
||||
// newServiceCmd is llmgw service: this machine's llmgw services, one by one.
|
||||
// The whole system is llmgw status, start, stop and restart.
|
||||
func newServiceCmd() *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "service",
|
||||
Short: "This machine's llmgw services: status, start, stop, restart",
|
||||
Long: "The control plane (server), node agent (agent) and model mirror (mirror) on\n" +
|
||||
"this machine alone, by systemd. The whole system is 'llmgw status', 'llmgw\n" +
|
||||
"stop', 'llmgw start' and 'llmgw restart', run on a control plane.",
|
||||
}
|
||||
c.AddCommand(serviceVerbs()...)
|
||||
return c
|
||||
}
|
||||
|
||||
func serviceVerbs() []*cobra.Command {
|
||||
status := &cobra.Command{
|
||||
Use: "status [server|agent|mirror]...",
|
||||
Short: "Whether llmgw's services on this machine run",
|
||||
Long: "Each llmgw service this machine has — the control plane (server), the\n" +
|
||||
"node agent (agent), the model mirror (mirror) — and whether it runs.\n" +
|
||||
"The cluster as a whole is 'llmgw ctl status'.",
|
||||
"node agent (agent), the model mirror (mirror) — and whether it runs.",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
units, err := localUnits(cmd, args)
|
||||
if err != nil {
|
||||
|
||||
@@ -3,11 +3,18 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) {
|
||||
@@ -42,7 +49,7 @@ func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) {
|
||||
return
|
||||
}
|
||||
var out bytes.Buffer
|
||||
c := newServiceCmds()[0]
|
||||
c := serviceVerbs()[0]
|
||||
c.SetOut(&out)
|
||||
c.SetContext(context.Background())
|
||||
if err := c.RunE(c, nil); err != nil {
|
||||
@@ -52,3 +59,46 @@ func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) {
|
||||
t.Errorf("%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopSetsTheClustersServingStopped(t *testing.T) {
|
||||
var put adminapi.SettingsRequest
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/api/v1/summary":
|
||||
_ = json.NewEncoder(w).Encode(adminapi.Summary{})
|
||||
case r.Method == "GET" && r.URL.Path == "/api/v1/settings":
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 3, Body: "router:\n retries: 2\n"})
|
||||
case r.Method == "PUT" && r.URL.Path == "/api/v1/settings":
|
||||
_ = json.NewDecoder(r.Body).Decode(&put)
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 4, Body: put.Body})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
dir := t.TempDir()
|
||||
cfg := filepath.Join(dir, "server.yaml")
|
||||
_ = os.WriteFile(filepath.Join(dir, "token"), []byte("t"), 0o600)
|
||||
_ = os.WriteFile(cfg, []byte("listen: \":8080\"\nagent_insecure: true\nadmin_listen: "+
|
||||
strings.TrimPrefix(srv.URL, "http://")+"\nadmin_remote: true\nadmin_token_file: "+filepath.Join(dir, "token")+"\n"), 0o600)
|
||||
|
||||
var stop *cobra.Command
|
||||
for _, c := range newSystemCmds() {
|
||||
if c.Name() == "stop" {
|
||||
stop = c
|
||||
}
|
||||
}
|
||||
var out bytes.Buffer
|
||||
stop.SetOut(&out)
|
||||
stop.SetArgs([]string{"--config", cfg, "--note", "maintenance"})
|
||||
if err := stop.ExecuteContext(context.Background()); err != nil {
|
||||
t.Fatalf("%v: %s", err, out.String())
|
||||
}
|
||||
if put.Base != 3 || !strings.Contains(put.Body, "stopped: true") || !strings.Contains(put.Body, "retries: 2") ||
|
||||
put.Note != "maintenance" {
|
||||
t.Errorf("sent %+v", put)
|
||||
}
|
||||
if !strings.Contains(out.String(), "stopping") {
|
||||
t.Errorf("%s", out.String())
|
||||
}
|
||||
}
|
||||
@@ -153,6 +153,9 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) {
|
||||
fmt.Fprintf(out, "STORE %s\n", strings.Join(parts, ", "))
|
||||
}
|
||||
|
||||
if s.Stopped {
|
||||
fmt.Fprintln(out, "SERVICE STOPPED: no model runs and requests are answered 503 (llmgw start)")
|
||||
}
|
||||
if s.GuardOff {
|
||||
fmt.Fprintln(out, "GUARD off: no request is checked, whatever the policies say (llmgw ctl guard on)")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
// systemClient is the admin API of the control plane on this machine, as its
|
||||
// server.yaml says, with its token.
|
||||
func systemClient(configPath string) func() *ctlClient {
|
||||
return func() *ctlClient {
|
||||
addr := "127.0.0.1:9090"
|
||||
var tokens []string
|
||||
if cfg, ok := loadServerConfig(configPath); ok {
|
||||
if cfg.AdminListen != "" {
|
||||
addr = cfg.AdminListen
|
||||
if strings.HasPrefix(addr, ":") {
|
||||
addr = "127.0.0.1" + addr
|
||||
}
|
||||
}
|
||||
tokens = append(tokens, adminTokenPath(configPath, cfg))
|
||||
}
|
||||
return newCtlClient(addr, tokens...)
|
||||
}
|
||||
}
|
||||
|
||||
// newSystemCmds are llmgw status, stop, start and restart: the whole system,
|
||||
// from a control plane.
|
||||
func newSystemCmds() []*cobra.Command {
|
||||
var configPath string
|
||||
client := systemClient("/etc/llmgw/server.yaml")
|
||||
bind := func(c *cobra.Command) *cobra.Command {
|
||||
c.Flags().StringVar(&configPath, "config", "/etc/llmgw/server.yaml", "this control plane's configuration")
|
||||
c.PreRun = func(*cobra.Command, []string) { client = systemClient(configPath) }
|
||||
return c
|
||||
}
|
||||
call := func() *ctlClient { return client() }
|
||||
|
||||
status := bind(&cobra.Command{
|
||||
Use: "status",
|
||||
Short: "The whole system: whether it serves, its control planes, nodes, models and store",
|
||||
Long: "Run on a control plane. Says whether the service is stopped (llmgw stop),\n" +
|
||||
"and everything 'llmgw ctl status' does. One machine's services are\n" +
|
||||
"'llmgw service status'.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var s adminapi.Summary
|
||||
if err := call().get(cmd.Context(), "/api/v1/summary", &s); err != nil {
|
||||
return notAControlPlane(err)
|
||||
}
|
||||
printSummary(cmd, s)
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
switchService := func(stop bool) *cobra.Command {
|
||||
use, short, long := "start", "Start the whole system serving again, after llmgw stop",
|
||||
"Every model comes back as its catalog entry asks: placed, loaded and\n"+
|
||||
"serving again, on every control plane within seconds. Run on a control\n"+
|
||||
"plane. The services themselves are never stopped by llmgw stop; a\n"+
|
||||
"machine's own are started with 'llmgw service start'."
|
||||
if stop {
|
||||
use, short, long = "stop", "Stop the whole system serving: every model taken down, requests answered 503",
|
||||
"Every model's replicas are taken down, freeing every GPU, and every\n"+
|
||||
"request for a model is answered 503 (system_stopped) until 'llmgw start'.\n"+
|
||||
"The control planes, agents and mirror keep running, so the console, the\n"+
|
||||
"admin pages and llmgw start still work, and no model's definition is\n"+
|
||||
"changed. In force on every control plane within seconds; kept in the\n"+
|
||||
"cluster's settings (serving.stopped), so a restart does not undo it."
|
||||
}
|
||||
var note string
|
||||
var wait bool
|
||||
c := bind(&cobra.Command{
|
||||
Use: use,
|
||||
Short: short,
|
||||
Long: long,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if note == "" {
|
||||
note = "llmgw " + use
|
||||
}
|
||||
var s adminapi.Summary
|
||||
if err := call().get(cmd.Context(), "/api/v1/summary", &s); err != nil {
|
||||
return notAControlPlane(err)
|
||||
}
|
||||
if s.Stopped == stop {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "the system is already %s\n", map[bool]string{true: "stopped", false: "serving"}[stop])
|
||||
return nil
|
||||
}
|
||||
if err := changeSettings(cmd, call, note, func(doc *settings.Doc) error {
|
||||
return doc.SetYAML([]string{"serving", "stopped"}, fmt.Sprint(stop))
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if !wait {
|
||||
if stop {
|
||||
fmt.Fprintln(out, "stopping: every model's replicas are being taken down; llmgw status follows it")
|
||||
} else {
|
||||
fmt.Fprintln(out, "starting: every model is placed again as it asks; llmgw status follows it")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return waitForService(cmd, call, stop)
|
||||
},
|
||||
})
|
||||
c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version and in the audit")
|
||||
c.Flags().BoolVar(&wait, "wait", false, "wait until every replica is down (stop) or the models have theirs (start)")
|
||||
return c
|
||||
}
|
||||
|
||||
var sshUser string
|
||||
restart := bind(&cobra.Command{
|
||||
Use: "restart",
|
||||
Short: "Restart every control plane and node agent; the engines keep serving",
|
||||
Long: "Every node's agent starts again, then the other control planes over ssh\n" +
|
||||
"as --ssh-user, then this one — the leader last, so it hands over to a\n" +
|
||||
"standby already restarted. The engines keep serving throughout and are\n" +
|
||||
"taken over again; nothing is placed anew. Run on a control plane, with\n" +
|
||||
"sudo. One machine's services are 'llmgw service restart'.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if os.Geteuid() != 0 {
|
||||
return errors.New("restarting this control plane needs root: run it with sudo")
|
||||
}
|
||||
ctx, out := cmd.Context(), cmd.OutOrStdout()
|
||||
var s adminapi.Summary
|
||||
if err := call().get(ctx, "/api/v1/summary", &s); err != nil {
|
||||
return notAControlPlane(err)
|
||||
}
|
||||
var agents adminapi.AgentRestartView
|
||||
if err := call().post(ctx, "/api/v1/agents/restart", []byte("{}"), &agents); err != nil {
|
||||
return fmt.Errorf("restarting the agents: %w", err)
|
||||
}
|
||||
fmt.Fprintf(out, "agents %d told to restart: %s\n", len(agents.Nodes), strings.Join(agents.Nodes, ", "))
|
||||
|
||||
// The other control planes, standbys before a leader.
|
||||
var others []adminapi.ControlPlane
|
||||
for _, cp := range s.ControlPlanes.Members {
|
||||
if !cp.This {
|
||||
others = append(others, cp)
|
||||
}
|
||||
}
|
||||
for pass := 0; pass < 2; pass++ {
|
||||
for _, cp := range others {
|
||||
leading := cp.State == "leading"
|
||||
if leading != (pass == 1) {
|
||||
continue
|
||||
}
|
||||
r := &onboard.SSH{Addr: cp.Address, User: sshUser}
|
||||
if res, err := r.Run(ctx, "sudo systemctl restart llmgw-server", nil); err != nil {
|
||||
return fmt.Errorf("%s (%s) could not be restarted over ssh as %q: %v %s; "+
|
||||
"run `sudo llmgw service restart server` there", cp.Name, cp.Address, sshUser, err, strings.TrimSpace(string(res)))
|
||||
}
|
||||
fmt.Fprintf(out, "%-10s restarted (%s)\n", cp.Name, cp.State)
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(unitDir + "/llmgw-server.service"); err == nil {
|
||||
fmt.Fprintln(out, "this restarting")
|
||||
if b, err := exec.CommandContext(ctx, "systemctl", "restart", "llmgw-server").CombinedOutput(); err != nil { //nolint:gosec // fixed command
|
||||
return fmt.Errorf("restarting this control plane: %v: %s", err, strings.TrimSpace(string(b)))
|
||||
}
|
||||
fmt.Fprintln(out, "this restarted")
|
||||
}
|
||||
fmt.Fprintln(out, "\nllmgw status shows the nodes coming back; their replicas are taken over as they are.")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
restart.Flags().StringVar(&sshUser, "ssh-user", os.Getenv("SUDO_USER"), "who to log in to the other control planes as")
|
||||
|
||||
return []*cobra.Command{status, switchService(true), switchService(false), restart}
|
||||
}
|
||||
|
||||
// waitForService waits until no replica runs (stopped) or every model has
|
||||
// the replicas it asks for (started).
|
||||
func waitForService(cmd *cobra.Command, call func() *ctlClient, stop bool) error {
|
||||
deadline := time.Now().Add(15 * time.Minute)
|
||||
for time.Now().Before(deadline) {
|
||||
var s adminapi.Summary
|
||||
if err := call().get(cmd.Context(), "/api/v1/summary", &s); err == nil {
|
||||
if stop && s.Replicas.Ready+s.Replicas.Starting+s.Replicas.Draining == 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "stopped: no replica runs")
|
||||
return nil
|
||||
}
|
||||
if !stop && s.Models.Satisfied >= s.Models.Total {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "serving: %d of %d models have the replicas they ask for\n",
|
||||
s.Models.Satisfied, s.Models.Total)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-cmd.Context().Done():
|
||||
return cmd.Context().Err()
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
}
|
||||
return errors.New("not there within fifteen minutes; llmgw status says where it stands")
|
||||
}
|
||||
|
||||
// notAControlPlane is a failure to reach the admin API, said as what it
|
||||
// most often means.
|
||||
func notAControlPlane(err error) error {
|
||||
return fmt.Errorf("%w\n(the whole system is stopped and started from a control plane; this machine's own "+
|
||||
"services are 'llmgw service status|start|stop|restart')", err)
|
||||
}
|
||||
Reference in new issue
Block a user