Decide engines per node, and let a node join without one it cannot run
engine_overrides give nodes picked by GPU class, id or labels their own engine versions; node add --disable-engine keeps one off a node; what vLLM cannot do on a node is a warning rather than a refused node; the CUDA requirement follows the version; a hand-named binary is no longer replaced by an installed one. Refs #2 (items 4 and 7). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cba3e329fc
commit
e313f32696
21 files changed
+532
-62
No files matched your search
@@ -38,8 +38,11 @@ that file when the report is not enough to go on.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
doctor.SetConfigLoader(config.LoadAgent)
|
||||
if opts.VLLMVersion == "" {
|
||||
switch opts.VLLMVersion {
|
||||
case "":
|
||||
opts.VLLMVersion = config.DefaultEngines["vllm"]
|
||||
case "none":
|
||||
opts.VLLMVersion = ""
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
+20
-1
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
@@ -113,13 +114,14 @@ engines are kept.`,
|
||||
f.StringVar(&na.opts.MgmtAddr, "mgmt-addr", "", "address engines are reached at (default: found by the agent)")
|
||||
f.StringVar(&na.opts.MgmtIface, "mgmt-if", "", "interface NCCL bootstraps over (default: the one holding that address)")
|
||||
f.StringVar(&na.opts.LlamaCppBin, "llamacpp-bin", "", "a llama-server you installed, instead of the one the agent installs")
|
||||
f.StringVar(&na.opts.VLLMBin, "vllm-bin", "", "a vllm you installed; also set engines.vllm: \"\" in server.yaml")
|
||||
f.StringVar(&na.opts.VLLMBin, "vllm-bin", "", "a vllm you installed, instead of the one the agent installs")
|
||||
f.StringVar(&na.opts.SGLangPython, "sglang-python", "", "python of an SGLang virtualenv you installed")
|
||||
f.StringVar(&na.opts.NvidiaSMI, "nvidia-smi", "", "where nvidia-smi is, when it is not on PATH")
|
||||
f.StringVar(&na.opts.IBRoot, "ib-root", "", "the sysfs InfiniBand directory, when not /sys/class/infiniband")
|
||||
f.StringVar(&na.opts.PKI, "pki", defaultPKIDir, "directory holding the certificate authority")
|
||||
f.StringVar(&na.opts.Releases, "releases", "https://gitea.mynodes.xyz/ilker/llmgw/releases/download", "where release files are downloaded from")
|
||||
f.BoolVar(&na.opts.IgnoreChecks, "ignore-checks", false, "add the node even when doctor finds a problem")
|
||||
f.StringArrayVar(&na.opts.DisableEngines, "disable-engine", nil, "an engine this node never installs, e.g. vllm on GPUs it does not run on; repeatable")
|
||||
f.StringVar(&na.admin, "admin", "", "the control plane's admin API, to watch the node join (default: from server.yaml, else 127.0.0.1:9090)")
|
||||
f.StringVar(&na.serverCfg, "config", "/etc/llmgw/server.yaml", "the control plane's configuration, for the admin API's address")
|
||||
f.BoolVar(&na.waitAll, "wait", false, "wait for every engine, vLLM included, not only llama.cpp")
|
||||
@@ -148,6 +150,23 @@ func (na *nodeAdd) prepare() error {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, e := range o.DisableEngines {
|
||||
if !slices.Contains(provision.Installable(), e) {
|
||||
return fmt.Errorf("--disable-engine %s: the engines are %s", e, strings.Join(provision.Installable(), ", "))
|
||||
}
|
||||
}
|
||||
// What the control plane will ask the node to install, so doctor checks
|
||||
// that vLLM rather than its own default. Its overrides for a GPU class
|
||||
// are not known until the node says what it has, and are not applied
|
||||
// here; doctor's verdict on vLLM is a warning either way.
|
||||
if o.VLLMVersion == "" {
|
||||
if cfg, err := config.Load(na.serverCfg); err == nil {
|
||||
o.VLLMVersion = cfg.WantedEngines()["vllm"]
|
||||
if o.VLLMVersion == "" {
|
||||
o.VLLMVersion = "none"
|
||||
}
|
||||
}
|
||||
}
|
||||
o.Version = version.Version
|
||||
if o.Binary == "" {
|
||||
exe, err := os.Executable()
|
||||
|
||||
+12
-6
@@ -5,19 +5,23 @@ import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/net/http2"
|
||||
"golang.org/x/net/http2/h2c"
|
||||
@@ -496,8 +500,10 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
|
||||
}
|
||||
}
|
||||
svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), requireIdentity, log)
|
||||
svc.SetEngines(cfg.WantedEngines())
|
||||
log.Info("nodes install these engines", "engines", cfg.WantedEngines())
|
||||
svc.SetEnginesFor(func(n inventory.Node) map[string]string {
|
||||
return cfg.EnginesFor(n.ID, slices.Sorted(maps.Keys(n.GPUsByClass())), n.Labels)
|
||||
})
|
||||
log.Info("nodes install these engines", "engines", cfg.WantedEngines(), "overrides", len(cfg.EngineOverrides))
|
||||
// A standby must refuse agents as well as refusing to place. Commands
|
||||
// travel down the stream the agent opened, so a standby holding one is a
|
||||
// node the leader cannot command — and the leader's placements would fail
|
||||
|
||||
Reference in new issue
Block a user