Release v0.10.4: CPU threads, tracing, and the rest of the review
CPU replicas start on half the machine's CPUs, not all of them: two busy llama.cpp servers each spinning on all 8 vCPUs fell to 0.8 tok/s apiece, where at 4 threads they keep 5.5. llamacpp.threads sets it per node. Inference requests can be traced over OTLP/HTTP (tracing.endpoint), with admission, schedule and upstream spans and traceparent sent to the engine. Security: onboarding no longer puts the cluster root in the system trust store and removes it where it was; the mirror's mutual-TLS API can be limited to named certificates (--api-client); init.log no longer keeps the administrator's password; a warning when auth is off on a public listen; the recorded actor is sanitised and sudo-aware; Ray's dashboard is off, and the remaining engine gaps are documented. reconciler.go and agent.go are split by topic, the status mapping of each failure is tested, and the plan's checklist is brought up to date. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
adb1b3cf2a
commit
62657f4c2a
41 files changed
+2485
-1609
No files matched your search
+31
-2
@@ -8,6 +8,7 @@ import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -232,7 +233,7 @@ func newMirrorRemoveCmd(dir *string) *cobra.Command {
|
||||
func newMirrorServeCmd(dir *string) *cobra.Command {
|
||||
var listen, apiListen, base string
|
||||
var tlsc tlsconf.Config
|
||||
var allow []string
|
||||
var allow, clients []string
|
||||
var allowPrivate bool
|
||||
|
||||
c := &cobra.Command{
|
||||
@@ -259,6 +260,9 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
|
||||
if err := tlsc.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(clients) > 0 && !tlsc.Enabled() {
|
||||
return fmt.Errorf("--api-client names a certificate, and needs --tls-ca, --tls-cert and --tls-key")
|
||||
}
|
||||
|
||||
out := cmd.OutOrStdout()
|
||||
log := slog.Default()
|
||||
@@ -282,7 +286,7 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
|
||||
}
|
||||
admin := &http.Server{
|
||||
Addr: apiListen,
|
||||
Handler: apiGate(tlsc.Enabled(), allowed, api.Handler()),
|
||||
Handler: onlyClients(clients, apiGate(tlsc.Enabled(), allowed, api.Handler())),
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
@@ -294,6 +298,12 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
|
||||
switch {
|
||||
case tlsc.Enabled():
|
||||
fmt.Fprintf(out, "management API on %s, mutual TLS against %s\n", apiListen, tlsc.CA)
|
||||
if len(clients) == 0 {
|
||||
// The same root signs every node's certificate, so without a
|
||||
// name any node could delete models here.
|
||||
log.Warn("the management API takes any certificate from " + tlsc.CA +
|
||||
", every node's included; name the control plane's with --api-client")
|
||||
}
|
||||
case len(allowed) > 0:
|
||||
fmt.Fprintf(out, "management API on %s, for %s and this machine only\n", apiListen, strings.Join(allow, ", "))
|
||||
default:
|
||||
@@ -317,6 +327,8 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
|
||||
"the address nodes use to reach this mirror, as it should appear in catalog sources")
|
||||
c.Flags().StringArrayVar(&allow, "api-allow", nil,
|
||||
"an address or network the management API answers, besides this machine; repeatable (default: this machine only)")
|
||||
c.Flags().StringArrayVar(&clients, "api-client", nil,
|
||||
"with mutual TLS, the common name of a certificate the management API answers; repeatable (default: any from --tls-ca)")
|
||||
c.Flags().BoolVar(&allowPrivate, "allow-private-sources", false,
|
||||
"let downloads reach loopback, private and link-local addresses")
|
||||
c.Flags().StringVar(&tlsc.CA, "tls-ca", "", "CA bundle that management clients must present a certificate from")
|
||||
@@ -452,6 +464,23 @@ func apiGate(mtls bool, allowed []netip.Prefix, next http.Handler) http.Handler
|
||||
return onlyFrom(allowed, next)
|
||||
}
|
||||
|
||||
// onlyClients answers only certificates with one of these common names, when
|
||||
// any are given. The handshake has already checked the certificate against
|
||||
// the root; this narrows which of the root's certificates may manage.
|
||||
func onlyClients(names []string, next http.Handler) http.Handler {
|
||||
if len(names) == 0 {
|
||||
return next
|
||||
}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 ||
|
||||
!slices.Contains(names, r.TLS.VerifiedChains[0][0].Subject.CommonName) {
|
||||
http.Error(w, "this mirror is managed by its control plane only", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// onlyFrom answers only this machine and the addresses allowed: the management
|
||||
// API can delete models and start downloads, and a mirror on its own machine
|
||||
// is reached over the network by one control plane and no one else.
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
@@ -37,3 +40,27 @@ func TestTheMirrorAPIAnswersOnlyThisMachineUnlessTold(t *testing.T) {
|
||||
t.Errorf("mTLS without an allowlist refused a client: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyANamedCertificateManagesTheMirror(t *testing.T) {
|
||||
ok := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})
|
||||
with := func(cn string) *http.Request {
|
||||
r := httptest.NewRequest("GET", "/", nil)
|
||||
if cn != "" {
|
||||
r.TLS = &tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{{{Subject: pkix.Name{CommonName: cn}}}}}
|
||||
}
|
||||
return r
|
||||
}
|
||||
h := onlyClients([]string{"control-plane"}, ok)
|
||||
for cn, want := range map[string]int{"control-plane": 200, "node01": 403, "": 403} {
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, with(cn))
|
||||
if w.Code != want {
|
||||
t.Errorf("%q: %d, expected %d", cn, w.Code, want)
|
||||
}
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
onlyClients(nil, ok).ServeHTTP(w, with("node01"))
|
||||
if w.Code != 200 {
|
||||
t.Error("no names given narrowed it anyway")
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"time"
|
||||
|
||||
@@ -127,6 +128,10 @@ func newModelRemoveCmd(client func() *ctlClient) *cobra.Command {
|
||||
|
||||
// actor names who made a change, for the audit trail on a stored model.
|
||||
func actor() string {
|
||||
// Under sudo, the person who ran it rather than root.
|
||||
if u := os.Getenv("SUDO_USER"); u != "" && os.Geteuid() == 0 {
|
||||
return u
|
||||
}
|
||||
if u, err := user.Current(); err == nil && u.Username != "" {
|
||||
return u.Username
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
@@ -90,6 +91,21 @@ func newServerCmd() *cobra.Command {
|
||||
"or give --endpoint for a router-only run")
|
||||
}
|
||||
|
||||
// Before the router, so its tracer is the configured one.
|
||||
shutdownTracing, err := obs.SetupTracing(cmd.Context(), obs.TracingOptions{
|
||||
Endpoint: cfg.Tracing.Endpoint, SampleRatio: cfg.Tracing.SampleRatio,
|
||||
Service: "llmgw-server", Version: version.Version,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
// Bounded: a collector that is gone must not hold up a shutdown.
|
||||
sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = shutdownTracing(sctx)
|
||||
}()
|
||||
|
||||
rt, err := router.New(cfg, log)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -389,17 +389,22 @@ func (in *serverInit) runInitTUI(ctx context.Context, out io.Writer) error {
|
||||
return fmt.Errorf("stopped; run init again to finish what was started")
|
||||
}
|
||||
|
||||
// The whole run is kept, root's only, since it holds the password. On a
|
||||
// failure it is printed too, for the lines the screen had no room for;
|
||||
// otherwise the screen's last frame says it, and only what must outlive
|
||||
// the screen is printed again: the password, and what to do next.
|
||||
// The whole run is kept, root's only, with the password taken out: it is
|
||||
// shown once, on the screen, and a file on disk would outlive that. On a
|
||||
// failure the run is printed too, for the lines the screen had no room
|
||||
// for; otherwise the screen's last frame says it, and only what must
|
||||
// outlive the screen is printed again: the password, and what to do next.
|
||||
logMu.Lock()
|
||||
defer logMu.Unlock()
|
||||
whole := strings.Join(log, "\n") + "\n"
|
||||
kept := whole
|
||||
if fm.password != "" {
|
||||
kept = strings.ReplaceAll(kept, fm.password, "[password shown once, not kept]")
|
||||
}
|
||||
saved := ""
|
||||
if in.stateDir != "" {
|
||||
p := filepath.Join(in.stateDir, "init.log")
|
||||
if err := os.WriteFile(p, []byte(whole), 0o600); err == nil {
|
||||
if err := os.WriteFile(p, []byte(kept), 0o600); err == nil {
|
||||
saved = p
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user