Release v0.10.4: CPU threads, tracing, and the rest of the review
Release / build (push) Successful in 4m11s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 4m57s

CPU replicas start on half the machine's CPUs, not all of them: two busy
llama.cpp servers each spinning on all 8 vCPUs fell to 0.8 tok/s apiece,
where at 4 threads they keep 5.5. llamacpp.threads sets it per node.

Inference requests can be traced over OTLP/HTTP (tracing.endpoint), with
admission, schedule and upstream spans and traceparent sent to the engine.

Security: onboarding no longer puts the cluster root in the system trust
store and removes it where it was; the mirror's mutual-TLS API can be
limited to named certificates (--api-client); init.log no longer keeps the
administrator's password; a warning when auth is off on a public listen;
the recorded actor is sanitised and sudo-aware; Ray's dashboard is off, and
the remaining engine gaps are documented.

reconciler.go and agent.go are split by topic, the status mapping of each
failure is tested, and the plan's checklist is brought up to date.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 04:00:35 +02:00
1 parent adb1b3cf2a
commit 62657f4c2a
41 files changed
+2485 -1609

No files matched your search

+31 -2
View File
@@ -8,6 +8,7 @@ import (
"net/http"
"net/netip"
"os"
"slices"
"strings"
"time"
@@ -232,7 +233,7 @@ func newMirrorRemoveCmd(dir *string) *cobra.Command {
func newMirrorServeCmd(dir *string) *cobra.Command {
var listen, apiListen, base string
var tlsc tlsconf.Config
var allow []string
var allow, clients []string
var allowPrivate bool
c := &cobra.Command{
@@ -259,6 +260,9 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
if err := tlsc.Validate(); err != nil {
return err
}
if len(clients) > 0 && !tlsc.Enabled() {
return fmt.Errorf("--api-client names a certificate, and needs --tls-ca, --tls-cert and --tls-key")
}
out := cmd.OutOrStdout()
log := slog.Default()
@@ -282,7 +286,7 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
}
admin := &http.Server{
Addr: apiListen,
Handler: apiGate(tlsc.Enabled(), allowed, api.Handler()),
Handler: onlyClients(clients, apiGate(tlsc.Enabled(), allowed, api.Handler())),
ReadHeaderTimeout: 10 * time.Second,
}
@@ -294,6 +298,12 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
switch {
case tlsc.Enabled():
fmt.Fprintf(out, "management API on %s, mutual TLS against %s\n", apiListen, tlsc.CA)
if len(clients) == 0 {
// The same root signs every node's certificate, so without a
// name any node could delete models here.
log.Warn("the management API takes any certificate from " + tlsc.CA +
", every node's included; name the control plane's with --api-client")
}
case len(allowed) > 0:
fmt.Fprintf(out, "management API on %s, for %s and this machine only\n", apiListen, strings.Join(allow, ", "))
default:
@@ -317,6 +327,8 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
"the address nodes use to reach this mirror, as it should appear in catalog sources")
c.Flags().StringArrayVar(&allow, "api-allow", nil,
"an address or network the management API answers, besides this machine; repeatable (default: this machine only)")
c.Flags().StringArrayVar(&clients, "api-client", nil,
"with mutual TLS, the common name of a certificate the management API answers; repeatable (default: any from --tls-ca)")
c.Flags().BoolVar(&allowPrivate, "allow-private-sources", false,
"let downloads reach loopback, private and link-local addresses")
c.Flags().StringVar(&tlsc.CA, "tls-ca", "", "CA bundle that management clients must present a certificate from")
@@ -452,6 +464,23 @@ func apiGate(mtls bool, allowed []netip.Prefix, next http.Handler) http.Handler
return onlyFrom(allowed, next)
}
// onlyClients answers only certificates with one of these common names, when
// any are given. The handshake has already checked the certificate against
// the root; this narrows which of the root's certificates may manage.
func onlyClients(names []string, next http.Handler) http.Handler {
if len(names) == 0 {
return next
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 ||
!slices.Contains(names, r.TLS.VerifiedChains[0][0].Subject.CommonName) {
http.Error(w, "this mirror is managed by its control plane only", http.StatusForbidden)
return
}
next.ServeHTTP(w, r)
})
}
// onlyFrom answers only this machine and the addresses allowed: the management
// API can delete models and start downloads, and a mirror on its own machine
// is reached over the network by one control plane and no one else.
+27
View File
@@ -1,6 +1,9 @@
package cli
import (
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"net/http"
"net/http/httptest"
"net/netip"
@@ -37,3 +40,27 @@ func TestTheMirrorAPIAnswersOnlyThisMachineUnlessTold(t *testing.T) {
t.Errorf("mTLS without an allowlist refused a client: %d", got)
}
}
func TestOnlyANamedCertificateManagesTheMirror(t *testing.T) {
ok := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})
with := func(cn string) *http.Request {
r := httptest.NewRequest("GET", "/", nil)
if cn != "" {
r.TLS = &tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{{{Subject: pkix.Name{CommonName: cn}}}}}
}
return r
}
h := onlyClients([]string{"control-plane"}, ok)
for cn, want := range map[string]int{"control-plane": 200, "node01": 403, "": 403} {
w := httptest.NewRecorder()
h.ServeHTTP(w, with(cn))
if w.Code != want {
t.Errorf("%q: %d, expected %d", cn, w.Code, want)
}
}
w := httptest.NewRecorder()
onlyClients(nil, ok).ServeHTTP(w, with("node01"))
if w.Code != 200 {
t.Error("no names given narrowed it anyway")
}
}
+5
View File
@@ -3,6 +3,7 @@ package cli
import (
"encoding/json"
"fmt"
"os"
"os/user"
"time"
@@ -127,6 +128,10 @@ func newModelRemoveCmd(client func() *ctlClient) *cobra.Command {
// actor names who made a change, for the audit trail on a stored model.
func actor() string {
// Under sudo, the person who ran it rather than root.
if u := os.Getenv("SUDO_USER"); u != "" && os.Geteuid() == 0 {
return u
}
if u, err := user.Current(); err == nil && u.Username != "" {
return u.Username
}
+16
View File
@@ -35,6 +35,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
@@ -90,6 +91,21 @@ func newServerCmd() *cobra.Command {
"or give --endpoint for a router-only run")
}
// Before the router, so its tracer is the configured one.
shutdownTracing, err := obs.SetupTracing(cmd.Context(), obs.TracingOptions{
Endpoint: cfg.Tracing.Endpoint, SampleRatio: cfg.Tracing.SampleRatio,
Service: "llmgw-server", Version: version.Version,
})
if err != nil {
return err
}
defer func() {
// Bounded: a collector that is gone must not hold up a shutdown.
sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = shutdownTracing(sctx)
}()
rt, err := router.New(cfg, log)
if err != nil {
return err
+10 -5
View File
@@ -389,17 +389,22 @@ func (in *serverInit) runInitTUI(ctx context.Context, out io.Writer) error {
return fmt.Errorf("stopped; run init again to finish what was started")
}
// The whole run is kept, root's only, since it holds the password. On a
// failure it is printed too, for the lines the screen had no room for;
// otherwise the screen's last frame says it, and only what must outlive
// the screen is printed again: the password, and what to do next.
// The whole run is kept, root's only, with the password taken out: it is
// shown once, on the screen, and a file on disk would outlive that. On a
// failure the run is printed too, for the lines the screen had no room
// for; otherwise the screen's last frame says it, and only what must
// outlive the screen is printed again: the password, and what to do next.
logMu.Lock()
defer logMu.Unlock()
whole := strings.Join(log, "\n") + "\n"
kept := whole
if fm.password != "" {
kept = strings.ReplaceAll(kept, fm.password, "[password shown once, not kept]")
}
saved := ""
if in.stateDir != "" {
p := filepath.Join(in.stateDir, "init.log")
if err := os.WriteFile(p, []byte(whole), 0o600); err == nil {
if err := os.WriteFile(p, []byte(kept), 0o600); err == nil {
saved = p
}
}