Say when a replica is running an edited profile, and check a source from the web too
CI / test (push) Successful in 1m43s
CI / test (push) Successful in 1m43s
Two defects, both of the same kind: something the CLI knew and the rest of the system did not. A profile can be edited while a replica is running against the old one. The reconciler deliberately leaves it alone — churning a replica that is serving costs more than the imperfection — but there was no way to see it, so the cluster could disagree with its own configuration indefinitely and read as healthy. Profile now carries a fingerprint, a replica records the one it was placed with, and both the API and `ctl replicas` say "stale profile" when they differ. An empty fingerprint is not stale: it means a replica adopted from an older agent, which is unknown rather than wrong. Adding a model from the admin page skipped the source check `ctl model add` has done all along, so a typo or a mirror that is down became a replica failing on a node minutes later, with an error naming the node. The check moves out of the CLI into a package both use. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
832d0a652c
commit
40f5c930c4
14 files changed
+281
-49
No files matched your search
+8
-1
@@ -344,8 +344,15 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
for _, s := range r.Shards {
|
||||
nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, s.GPUs))
|
||||
}
|
||||
// The state says stale rather than a column of its own: most
|
||||
// replicas are not, a column would be empty for all of them,
|
||||
// and when one is it is the first thing worth knowing about it.
|
||||
state := r.State
|
||||
if r.Stale {
|
||||
state += " (stale profile)"
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
r.ID, r.Model, r.State, r.Pool, join(nodes),
|
||||
r.ID, r.Model, state, r.Pool, join(nodes),
|
||||
(time.Duration(r.AgeSec) * time.Second).String(),
|
||||
idleColumn(r.IdleSec), r.Endpoint)
|
||||
// A line rather than a column, because most replicas are not
|
||||
|
||||
@@ -3,6 +3,8 @@ package cli
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
@@ -44,3 +46,29 @@ func TestTheFilenameIsTakenFromTheURLWhenNotGiven(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVersionAnswersHeadAsWellAsGet(t *testing.T) {
|
||||
// A sharded model's catalog source is the version directory, and the serve
|
||||
// path checks a source with HEAD before accepting it. If the manifest
|
||||
// answered only GET, every multi-file model would be refused as missing.
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "llama", "v1"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors"} {
|
||||
if err := os.WriteFile(filepath.Join(root, "llama", "v1", name), []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
h := readOnly(withManifest(root, http.FileServer(http.Dir(root))))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/llama/v1/", nil))
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("a version should answer HEAD; got %d", w.Code)
|
||||
}
|
||||
if got := w.Header().Get("Content-Type"); got != "application/json" {
|
||||
t.Errorf("expected a manifest, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
)
|
||||
|
||||
// newModelCmd adds and removes models without a restart.
|
||||
@@ -70,13 +71,13 @@ func newModelAddCmd(client func() *ctlClient) *cobra.Command {
|
||||
// minutes later on a node — with an error naming the node rather
|
||||
// than the typo.
|
||||
if !skipCheck {
|
||||
size, err := checkSource(cmd.Context(), m.Source)
|
||||
size, err := sourcecheck.Check(cmd.Context(), m.Source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w\n Pass --no-source-check to add it anyway, "+
|
||||
"which is right when the mirror is not up yet or the nodes "+
|
||||
"reach it by an address this machine does not.", err)
|
||||
}
|
||||
if _, isURL := httpSource(m.Source); isURL {
|
||||
if _, isURL := sourcecheck.HTTPSource(m.Source); isURL {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), sourceNote(m.Source, size))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// checkSource asks whether a model's weights are actually there.
|
||||
//
|
||||
// A source that does not resolve is caught today by a replica failing to start,
|
||||
// minutes later, on a node — and the error names the node rather than the typo.
|
||||
// Asking now turns that into a sentence while the operator still has the file
|
||||
// open.
|
||||
//
|
||||
// Only URLs. A path is checked on the node that will use it, by the node, which
|
||||
// is the only machine whose answer means anything; a bare model id is resolved
|
||||
// by the engine and is not this function's business.
|
||||
func checkSource(ctx context.Context, source string) (size int64, err error) {
|
||||
url, ok := httpSource(source)
|
||||
if !ok {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot reach %s: %w", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return 0, fmt.Errorf("%s answered %s", url, resp.Status)
|
||||
}
|
||||
// Absent when the server does not send it, which is not an error: a
|
||||
// resumable fetch does not need the length in advance, and refusing here
|
||||
// would reject a mirror that is merely terse. Go reports that as -1;
|
||||
// normalised to 0 so callers have one spelling of "unknown".
|
||||
if resp.ContentLength < 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return resp.ContentLength, nil
|
||||
}
|
||||
|
||||
// httpSource returns the URL to check, and whether there is one.
|
||||
//
|
||||
// hf: is expanded the way the fetcher expands it, so the check asks about the
|
||||
// same file the download will ask for rather than about a string that looks
|
||||
// similar.
|
||||
func httpSource(source string) (string, bool) {
|
||||
switch {
|
||||
case strings.HasPrefix(source, "http://"), strings.HasPrefix(source, "https://"):
|
||||
return source, true
|
||||
case strings.HasPrefix(source, "hf:"):
|
||||
rest := strings.TrimPrefix(source, "hf:")
|
||||
parts := strings.SplitN(rest, "/", 3)
|
||||
if len(parts) != 3 {
|
||||
// A repository rather than a file. vLLM and SGLang download those
|
||||
// themselves, so there is nothing here to check.
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("https://huggingface.co/%s/%s/resolve/main/%s",
|
||||
parts[0], parts[1], parts[2]), true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// sourceNote is what to print about a checked source.
|
||||
//
|
||||
// It says where the answer came from, because the control plane's network is
|
||||
// not the nodes' network: a mirror this machine can reach may be unreachable
|
||||
// from a node behind a different firewall, and the reverse. The check catches
|
||||
// typos and a mirror that is down, which is most of what goes wrong, and it
|
||||
// should not be read as proof that the fetch will work.
|
||||
func sourceNote(source string, size int64) string {
|
||||
if size > 0 {
|
||||
return fmt.Sprintf(" source reachable from here: %s (%s)", source, fileSize(size))
|
||||
}
|
||||
return fmt.Sprintf(" source reachable from here: %s (size not advertised)", source)
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOnlyURLsAreChecked(t *testing.T) {
|
||||
// A path is checked by the node that will use it, which is the only
|
||||
// machine whose answer means anything — the control plane may not even
|
||||
// have that filesystem. A bare model id is the engine's business.
|
||||
for _, src := range []string{
|
||||
"/var/lib/llmgw-models/x.gguf", "./x.gguf", "Qwen/Qwen2.5-7B-Instruct",
|
||||
} {
|
||||
if _, ok := httpSource(src); ok {
|
||||
t.Errorf("%q was treated as a URL to check", src)
|
||||
}
|
||||
if _, err := checkSource(context.Background(), src); err != nil {
|
||||
t.Errorf("checking %q returned %v; it should be skipped", src, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnHFFileIsCheckedAtTheURLTheFetcherWouldUse(t *testing.T) {
|
||||
// Asking about a string that merely looks similar would be worse than not
|
||||
// asking: it would pass while the download later fails.
|
||||
got, ok := httpSource("hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/q4.gguf")
|
||||
if !ok {
|
||||
t.Fatal("an hf: file was not recognised")
|
||||
}
|
||||
want := "https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/resolve/main/q4.gguf"
|
||||
if got != want {
|
||||
t.Errorf("hf: resolved to %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnHFRepositoryIsNotChecked(t *testing.T) {
|
||||
// vLLM and SGLang download whole repositories themselves; there is no one
|
||||
// file to ask about.
|
||||
if _, ok := httpSource("hf:Qwen/Qwen2.5-7B-Instruct"); ok {
|
||||
t.Error("a repository was treated as a single file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReachableSourceReportsItsSize(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodHead {
|
||||
t.Errorf("the check used %s, not HEAD — it must not download the file", r.Method)
|
||||
}
|
||||
w.Header().Set("Content-Length", "1073741824")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
size, err := checkSource(context.Background(), srv.URL+"/model.gguf")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if size != 1<<30 {
|
||||
t.Errorf("size = %d", size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMissingFileSaysSoWithTheURL(t *testing.T) {
|
||||
// The whole point: a typo becomes a sentence while the operator still has
|
||||
// the file open, rather than a replica failing minutes later on a node
|
||||
// with an error that names the node.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
_, err := checkSource(context.Background(), srv.URL+"/typo.gguf")
|
||||
if err == nil {
|
||||
t.Fatal("a 404 was accepted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "typo.gguf") {
|
||||
t.Errorf("the error does not name the URL: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAServerThatSendsNoLengthIsStillFine(t *testing.T) {
|
||||
// A resumable fetch does not need the length in advance, and refusing here
|
||||
// would reject a mirror that is merely terse.
|
||||
// A server that does not know the length omits the header rather than
|
||||
// sending a negative one, which is what a real mirror streaming from a
|
||||
// pipe does.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
size, err := checkSource(context.Background(), srv.URL+"/x.gguf")
|
||||
if err != nil {
|
||||
t.Errorf("a terse server was rejected: %v", err)
|
||||
}
|
||||
if size != 0 {
|
||||
t.Errorf("an unknown length came back as %d, want 0", size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNoteSaysWhereTheAnswerCameFrom(t *testing.T) {
|
||||
// The control plane's network is not the nodes' network. The check catches
|
||||
// typos and a mirror that is down; it is not proof the fetch will work,
|
||||
// and the wording must not suggest it is.
|
||||
note := sourceNote("http://models.internal/x.gguf", 1<<30)
|
||||
if !strings.Contains(note, "from here") {
|
||||
t.Errorf("the note overstates what was checked: %s", note)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package cli
|
||||
|
||||
import "fmt"
|
||||
|
||||
// sourceNote is what to print about a checked source.
|
||||
//
|
||||
// It says where the answer came from, because the control plane's network is
|
||||
// not the nodes' network: a mirror this machine can reach may be unreachable
|
||||
// from a node behind a different firewall, and the reverse. The check catches
|
||||
// typos and a mirror that is down, which is most of what goes wrong, and it
|
||||
// should not be read as proof that the fetch will work.
|
||||
func sourceNote(source string, size int64) string {
|
||||
if size > 0 {
|
||||
return fmt.Sprintf(" source reachable from here: %s (%s)", source, fileSize(size))
|
||||
}
|
||||
return fmt.Sprintf(" source reachable from here: %s (size not advertised)", source)
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTheNoteSaysWhereTheAnswerCameFrom(t *testing.T) {
|
||||
// The control plane's network is not the nodes' network. The check catches
|
||||
// typos and a mirror that is down; it is not proof the fetch will work,
|
||||
// and the wording must not suggest it is.
|
||||
note := sourceNote("http://models.internal/x.gguf", 1<<30)
|
||||
if !strings.Contains(note, "from here") {
|
||||
t.Errorf("the note overstates what was checked: %s", note)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user