A model can be stopped and kept, or removed with its files from the mirror and the nodes
CI / test (push) Successful in 10m37s
CI / test (push) Successful in 10m37s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5726ceb84e
commit
59187cdae5
19 files changed
+760
-14
No files matched your search
@@ -228,6 +228,8 @@ func (s *Service) Remove(ctx context.Context, name string) error {
|
||||
func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) {
|
||||
final, err := s.update(ctx, name, by, "change its scaling", func(m *catalog.Model) error {
|
||||
m.Scaling = sc
|
||||
// A number set is what it runs now: a stopped model is started by it.
|
||||
m.Stopped = nil
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
@@ -237,6 +239,39 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by
|
||||
return final, err
|
||||
}
|
||||
|
||||
// Stop takes a model out of service and keeps it: none of its replicas run,
|
||||
// and what it was scaled to is kept for Start.
|
||||
func (s *Service) Stop(ctx context.Context, name, by string) (catalog.Model, error) {
|
||||
final, err := s.update(ctx, name, by, "stop it", func(m *catalog.Model) error {
|
||||
if m.Stopped != nil {
|
||||
return fmt.Errorf("model %q is already stopped; llmgw ctl model start %s runs it again", name, name)
|
||||
}
|
||||
was := m.Scaling
|
||||
m.Stopped = &was
|
||||
m.Scaling = catalog.Scaling{IdleTimeout: was.IdleTimeout}
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
s.log.Info("model stopped", "model", name, "by", by)
|
||||
}
|
||||
return final, err
|
||||
}
|
||||
|
||||
// Start runs a stopped model again, scaled as it was when it was stopped.
|
||||
func (s *Service) Start(ctx context.Context, name, by string) (catalog.Model, error) {
|
||||
final, err := s.update(ctx, name, by, "start it", func(m *catalog.Model) error {
|
||||
if m.Stopped == nil {
|
||||
return fmt.Errorf("model %q is not stopped; llmgw ctl scale changes how many replicas it runs", name)
|
||||
}
|
||||
m.Scaling, m.Stopped = *m.Stopped, nil
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
s.log.Info("model started", "model", name, "min", final.Scaling.Min, "max", final.Scaling.Max, "by", by)
|
||||
}
|
||||
return final, err
|
||||
}
|
||||
|
||||
// Resources sets what a model's profile reserves of a node: host memory, and
|
||||
// the CPU cores a replica without a GPU computes on. An empty profile name
|
||||
// sets every profile.
|
||||
|
||||
@@ -400,3 +400,41 @@ func TestAddNewLeavesAModelAddedMeanwhile(t *testing.T) {
|
||||
t.Errorf("a new model: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoppedModelRunsNoneAndStartsAsItWas(t *testing.T) {
|
||||
s, _, rec := newService(t)
|
||||
if err := s.Add(t.Context(), model("added"), "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Scale(t.Context(), "added", catalog.Scaling{Min: 2, Max: 4, IdleTimeout: 20 * time.Minute}, "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := s.Stop(t.Context(), "added", "ilker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Scaling.Min != 0 || got.Scaling.Max != 0 || got.Stopped == nil || got.Stopped.Min != 2 {
|
||||
t.Fatalf("stopped: %+v %+v", got.Scaling, got.Stopped)
|
||||
}
|
||||
if m, _ := rec.last.Resolve("added"); m.Scaling.Max != 0 || m.Scaling.OnDemand() {
|
||||
t.Errorf("the catalog in force runs it: %+v", m.Scaling)
|
||||
}
|
||||
if _, err := s.Stop(t.Context(), "added", "ilker"); err == nil {
|
||||
t.Error("stopped twice")
|
||||
}
|
||||
got, err = s.Start(t.Context(), "added", "ilker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Scaling.Min != 2 || got.Scaling.Max != 4 || got.Scaling.IdleTimeout != 20*time.Minute || got.Stopped != nil {
|
||||
t.Errorf("started: %+v %+v", got.Scaling, got.Stopped)
|
||||
}
|
||||
if _, err := s.Start(t.Context(), "added", "ilker"); err == nil {
|
||||
t.Error("started what was not stopped")
|
||||
}
|
||||
// A scale while stopped starts it at that number.
|
||||
_, _ = s.Stop(t.Context(), "added", "ilker")
|
||||
if got, _ := s.Scale(t.Context(), "added", catalog.Scaling{Min: 1}, "ilker"); got.Stopped != nil {
|
||||
t.Error("still stopped after a scale")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user