The control plane uses whichever mirror answers
mirror.peers names the mirrors kept in step with mirror.api. The mirror's management calls go to the first that can be reached, and each mirror is asked every fifteen seconds whether it answers: a shard is sent its model's and adapters' address on one that does while the mirror the catalog names is down. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5bc71b3d0a
commit
8a76a53afb
11 files changed
+276
-21
No files matched your search
+55
-17
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -24,6 +25,29 @@ import (
|
||||
type Client struct {
|
||||
base string
|
||||
hc *http.Client
|
||||
// others are the mirrors kept in step with this one, asked in turn when
|
||||
// it cannot be reached: the same models, on another machine.
|
||||
others []string
|
||||
}
|
||||
|
||||
// NewFailoverClient is NewClient over several mirrors kept in step: the
|
||||
// first answers, and the next is asked when it cannot be reached.
|
||||
func NewFailoverClient(addrs []string, tc tlsconf.Config, timeout time.Duration) (*Client, error) {
|
||||
if len(addrs) == 0 {
|
||||
return nil, fmt.Errorf("no mirror address")
|
||||
}
|
||||
c, err := NewClient(addrs[0], tc, timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, a := range addrs[1:] {
|
||||
u, err := url.Parse(a)
|
||||
if err != nil || u.Host == "" {
|
||||
return nil, fmt.Errorf("the mirror address %q is not a URL with a host", a)
|
||||
}
|
||||
c.others = append(c.others, strings.TrimSuffix(a, "/"))
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// NewClient dials a mirror's management API, with mutual TLS when configured.
|
||||
@@ -149,28 +173,42 @@ func (c *Client) Remove(ctx context.Context, model, version string) ([]string, e
|
||||
}
|
||||
|
||||
func (c *Client) do(ctx context.Context, method, path string, body, out any) error {
|
||||
var rdr io.Reader
|
||||
var b []byte
|
||||
if body != nil {
|
||||
b, err := json.Marshal(body)
|
||||
var err error
|
||||
if b, err = json.Marshal(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The mirror first, then those kept in step with it, while each cannot
|
||||
// be reached at all: one that answers, even with a refusal, is the one.
|
||||
var resp *http.Response
|
||||
var errs []error
|
||||
for _, base := range append([]string{c.base}, c.others...) {
|
||||
var rdr io.Reader
|
||||
if body != nil {
|
||||
rdr = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, base+path, rdr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rdr = bytes.NewReader(b)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if resp, err = c.hc.Do(req); err == nil {
|
||||
break
|
||||
}
|
||||
// Wrapped with the address because the failure an operator sees on
|
||||
// a page is otherwise "connection refused" with no clue which of
|
||||
// the machines refused it.
|
||||
errs = append(errs, fmt.Errorf("reaching the mirror at %s: %w", base, err))
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.base+path, rdr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
|
||||
resp, err := c.hc.Do(req)
|
||||
if err != nil {
|
||||
// Wrapped with the address because the failure an operator sees on a
|
||||
// page is otherwise "connection refused" with no clue which of the two
|
||||
// machines refused it.
|
||||
return fmt.Errorf("reaching the mirror at %s: %w", c.base, err)
|
||||
if resp == nil {
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
|
||||
@@ -125,3 +125,50 @@ func TestACopyWhoseChecksumIsNotThePeersIsNotKept(t *testing.T) {
|
||||
t.Errorf("a copy with the wrong checksum was kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A mirror that cannot be reached is passed over for one kept in step with it.
|
||||
func TestTheNextMirrorIsAskedWhenTheFirstCannotBeReached(t *testing.T) {
|
||||
a, b := newTestMirror(t), newTestMirror(t)
|
||||
c, err := NewFailoverClient([]string{a.api.URL, b.api.URL}, tlsconf.Config{}, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if list, err := c.List(t.Context()); err != nil || list.Base != a.files.URL {
|
||||
t.Fatalf("both up: %+v %v", list, err)
|
||||
}
|
||||
a.api.Close()
|
||||
if list, err := c.List(t.Context()); err != nil || list.Base != b.files.URL {
|
||||
t.Errorf("the first down: %+v %v", list, err)
|
||||
}
|
||||
b.api.Close()
|
||||
if _, err := c.List(t.Context()); err == nil {
|
||||
t.Error("both down, and an answer")
|
||||
}
|
||||
}
|
||||
|
||||
// A node is given a model's file on a mirror that answers.
|
||||
func TestAModelsAddressIsMovedToAMirrorThatAnswers(t *testing.T) {
|
||||
a, b := newTestMirror(t), newTestMirror(t)
|
||||
s, err := NewSources([]string{a.api.URL, b.api.URL}, tlsconf.Config{}, discardLog())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.Check(t.Context())
|
||||
src := a.files.URL + "/qwen/v1/m.gguf"
|
||||
if got := s.Rewrite(src); got != src {
|
||||
t.Errorf("its own mirror up: %s", got)
|
||||
}
|
||||
a.api.Close()
|
||||
s.Check(t.Context())
|
||||
if got := s.Rewrite(src); got != b.files.URL+"/qwen/v1/m.gguf" {
|
||||
t.Errorf("its own mirror down: %s", got)
|
||||
}
|
||||
if got := s.Rewrite("https://huggingface.co/x/y"); got != "https://huggingface.co/x/y" {
|
||||
t.Errorf("an address on no mirror: %s", got)
|
||||
}
|
||||
b.api.Close()
|
||||
s.Check(t.Context())
|
||||
if got := s.Rewrite(src); got != src {
|
||||
t.Errorf("none up: %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package mirror
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
// Sources is where a node fetches a model while one mirror is down. A model
|
||||
// in the catalog names one mirror's address; when that mirror does not
|
||||
// answer, the address a node is given is the same file on one that does.
|
||||
|
||||
// SourcesInterval is how often each mirror is asked whether it answers.
|
||||
const SourcesInterval = 15 * time.Second
|
||||
|
||||
// Sources watches mirrors kept in step, and rewrites a model's address to
|
||||
// one that answers.
|
||||
type Sources struct {
|
||||
clients []*Client
|
||||
log *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
bases []string
|
||||
healthy []bool
|
||||
}
|
||||
|
||||
// NewSources watches the mirrors at these management addresses.
|
||||
func NewSources(addrs []string, tc tlsconf.Config, log *slog.Logger) (*Sources, error) {
|
||||
s := &Sources{log: log, bases: make([]string, len(addrs)), healthy: make([]bool, len(addrs))}
|
||||
for _, a := range addrs {
|
||||
c, err := NewClient(a, tc, 5*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.clients = append(s.clients, c)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Run asks each mirror, every while, until ctx ends.
|
||||
func (s *Sources) Run(ctx context.Context) {
|
||||
t := time.NewTicker(SourcesInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
s.Check(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check asks each mirror once: whether it answers, and the address its
|
||||
// files are served at.
|
||||
func (s *Sources) Check(ctx context.Context) {
|
||||
for i, c := range s.clients {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
list, err := c.List(ctx)
|
||||
cancel()
|
||||
s.mu.Lock()
|
||||
was := s.healthy[i]
|
||||
s.healthy[i] = err == nil
|
||||
if err == nil && list.Base != "" {
|
||||
s.bases[i] = strings.TrimSuffix(list.Base, "/")
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if was && err != nil {
|
||||
s.log.Warn("a mirror does not answer; models it holds are fetched from another", "mirror", c.Addr(), "err", err)
|
||||
} else if !was && err == nil {
|
||||
s.log.Info("a mirror answers", "mirror", c.Addr(), "files", list.Base)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rewrite is a model's address on a mirror that answers: as it is when its
|
||||
// own mirror answers or no other does, else the same path on the first that
|
||||
// does.
|
||||
func (s *Sources) Rewrite(src string) string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
from := -1
|
||||
for i, b := range s.bases {
|
||||
if b != "" && (src == b || strings.HasPrefix(src, b+"/")) {
|
||||
from = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if from < 0 || s.healthy[from] {
|
||||
return src
|
||||
}
|
||||
for i, b := range s.bases {
|
||||
if i != from && b != "" && s.healthy[i] {
|
||||
return b + strings.TrimPrefix(src, s.bases[from])
|
||||
}
|
||||
}
|
||||
return src
|
||||
}
|
||||
Reference in new issue
Block a user