The control plane uses whichever mirror answers

mirror.peers names the mirrors kept in step with mirror.api. The
mirror's management calls go to the first that can be reached, and
each mirror is asked every fifteen seconds whether it answers: a shard
is sent its model's and adapters' address on one that does while the
mirror the catalog names is down.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 13:43:27 +01:00
1 parent 5bc71b3d0a
commit 8a76a53afb
11 files changed
+276 -21

No files matched your search

+55 -17
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -24,6 +25,29 @@ import (
type Client struct {
base string
hc *http.Client
// others are the mirrors kept in step with this one, asked in turn when
// it cannot be reached: the same models, on another machine.
others []string
}
// NewFailoverClient is NewClient over several mirrors kept in step: the
// first answers, and the next is asked when it cannot be reached.
func NewFailoverClient(addrs []string, tc tlsconf.Config, timeout time.Duration) (*Client, error) {
if len(addrs) == 0 {
return nil, fmt.Errorf("no mirror address")
}
c, err := NewClient(addrs[0], tc, timeout)
if err != nil {
return nil, err
}
for _, a := range addrs[1:] {
u, err := url.Parse(a)
if err != nil || u.Host == "" {
return nil, fmt.Errorf("the mirror address %q is not a URL with a host", a)
}
c.others = append(c.others, strings.TrimSuffix(a, "/"))
}
return c, nil
}
// NewClient dials a mirror's management API, with mutual TLS when configured.
@@ -149,28 +173,42 @@ func (c *Client) Remove(ctx context.Context, model, version string) ([]string, e
}
func (c *Client) do(ctx context.Context, method, path string, body, out any) error {
var rdr io.Reader
var b []byte
if body != nil {
b, err := json.Marshal(body)
var err error
if b, err = json.Marshal(body); err != nil {
return err
}
}
// The mirror first, then those kept in step with it, while each cannot
// be reached at all: one that answers, even with a refusal, is the one.
var resp *http.Response
var errs []error
for _, base := range append([]string{c.base}, c.others...) {
var rdr io.Reader
if body != nil {
rdr = bytes.NewReader(b)
}
req, err := http.NewRequestWithContext(ctx, method, base+path, rdr)
if err != nil {
return err
}
rdr = bytes.NewReader(b)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
if resp, err = c.hc.Do(req); err == nil {
break
}
// Wrapped with the address because the failure an operator sees on
// a page is otherwise "connection refused" with no clue which of
// the machines refused it.
errs = append(errs, fmt.Errorf("reaching the mirror at %s: %w", base, err))
if ctx.Err() != nil {
break
}
}
req, err := http.NewRequestWithContext(ctx, method, c.base+path, rdr)
if err != nil {
return err
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := c.hc.Do(req)
if err != nil {
// Wrapped with the address because the failure an operator sees on a
// page is otherwise "connection refused" with no clue which of the two
// machines refused it.
return fmt.Errorf("reaching the mirror at %s: %w", c.base, err)
if resp == nil {
return errors.Join(errs...)
}
defer resp.Body.Close()