The mirror holds the weights, so the mirror does the downloading. A control plane that had to pull the bytes itself and push them here would carry every model twice over the network, through the one machine least able to spare the bandwidth. Two listeners, and the split is the point. Weights stay on plain HTTP where every node can reach them: that path is eight nodes pulling forty gigabytes, and putting a client certificate in front of it would mean re-certifying every agent's fetcher in order to change how models are administered. Management gets its own port, which can be given the fleet's CA — the same tlsconf the agent stream uses, so this is a configuration rather than a new idea. A fetch answers 202, not 200. The bytes are not here yet, and a caller that read it as "done" would point a catalog at a file still arriving, which every node would then copy. Progress is asked for rather than waited on, because no client between here and the operator will hold a connection open for an hour. A failed fetch is remembered with its reason. Forgetting it would leave the control plane waiting on a model that is never coming, with nothing to say why. Status falls back to reading the disk, so a restart does not report a model that is plainly there as unknown. Removal takes the whole version: a model split across files is useless with one of them missing, and half a version on disk looks present and will not load. It refuses while a download is running, because the fetch would carry on writing and the file would reappear, looking like the delete had silently failed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
252 lines
7.3 KiB
Go
252 lines
7.3 KiB
Go
package mirror
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// State is where a model stands in the mirror.
|
|
type State string
|
|
|
|
// The states a model can be in.
|
|
const (
|
|
// Fetching means a download is running. It is the state that makes this
|
|
// service worth an API: without it the control plane cannot tell a model
|
|
// that is arriving from one that will never arrive.
|
|
Fetching State = "fetching"
|
|
Present State = "present"
|
|
Failed State = "failed"
|
|
)
|
|
|
|
// Status is one model's state, and how far a download has got.
|
|
type Status struct {
|
|
Model string `json:"model"`
|
|
Version string `json:"version"`
|
|
File string `json:"file"`
|
|
State State `json:"state"`
|
|
Source string `json:"source,omitempty"`
|
|
|
|
BytesDone int64 `json:"bytes_done"`
|
|
// BytesTotal is 0 when the upstream server did not say. A percentage would
|
|
// then be invented, and an invented one that stops moving is worse than a
|
|
// byte count that keeps rising.
|
|
BytesTotal int64 `json:"bytes_total,omitempty"`
|
|
SHA256 string `json:"sha256,omitempty"`
|
|
Err string `json:"error,omitempty"`
|
|
|
|
StartedAt time.Time `json:"started_at,omitzero"`
|
|
FinishedAt time.Time `json:"finished_at,omitzero"`
|
|
}
|
|
|
|
// Percent reports progress, and false when the total is unknown.
|
|
func (s Status) Percent() (int, bool) {
|
|
if s.BytesTotal <= 0 {
|
|
return 0, false
|
|
}
|
|
return int(s.BytesDone * 100 / s.BytesTotal), true
|
|
}
|
|
|
|
// Manager runs downloads in the background and remembers how they went.
|
|
//
|
|
// Background because a model is tens of gigabytes: a request that waited for
|
|
// one would time out in every client between here and the operator, and the
|
|
// operator would learn nothing about why. So a fetch is started, and its
|
|
// progress is asked for.
|
|
type Manager struct {
|
|
root string
|
|
log *slog.Logger
|
|
|
|
mu sync.Mutex
|
|
// running is keyed by model/version, which is how a second request for a
|
|
// download already in flight joins it rather than starting a rival one
|
|
// writing to the same file.
|
|
running map[string]*Status
|
|
}
|
|
|
|
// NewManager prepares a manager over a mirror root.
|
|
func NewManager(root string, log *slog.Logger) *Manager {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Manager{root: root, log: log, running: map[string]*Status{}}
|
|
}
|
|
|
|
// ErrAlreadyFetching means a download for that model and version is running.
|
|
var ErrAlreadyFetching = errors.New("a download for that version is already running")
|
|
|
|
// Start begins a download and returns once it is under way.
|
|
//
|
|
// The context bounds the caller's request, not the download: a fetch outlives
|
|
// the API call that asked for it, which is the point of doing it here rather
|
|
// than making the caller hold a connection open for an hour.
|
|
func (m *Manager) Start(source string, e Entry) (Status, error) {
|
|
if source == "" {
|
|
return Status{}, fmt.Errorf("a source URL is required")
|
|
}
|
|
key := e.Model + "/" + e.Version
|
|
|
|
m.mu.Lock()
|
|
if st, ok := m.running[key]; ok && st.State == Fetching {
|
|
// Copied under the lock. The download goroutine writes progress into
|
|
// this struct, so dereferencing it after unlocking would be a read
|
|
// racing a write — of a byte count nobody would notice was wrong.
|
|
out := *st
|
|
m.mu.Unlock()
|
|
return out, ErrAlreadyFetching
|
|
}
|
|
st := &Status{
|
|
Model: e.Model, Version: e.Version, File: e.File,
|
|
State: Fetching, Source: source, StartedAt: time.Now(),
|
|
}
|
|
m.running[key] = st
|
|
out := *st
|
|
m.mu.Unlock()
|
|
|
|
go m.run(source, e, key)
|
|
return out, nil
|
|
}
|
|
|
|
func (m *Manager) run(source string, e Entry, key string) {
|
|
// Its own context, deliberately unbounded by any request. A download is
|
|
// minutes to hours and the thing that asked for it is long gone.
|
|
ctx := context.Background()
|
|
|
|
added, err := Add(ctx, m.root, source, e, func(done, total int64) {
|
|
m.mu.Lock()
|
|
if st, ok := m.running[key]; ok {
|
|
st.BytesDone, st.BytesTotal = done, total
|
|
}
|
|
m.mu.Unlock()
|
|
})
|
|
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
st, ok := m.running[key]
|
|
if !ok {
|
|
return
|
|
}
|
|
st.FinishedAt = time.Now()
|
|
if err != nil {
|
|
st.State, st.Err = Failed, err.Error()
|
|
// Kept rather than forgotten: a failed fetch that vanished would leave
|
|
// the control plane waiting for a model that is never coming, with
|
|
// nothing to say why.
|
|
m.log.Error("mirror fetch failed", "model", e.Model, "version", e.Version,
|
|
"source", source, "err", err)
|
|
return
|
|
}
|
|
st.State = Present
|
|
st.BytesDone, st.BytesTotal = added.Entry.Bytes, added.Entry.Bytes
|
|
st.SHA256 = added.SHA256
|
|
m.log.Info("mirror fetched a model", "model", e.Model, "version", e.Version,
|
|
"bytes", added.Entry.Bytes, "sha256", added.SHA256)
|
|
}
|
|
|
|
// Status reports one model's state, reading the disk when no download for it
|
|
// is remembered.
|
|
//
|
|
// The disk is the truth: a restart forgets every download, and a model that is
|
|
// present should not be reported as unknown merely because this process did not
|
|
// fetch it.
|
|
func (m *Manager) Status(model, version string) (Status, bool) {
|
|
key := model + "/" + version
|
|
|
|
m.mu.Lock()
|
|
st, ok := m.running[key]
|
|
if ok {
|
|
out := *st
|
|
m.mu.Unlock()
|
|
return out, true
|
|
}
|
|
m.mu.Unlock()
|
|
|
|
entries, err := List(m.root)
|
|
if err != nil {
|
|
return Status{}, false
|
|
}
|
|
for _, e := range entries {
|
|
if e.Model == model && e.Version == version {
|
|
return Status{
|
|
Model: e.Model, Version: e.Version, File: e.File,
|
|
State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes,
|
|
}, true
|
|
}
|
|
}
|
|
return Status{}, false
|
|
}
|
|
|
|
// All reports everything the mirror holds or is fetching.
|
|
//
|
|
// Both, because a control plane deciding whether to place a model needs to
|
|
// know the difference between a file that is here and one that is on its way —
|
|
// and to see that the one it asked for is on its way at all.
|
|
func (m *Manager) All() ([]Status, error) {
|
|
entries, err := List(m.root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]Status, 0, len(entries))
|
|
seen := map[string]struct{}{}
|
|
for _, e := range entries {
|
|
out = append(out, Status{
|
|
Model: e.Model, Version: e.Version, File: e.File,
|
|
State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes,
|
|
})
|
|
seen[e.Model+"/"+e.Version] = struct{}{}
|
|
}
|
|
|
|
m.mu.Lock()
|
|
for key, st := range m.running {
|
|
if _, onDisk := seen[key]; onDisk {
|
|
continue
|
|
}
|
|
out = append(out, *st)
|
|
}
|
|
m.mu.Unlock()
|
|
return out, nil
|
|
}
|
|
|
|
// Forget drops a remembered download, so a failed one can be retried.
|
|
//
|
|
// Without it a fetch that failed once would refuse every later attempt at the
|
|
// same version, and the only way through would be restarting the mirror.
|
|
func (m *Manager) Forget(model, version string) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
delete(m.running, model+"/"+version)
|
|
}
|
|
|
|
// RemoveVersion deletes every file under one model version and reports what
|
|
// went.
|
|
//
|
|
// The version is the unit, not the file: a model split across several files is
|
|
// useless with one of them missing, so removing part of it leaves a directory
|
|
// that looks present and cannot be loaded.
|
|
func (m *Manager) RemoveVersion(model, version string) ([]string, error) {
|
|
entries, err := List(m.root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var removed []string
|
|
for _, e := range entries {
|
|
if e.Model != model || e.Version != version {
|
|
continue
|
|
}
|
|
if err := Remove(m.root, e); err != nil {
|
|
return removed, err
|
|
}
|
|
removed = append(removed, e.File)
|
|
}
|
|
if len(removed) == 0 {
|
|
return nil, fmt.Errorf("%s/%s: %w", model, version, os.ErrNotExist)
|
|
}
|
|
m.Forget(model, version)
|
|
m.log.Info("mirror removed a version", "model", model, "version", version, "files", len(removed))
|
|
return removed, nil
|
|
}
|