package mirror import ( "context" "errors" "fmt" "log/slog" "os" "sync" "time" ) // State is where a model stands in the mirror. type State string // The states a model can be in. const ( // Fetching means a download is running. It is the state that makes this // service worth an API: without it the control plane cannot tell a model // that is arriving from one that will never arrive. Fetching State = "fetching" Present State = "present" Failed State = "failed" ) // Status is one model's state, and how far a download has got. type Status struct { Model string `json:"model"` Version string `json:"version"` File string `json:"file"` State State `json:"state"` Source string `json:"source,omitempty"` BytesDone int64 `json:"bytes_done"` // BytesTotal is 0 when the upstream server did not say. A percentage would // then be invented, and an invented one that stops moving is worse than a // byte count that keeps rising. BytesTotal int64 `json:"bytes_total,omitempty"` SHA256 string `json:"sha256,omitempty"` Err string `json:"error,omitempty"` StartedAt time.Time `json:"started_at,omitzero"` FinishedAt time.Time `json:"finished_at,omitzero"` } // Percent reports progress, and false when the total is unknown. func (s Status) Percent() (int, bool) { if s.BytesTotal <= 0 { return 0, false } return int(s.BytesDone * 100 / s.BytesTotal), true } // Manager runs downloads in the background and remembers how they went. // // Background because a model is tens of gigabytes: a request that waited for // one would time out in every client between here and the operator, and the // operator would learn nothing about why. So a fetch is started, and its // progress is asked for. type Manager struct { root string log *slog.Logger mu sync.Mutex // running is keyed by model/version, which is how a second request for a // download already in flight joins it rather than starting a rival one // writing to the same file. running map[string]*Status } // NewManager prepares a manager over a mirror root. func NewManager(root string, log *slog.Logger) *Manager { if log == nil { log = slog.Default() } return &Manager{root: root, log: log, running: map[string]*Status{}} } // ErrAlreadyFetching means a download for that model and version is running. var ErrAlreadyFetching = errors.New("a download for that version is already running") // Start begins a download and returns once it is under way. // // The context bounds the caller's request, not the download: a fetch outlives // the API call that asked for it, which is the point of doing it here rather // than making the caller hold a connection open for an hour. func (m *Manager) Start(source string, e Entry) (Status, error) { if source == "" { return Status{}, fmt.Errorf("a source URL is required") } key := e.Model + "/" + e.Version m.mu.Lock() if st, ok := m.running[key]; ok && st.State == Fetching { // Copied under the lock. The download goroutine writes progress into // this struct, so dereferencing it after unlocking would be a read // racing a write — of a byte count nobody would notice was wrong. out := *st m.mu.Unlock() return out, ErrAlreadyFetching } st := &Status{ Model: e.Model, Version: e.Version, File: e.File, State: Fetching, Source: source, StartedAt: time.Now(), } m.running[key] = st out := *st m.mu.Unlock() go m.run(source, e, key) return out, nil } func (m *Manager) run(source string, e Entry, key string) { // Its own context, deliberately unbounded by any request. A download is // minutes to hours and the thing that asked for it is long gone. ctx := context.Background() added, err := Add(ctx, m.root, source, e, func(done, total int64) { m.mu.Lock() if st, ok := m.running[key]; ok { st.BytesDone, st.BytesTotal = done, total } m.mu.Unlock() }) m.mu.Lock() defer m.mu.Unlock() st, ok := m.running[key] if !ok { return } st.FinishedAt = time.Now() if err != nil { st.State, st.Err = Failed, err.Error() // Kept rather than forgotten: a failed fetch that vanished would leave // the control plane waiting for a model that is never coming, with // nothing to say why. m.log.Error("mirror fetch failed", "model", e.Model, "version", e.Version, "source", source, "err", err) return } st.State = Present st.BytesDone, st.BytesTotal = added.Entry.Bytes, added.Entry.Bytes st.SHA256 = added.SHA256 m.log.Info("mirror fetched a model", "model", e.Model, "version", e.Version, "bytes", added.Entry.Bytes, "sha256", added.SHA256) } // Status reports one model's state, reading the disk when no download for it // is remembered. // // The disk is the truth: a restart forgets every download, and a model that is // present should not be reported as unknown merely because this process did not // fetch it. func (m *Manager) Status(model, version string) (Status, bool) { key := model + "/" + version m.mu.Lock() st, ok := m.running[key] if ok { out := *st m.mu.Unlock() return out, true } m.mu.Unlock() entries, err := List(m.root) if err != nil { return Status{}, false } for _, e := range entries { if e.Model == model && e.Version == version { return Status{ Model: e.Model, Version: e.Version, File: e.File, State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes, }, true } } return Status{}, false } // All reports everything the mirror holds or is fetching. // // Both, because a control plane deciding whether to place a model needs to // know the difference between a file that is here and one that is on its way — // and to see that the one it asked for is on its way at all. func (m *Manager) All() ([]Status, error) { entries, err := List(m.root) if err != nil { return nil, err } // Taken before the disk entries are built so each one can be told what this // process knows about it that the filesystem cannot say. m.mu.Lock() known := make(map[string]Status, len(m.running)) for key, st := range m.running { known[key] = *st } m.mu.Unlock() out := make([]Status, 0, len(entries)) seen := map[string]struct{}{} for _, e := range entries { key := e.Model + "/" + e.Version st := Status{ Model: e.Model, Version: e.Version, File: e.File, State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes, } // The checksum and the source are not recoverable from a directory // listing, and the checksum is the one field worth having: it is what // an operator compares against what upstream published. Dropping it in // favour of the disk entry lost it at the exact moment the file became // verifiable — the download finishing was what made it disappear. if k, ok := known[key]; ok && k.File == e.File { st.SHA256, st.Source = k.SHA256, k.Source } out = append(out, st) seen[key] = struct{}{} } for key, st := range known { if _, onDisk := seen[key]; onDisk { continue } out = append(out, st) } return out, nil } // Forget drops a remembered download, so a failed one can be retried. // // Without it a fetch that failed once would refuse every later attempt at the // same version, and the only way through would be restarting the mirror. func (m *Manager) Forget(model, version string) { m.mu.Lock() defer m.mu.Unlock() delete(m.running, model+"/"+version) } // RemoveVersion deletes every file under one model version and reports what // went. // // The version is the unit, not the file: a model split across several files is // useless with one of them missing, so removing part of it leaves a directory // that looks present and cannot be loaded. func (m *Manager) RemoveVersion(model, version string) ([]string, error) { entries, err := List(m.root) if err != nil { return nil, err } var removed []string for _, e := range entries { if e.Model != model || e.Version != version { continue } if err := Remove(m.root, e); err != nil { return removed, err } removed = append(removed, e.File) } if len(removed) == 0 { return nil, fmt.Errorf("%s/%s: %w", model, version, os.ErrNotExist) } m.Forget(model, version) m.log.Info("mirror removed a version", "model", model, "version", version, "files", len(removed)) return removed, nil }