llmgw/internal/agent/gpucheck.go

package agent

import (
	"context"
	"fmt"
	"os"
	"slices"
	"strconv"
	"strings"
	"time"

	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)

// gpuProcesses and processGroup are variables so tests can stand in for
// nvidia-smi and /proc.
var (
	gpuProcesses = discovery.GPUProcesses
	gpuMemory    = discovery.GPUMemoryByProcess
	processGroup = pgidOf
	// gpuCheckWait is how long a shard has, once it answers, to show up on
	// its GPUs: the weights are on them by then, but nvidia-smi can lag.
	gpuCheckWait = 15 * time.Second
)

// onItsGPUs checks that a shard placed on GPUs computes on them.
//
// For an engine that falls back to the CPU when its GPU backend does not load,
// "it answers" is not "it is ready": llama.cpp built for CUDA but missing
// cuBLAS comes up on the CPU, answers at a tenth of the speed, and the GPU it
// was given sits idle and reserved. What it computes on is what nvidia-smi
// lists it on, so a shard none of whose processes hold memory on one of its
// GPUs is refused, with what the engine itself can see.
func (a *Agent) onItsGPUs(ctx context.Context, eng engines.Engine, spec engines.ShardSpec, pid int) error {
	fb, ok := eng.(engines.CPUFallback)
	if !ok || !fb.FallsBackToCPU() || len(spec.GPUIndices) == 0 || pid <= 0 {
		return nil
	}
	group := processGroup(pid)
	deadline := time.Now().Add(gpuCheckWait)
	var missing []int
	for {
		procs, err := gpuProcesses(ctx, a.cfg.Discovery.NvidiaSMI)
		if err != nil || procs == nil {
			// No nvidia-smi to ask is no evidence either way, and a node that
			// was given GPUs has one; say so and let the shard serve.
			a.log.Warn("could not check that a GPU shard runs on its GPUs", "shard", spec.ShardID, "err", err)
			return nil
		}
		missing = missing[:0]
		for _, i := range spec.GPUIndices {
			if !holds(procs[i], pid, group) {
				missing = append(missing, i)
			}
		}
		if len(missing) == 0 {
			return nil
		}
		if time.Now().After(deadline) {
			break
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(time.Second):
		}
	}
	seen := "it lists no GPU at all"
	if devs, err := fb.Devices(ctx); err != nil {
		seen = "listing its devices failed: " + err.Error()
	} else if len(devs) > 0 {
		seen = "it lists " + strings.Join(devs, "; ")
	}
	return fmt.Errorf("placed on GPU %s but computing on the CPU: the engine holds no memory there. "+
		"Its GPU backend did not load (%s). Check the build with `ldd` on its libggml-cuda.so, "+
		"or run `llmgw doctor` on this node", joinIndices(missing), seen)
}

// foreignGPUMemory is what each GPU holds for anything other than this
// agent's shards, by index, and each GPU's memory in use. Nil when nvidia-smi
// cannot be asked, which is no reading rather than an empty card.
//
// Everything allocated less what the shards' processes hold, rather than the
// sum of the other processes: a process in another container's PID namespace
// holds memory nvidia-smi lists under no PID this node can see, and it is
// still not memory placement can hand out.
func (a *Agent) foreignGPUMemory(ctx context.Context, shards []int) (foreign, used []int32) {
	mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
	if err != nil || mem == nil {
		return nil, nil
	}
	groups := make([]int, len(shards))
	for i, pid := range shards {
		groups[i] = processGroup(pid)
	}
	ours := func(p int) bool {
		for i, pid := range shards {
			if holds([]int{p}, pid, groups[i]) {
				return true
			}
		}
		return false
	}
	for _, g := range mem {
		f := g.UsedMB
		for _, p := range g.Procs {
			if ours(p.PID) {
				f -= p.UsedMB
			}
		}
		foreign = append(foreign, int32(max(f, 0)))
		used = append(used, int32(g.UsedMB))
	}
	return foreign, used
}

// holds reports whether any of pids is the shard's process or in its group:
// an engine that computes in a child process is still the shard.
func holds(pids []int, pid, group int) bool {
	for _, p := range pids {
		if p == pid || (group > 0 && processGroup(p) == group) {
			return true
		}
	}
	return false
}

// pgidOf reads a process's group from /proc; zero when it cannot.
func pgidOf(pid int) int {
	b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
	if err != nil {
		return 0
	}
	// The command name, in parentheses, may hold spaces; the fields after its
	// closing parenthesis are state, ppid, pgrp.
	s := string(b)
	i := strings.LastIndexByte(s, ')')
	if i < 0 {
		return 0
	}
	f := strings.Fields(s[i+1:])
	if len(f) < 3 {
		return 0
	}
	n, _ := strconv.Atoi(f[2])
	return n
}

func joinIndices(is []int) string {
	s := make([]string, len(is))
	for k, i := range is {
		s[k] = strconv.Itoa(i)
	}
	return strings.Join(s, ", ")
}

// engineCompute is where an engine said it put its model, from its output
// so far — llama.cpp's, which says — and false for one that said nothing. A
// shard given GPUs that its engine says it computes on the CPU is an error,
// found this way even where nvidia-smi cannot be asked.
func engineCompute(eng engines.Engine, gpus []int, lines []string) (engines.Compute, bool, error) {
	if _, ok := eng.(engines.CPUFallback); !ok {
		return engines.Compute{}, false, nil
	}
	read, ok := engines.ReadLlamaCompute(lines)
	if !ok {
		return engines.Compute{}, false, nil
	}
	if len(gpus) > 0 && read.OnTheCPU() {
		why := "its GPU backend saw no device it could use"
		if read.Total > 0 && !read.NoGPU {
			why = fmt.Sprintf("it offloaded %d of %d layers", read.Offloaded, read.Total)
		}
		return read, true, fmt.Errorf("placed on GPU %v but computing on the CPU: %s, by llama.cpp's own account "+
			"(llmgw ctl engine-check on this node says why)", gpus, why)
	}
	return read, true, nil
}

// computeToWire is a compute as it travels to the control plane.
func computeToWire(c *engines.Compute) *llmgwv1.Compute {
	if c == nil {
		return nil
	}
	return &llmgwv1.Compute{Backend: c.Backend, Devices: c.Devices,
		LayersOffloaded: int32(c.Offloaded), LayersTotal: int32(c.Total)} //nolint:gosec // layer counts
}

// gpuCompute is where a shard computes as nvidia-smi sees it: the CPU for a
// shard given no GPU, else the GPUs whose memory its processes hold, and how
// much. False when nvidia-smi cannot be asked. Engine-agnostic, and needing
// no log line: llama.cpp says where its layers went only at -lv 4.
func (a *Agent) gpuCompute(ctx context.Context, gpus []int, pid int) (engines.Compute, bool) {
	if len(gpus) == 0 {
		return engines.Compute{Backend: "CPU"}, true
	}
	mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
	if err != nil || mem == nil || pid <= 0 {
		return engines.Compute{}, false
	}
	group := processGroup(pid)
	c := engines.Compute{Backend: "CUDA"}
	for _, g := range mem {
		if !slices.Contains(gpus, g.Index) {
			continue
		}
		used := 0
		for _, p := range g.Procs {
			if holds([]int{p.PID}, pid, group) {
				used += p.UsedMB
			}
		}
		if used > 0 {
			c.Devices = append(c.Devices, fmt.Sprintf("GPU %d: %.1f GB", g.Index, float64(used)/1024))
		}
	}
	if len(c.Devices) == 0 {
		// Placed on GPUs and holding none of their memory.
		c.Backend = "CPU"
	}
	return c, true
}