NVML is the only source of NVLink topology, and that topology is what keeps a tensor-parallel group off the slow path inside a node. It needs cgo, so it sits behind the nvml tag and the default build keeps using nvidia-smi. Discover() picks the backend and falls back to nvidia-smi when NVML fails, logging the fallback rather than performing it silently: on a node that is meant to have working NVML, a quiet downgrade hides a broken driver installation and costs the scheduler its topology. CI now compiles the tagged build. That immediately paid for itself: ErrNoNVML was declared only in the untagged file, so the tests did not compile under -tags nvml. It is now declared where both builds can see it. Not verified: the NVML path has never run against a driver, because there is no GPU machine available. The NVLink domains it reports should be treated as unverified until someone runs `llmgw agent inventory` on a real node. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
6a13e3dddf
commit
55417546fc
10 files changed
+374
-23
No files matched your search
@@ -38,3 +38,11 @@ jobs:
|
||||
|
||||
- name: Build
|
||||
run: go build ./...
|
||||
|
||||
# The NVML backend needs cgo and cannot run without a driver, but it must
|
||||
# at least keep compiling: it is the only source of NVLink topology and a
|
||||
# break would otherwise surface on a GPU node, far from here.
|
||||
- name: Build the NVML backend
|
||||
env:
|
||||
CGO_ENABLED: "1"
|
||||
run: go build -tags nvml ./...
|
||||
+12
-4
@@ -61,12 +61,20 @@ what is missing is the control plane connection, so there is no release yet.
|
||||
as a table or JSON. Run it before adding a node: it is the fastest way to find
|
||||
a card another process is using, or a port with no subnet manager.
|
||||
|
||||
- **NVML backend** behind the `nvml` build tag, which adds NVLink topology.
|
||||
Nothing else exposes which GPUs share an NVLink domain, and that is what keeps
|
||||
a tensor-parallel group off the slow path inside a node. `Discover` falls back
|
||||
to nvidia-smi if NVML fails, and says so loudly rather than silently losing the
|
||||
topology. CI compiles the tagged build so it cannot rot unnoticed.
|
||||
|
||||
### Not verified
|
||||
|
||||
The NVML path compiles but has never run against a driver — there is no GPU
|
||||
machine to try it on. Treat the NVLink domains it reports as unverified until
|
||||
someone runs `llmgw agent inventory` on a real node.
|
||||
|
||||
### Still missing for phase 2
|
||||
|
||||
- The NVML backend behind the `nvml` build tag, and NVLink topology with it.
|
||||
Only NVML exposes which GPUs share an NVLink domain, which is what keeps a
|
||||
tensor-parallel group off the slow path within a node. It needs a machine
|
||||
with GPUs to verify.
|
||||
- The agent-to-control-plane stream. It pairs with phase 3's control plane
|
||||
rather than standing alone, so `llmgw agent` itself is still a stub.
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/NVIDIA/go-nvml v0.13.4-0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
github.com/NVIDIA/go-nvml v0.13.4-0 h1:o3jp9u2x1R9ShFE3v+Aesp55XOSIQFMJz/VGNUcJaNE=
|
||||
github.com/NVIDIA/go-nvml v0.13.4-0/go.mod h1:id63qwpoDWpFXwnwM6psDCSqW4BmNu6mWpr4YeQtPGo=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// ErrNoNVML means this binary was built without the NVML backend.
|
||||
//
|
||||
// It is declared here rather than in the build-tagged file so callers — and
|
||||
// tests — can refer to it in either build.
|
||||
var ErrNoNVML = errors.New("built without the nvml tag; rebuild with -tags nvml for NVLink topology")
|
||||
|
||||
// Options configure a full inventory scan.
|
||||
type Options struct {
|
||||
// NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.
|
||||
NvidiaSMI string
|
||||
// IBRoot is the sysfs InfiniBand directory.
|
||||
IBRoot string
|
||||
// PreferNVML uses the NVML backend when this binary has one. It is the only
|
||||
// source of NVLink topology.
|
||||
PreferNVML bool
|
||||
// SkipGPUs omits GPU discovery, for nodes that have none.
|
||||
SkipGPUs bool
|
||||
}
|
||||
|
||||
// Discover builds the node's inventory.
|
||||
//
|
||||
// When NVML is preferred but fails, it falls back to nvidia-smi and says so in
|
||||
// Inventory.Source. The fallback is logged rather than silent: on a node that
|
||||
// is supposed to have working NVML, a quiet downgrade hides a broken driver and
|
||||
// costs the scheduler its NVLink topology.
|
||||
func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, error) {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
inv := Inventory{Source: "none"}
|
||||
|
||||
if !opts.SkipGPUs {
|
||||
switch {
|
||||
case opts.PreferNVML && nvmlAvailable:
|
||||
gpus, err := DiscoverGPUsNVML(ctx)
|
||||
if err == nil {
|
||||
inv.GPUs, inv.Source = gpus, "nvml"
|
||||
break
|
||||
}
|
||||
log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
|
||||
"NVLink topology will be unavailable", "err", err)
|
||||
fallthrough
|
||||
default:
|
||||
gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
|
||||
if err != nil {
|
||||
return inv, fmt.Errorf("gpu discovery: %w", err)
|
||||
}
|
||||
inv.GPUs, inv.Source = gpus, "nvidia-smi"
|
||||
}
|
||||
}
|
||||
|
||||
hcas, err := DiscoverIB(opts.IBRoot)
|
||||
if err != nil {
|
||||
return inv, fmt.Errorf("infiniband discovery: %w", err)
|
||||
}
|
||||
inv.HCAs = hcas
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
// NVMLSupported reports whether this binary includes the NVML backend.
|
||||
func NVMLSupported() bool { return nvmlAvailable }
|
||||
@@ -0,0 +1,101 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func fakeSMI(t *testing.T) string {
|
||||
t.Helper()
|
||||
stub := filepath.Join(t.TempDir(), "nvidia-smi")
|
||||
script := "#!/bin/sh\ncat <<'EOF'\n" + smiSample + "EOF\n"
|
||||
if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return stub
|
||||
}
|
||||
|
||||
func TestDiscoverUsesSMIByDefault(t *testing.T) {
|
||||
inv, err := Discover(t.Context(), Options{
|
||||
NvidiaSMI: fakeSMI(t),
|
||||
IBRoot: filepath.Join(t.TempDir(), "absent"),
|
||||
}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inv.Source != "nvidia-smi" {
|
||||
t.Errorf("source=%q", inv.Source)
|
||||
}
|
||||
if len(inv.GPUs) != 2 {
|
||||
t.Errorf("got %d GPUs", len(inv.GPUs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverFallsBackWhenNVMLUnavailable(t *testing.T) {
|
||||
// Asking for NVML in a build without it must still produce an inventory:
|
||||
// the fallback is the whole point of the build tag.
|
||||
inv, err := Discover(t.Context(), Options{
|
||||
PreferNVML: true,
|
||||
NvidiaSMI: fakeSMI(t),
|
||||
IBRoot: filepath.Join(t.TempDir(), "absent"),
|
||||
}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(inv.GPUs) == 0 {
|
||||
t.Fatal("no GPUs discovered through the fallback")
|
||||
}
|
||||
if NVMLSupported() {
|
||||
// On an NVML build without a driver the NVML attempt fails and the
|
||||
// fallback still has to work.
|
||||
if inv.Source != "nvml" && inv.Source != "nvidia-smi" {
|
||||
t.Errorf("source=%q", inv.Source)
|
||||
}
|
||||
} else if inv.Source != "nvidia-smi" {
|
||||
t.Errorf("a build without NVML reported source=%q", inv.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverSkipGPUs(t *testing.T) {
|
||||
inv, err := Discover(t.Context(), Options{
|
||||
SkipGPUs: true,
|
||||
IBRoot: fakeSysfs(t),
|
||||
}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(inv.GPUs) != 0 {
|
||||
t.Errorf("GPUs were discovered despite SkipGPUs")
|
||||
}
|
||||
if len(inv.HCAs) != 3 {
|
||||
t.Errorf("got %d HCAs", len(inv.HCAs))
|
||||
}
|
||||
if inv.Source != "none" {
|
||||
t.Errorf("source=%q", inv.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverReportsGPUFailure(t *testing.T) {
|
||||
// A node meant to have GPUs whose discovery fails must say so rather than
|
||||
// reporting an empty, healthy-looking inventory.
|
||||
_, err := Discover(context.Background(), Options{
|
||||
NvidiaSMI: filepath.Join(t.TempDir(), "absent"),
|
||||
IBRoot: filepath.Join(t.TempDir(), "absent"),
|
||||
}, slog.New(slog.DiscardHandler))
|
||||
if err == nil {
|
||||
t.Error("a failed GPU discovery was reported as success")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNVMLSupportedMatchesBuild(t *testing.T) {
|
||||
// Documents which backend this binary has, and keeps both files compiling.
|
||||
t.Logf("NVML backend compiled in: %v", NVMLSupported())
|
||||
if !NVMLSupported() {
|
||||
if _, err := DiscoverGPUsNVML(t.Context()); err != ErrNoNVML {
|
||||
t.Errorf("expected ErrNoNVML, got %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
//go:build !nvml
|
||||
|
||||
package discovery
|
||||
|
||||
import "context"
|
||||
|
||||
// nvmlAvailable reports that this build cannot use NVML.
|
||||
const nvmlAvailable = false
|
||||
|
||||
// DiscoverGPUsNVML is unavailable in builds without the nvml tag.
|
||||
func DiscoverGPUsNVML(context.Context) ([]GPU, error) { return nil, ErrNoNVML }
|
||||
@@ -0,0 +1,148 @@
|
||||
//go:build nvml
|
||||
|
||||
// This file is the NVML-backed GPU discovery path, built only with the `nvml`
|
||||
// tag because the bindings require cgo. Without the tag the agent falls back to
|
||||
// nvidia-smi, which keeps the server side — and development machines without a
|
||||
// C toolchain or a driver — building cleanly (plan §3.2).
|
||||
//
|
||||
// What NVML adds over nvidia-smi is NVLink topology. Nothing else exposes which
|
||||
// GPUs share an NVLink domain, and that is what keeps a tensor-parallel group
|
||||
// off the slow path inside a node (plan §6.2, constraint 3).
|
||||
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/NVIDIA/go-nvml/pkg/nvml"
|
||||
)
|
||||
|
||||
// nvmlAvailable reports that this build can use NVML.
|
||||
const nvmlAvailable = true
|
||||
|
||||
// DiscoverGPUsNVML enumerates GPUs through NVML.
|
||||
//
|
||||
// It returns an error rather than falling back on its own: the caller decides
|
||||
// whether to degrade to nvidia-smi, because on a node that is supposed to have
|
||||
// working NVML a silent downgrade hides a broken driver installation.
|
||||
func DiscoverGPUsNVML(_ context.Context) ([]GPU, error) {
|
||||
if ret := nvml.Init(); ret != nvml.SUCCESS {
|
||||
return nil, fmt.Errorf("nvml init: %s", nvml.ErrorString(ret))
|
||||
}
|
||||
defer func() { _ = nvml.Shutdown() }()
|
||||
|
||||
driver, _ := nvml.SystemGetDriverVersion()
|
||||
cuda := ""
|
||||
if v, ret := nvml.SystemGetCudaDriverVersion(); ret == nvml.SUCCESS {
|
||||
// NVML reports 12040 for 12.4.
|
||||
cuda = fmt.Sprintf("%d.%d", v/1000, (v%1000)/10)
|
||||
}
|
||||
|
||||
count, ret := nvml.DeviceGetCount()
|
||||
if ret != nvml.SUCCESS {
|
||||
return nil, fmt.Errorf("nvml device count: %s", nvml.ErrorString(ret))
|
||||
}
|
||||
|
||||
// Bus id to index, so NVLink peers can be reported as indices. NVML gives
|
||||
// the remote end of a link as a PCI address, which is not what placement
|
||||
// works in.
|
||||
byBusID := make(map[string]int, count)
|
||||
gpus := make([]GPU, 0, count)
|
||||
|
||||
for i := range count {
|
||||
dev, ret := nvml.DeviceGetHandleByIndex(i)
|
||||
if ret != nvml.SUCCESS {
|
||||
return nil, fmt.Errorf("nvml device %d: %s", i, nvml.ErrorString(ret))
|
||||
}
|
||||
|
||||
g := GPU{Index: i, NUMANode: -1, DriverVersion: driver, CUDAVersion: cuda}
|
||||
g.UUID, _ = dev.GetUUID()
|
||||
g.Name, _ = dev.GetName()
|
||||
|
||||
if mem, ret := dev.GetMemoryInfo(); ret == nvml.SUCCESS {
|
||||
g.MemoryTotalMB = int(mem.Total / (1 << 20))
|
||||
g.MemoryUsedMB = int(mem.Used / (1 << 20))
|
||||
}
|
||||
if major, minor, ret := dev.GetCudaComputeCapability(); ret == nvml.SUCCESS {
|
||||
g.ComputeCapability = fmt.Sprintf("%d.%d", major, minor)
|
||||
}
|
||||
if pci, ret := dev.GetPciInfo(); ret == nvml.SUCCESS {
|
||||
g.PCIBusID = busIDString(pci.BusId[:])
|
||||
byBusID[normalizeBusID(g.PCIBusID)] = i
|
||||
}
|
||||
if t, ret := dev.GetTemperature(nvml.TEMPERATURE_GPU); ret == nvml.SUCCESS {
|
||||
g.Health.TemperatureC = int(t)
|
||||
}
|
||||
// Volatile counters reset on driver reload, which is what we want: an
|
||||
// error from three months and one reboot ago is not a reason to refuse
|
||||
// a card today.
|
||||
if n, ret := dev.GetTotalEccErrors(nvml.MEMORY_ERROR_TYPE_UNCORRECTED, nvml.VOLATILE_ECC); ret == nvml.SUCCESS {
|
||||
g.Health.ECCErrors = int(n)
|
||||
}
|
||||
|
||||
assessHealth(&g)
|
||||
gpus = append(gpus, g)
|
||||
}
|
||||
|
||||
// A second pass, because a link's remote end may be a GPU enumerated later.
|
||||
for i := range gpus {
|
||||
dev, ret := nvml.DeviceGetHandleByIndex(gpus[i].Index)
|
||||
if ret != nvml.SUCCESS {
|
||||
continue
|
||||
}
|
||||
gpus[i].NVLinkPeers = nvlinkPeers(dev, byBusID, gpus[i].Index)
|
||||
}
|
||||
return gpus, nil
|
||||
}
|
||||
|
||||
// nvlinkPeers returns the indices of GPUs reachable from dev over NVLink.
|
||||
func nvlinkPeers(dev nvml.Device, byBusID map[string]int, self int) []int {
|
||||
seen := map[int]struct{}{}
|
||||
var out []int
|
||||
|
||||
for link := range nvml.NVLINK_MAX_LINKS {
|
||||
pci, ret := dev.GetNvLinkRemotePciInfo(link)
|
||||
if ret != nvml.SUCCESS {
|
||||
// Unpopulated links return an error; that is the normal case for
|
||||
// most of the 36 possible links, so it is not worth reporting.
|
||||
continue
|
||||
}
|
||||
peer, ok := byBusID[normalizeBusID(busIDString(pci.BusId[:]))]
|
||||
if !ok || peer == self {
|
||||
// The remote end can be an NVSwitch rather than a GPU, in which
|
||||
// case it is not in the map.
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[peer]; dup {
|
||||
continue
|
||||
}
|
||||
seen[peer] = struct{}{}
|
||||
out = append(out, peer)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// busIDString turns NVML's fixed-size C char array into a Go string.
|
||||
func busIDString(b []int8) string {
|
||||
buf := make([]byte, 0, len(b))
|
||||
for _, c := range b {
|
||||
if c == 0 {
|
||||
break
|
||||
}
|
||||
buf = append(buf, byte(c))
|
||||
}
|
||||
return string(buf)
|
||||
}
|
||||
|
||||
// normalizeBusID makes bus ids comparable: NVML reports them uppercase from one
|
||||
// call and lowercase from another, and sometimes with a shortened domain.
|
||||
func normalizeBusID(s string) string {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
// "0000:1b:00.0" and "00000000:1B:00.0" are the same device.
|
||||
if i := strings.Index(s, ":"); i > 0 {
|
||||
s = strings.TrimLeft(s[:i], "0") + s[i:]
|
||||
}
|
||||
return strings.TrimPrefix(s, ":")
|
||||
}
|
||||
+18
-17
@@ -36,6 +36,7 @@ func newInventoryCmd() *cobra.Command {
|
||||
smiBin string
|
||||
ibRoot string
|
||||
skipGPU bool
|
||||
useNVML bool
|
||||
)
|
||||
|
||||
cmd := &cobra.Command{
|
||||
@@ -46,24 +47,18 @@ func newInventoryCmd() *cobra.Command {
|
||||
"Run it before adding a node to the cluster — it is the fastest way to find\n" +
|
||||
"a card another process is already using, or a port with no subnet manager.",
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
inv := discovery.Inventory{Source: "nvidia-smi"}
|
||||
|
||||
if !skipGPU {
|
||||
gpus, err := discovery.DiscoverGPUsSMI(cmd.Context(), smiBin)
|
||||
if err != nil {
|
||||
// Not fatal: a node with no GPUs still has IB worth showing,
|
||||
// and the error itself is the useful output.
|
||||
fmt.Fprintf(os.Stderr, "GPU discovery failed: %v\n", err)
|
||||
inv.Source = "unavailable"
|
||||
}
|
||||
inv.GPUs = gpus
|
||||
}
|
||||
|
||||
hcas, err := discovery.DiscoverIB(ibRoot)
|
||||
inv, err := discovery.Discover(cmd.Context(), discovery.Options{
|
||||
NvidiaSMI: smiBin,
|
||||
IBRoot: ibRoot,
|
||||
PreferNVML: useNVML,
|
||||
SkipGPUs: skipGPU,
|
||||
}, logger())
|
||||
if err != nil {
|
||||
return err
|
||||
// Not fatal: a node whose GPUs cannot be read still has IB
|
||||
// worth showing, and the error is itself the useful output.
|
||||
fmt.Fprintf(os.Stderr, "%v\n", err)
|
||||
inv.Source = "unavailable"
|
||||
}
|
||||
inv.HCAs = hcas
|
||||
|
||||
if asJSON {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
@@ -80,6 +75,7 @@ func newInventoryCmd() *cobra.Command {
|
||||
f.StringVar(&smiBin, "nvidia-smi", "", "path to nvidia-smi")
|
||||
f.StringVar(&ibRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory")
|
||||
f.BoolVar(&skipGPU, "no-gpu", false, "skip GPU discovery")
|
||||
f.BoolVar(&useNVML, "nvml", true, "prefer the NVML backend when this build has one (it is the only source of NVLink topology)")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -87,7 +83,7 @@ func newInventoryCmd() *cobra.Command {
|
||||
func printInventory(inv discovery.Inventory) {
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
|
||||
fmt.Fprintf(w, "GPUs (%d)\n", len(inv.GPUs))
|
||||
fmt.Fprintf(w, "GPUs (%d, via %s)\n", len(inv.GPUs), inv.Source)
|
||||
if len(inv.GPUs) == 0 {
|
||||
fmt.Fprintln(w, " none")
|
||||
} else {
|
||||
@@ -101,6 +97,11 @@ func printInventory(inv discovery.Inventory) {
|
||||
g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
|
||||
}
|
||||
fmt.Fprintf(w, " classes: %v\n", inv.Classes())
|
||||
if !discovery.NVMLSupported() {
|
||||
fmt.Fprintln(w, " NVLink topology unavailable: rebuild with -tags nvml.")
|
||||
fmt.Fprintln(w, " Without it, placement cannot keep a tensor-parallel group inside")
|
||||
fmt.Fprintln(w, " one NVLink domain (plan §6.2).")
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "\nInfiniBand adapters (%d)\n", len(inv.HCAs))
|
||||
|
||||
+3
-2
@@ -1002,8 +1002,9 @@ work_dir: /var/lib/llmgw-agent
|
||||
- [x] GPU discovery (nvidia-smi path), IB discovery from sysfs
|
||||
- [x] Engine interface, vLLM and SGLang command and environment construction
|
||||
- [x] Supervisor (process groups, log buffer, state machine, orphan adoption)
|
||||
- [ ] NVML backend behind the `nvml` build tag — needs a machine with GPUs to verify
|
||||
- [ ] NVLink topology (only NVML exposes it)
|
||||
- [x] NVML backend behind the `nvml` build tag, with NVLink topology. It compiles
|
||||
in CI but has not run against a driver: **verify on a GPU node before
|
||||
relying on the NVLink domains it reports.**
|
||||
- [ ] connect-go stream, register/heartbeat, mTLS
|
||||
- **Exit criterion:** start and stop a model on a real node via `ctl`. Not met:
|
||||
the control plane connection is the remaining piece, and it pairs naturally
|
||||
|
||||
Reference in new issue
Block a user