diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index d26030c..6316490 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -2637,7 +2637,16 @@ type EngineWant struct { Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` // "b10991" // variant is the build to install, when one is chosen for the node rather // than left to its hardware: "cu13", "cu129". Empty is the agent's choice. - Variant string `protobuf:"bytes,3,opt,name=variant,proto3" json:"variant,omitempty"` + Variant string `protobuf:"bytes,3,opt,name=variant,proto3" json:"variant,omitempty"` + // offline_base, on a cluster with no way to the internet, is where this + // engine version's files are on its mirror: + // http://mirror:8081/_llmgw/engines/llamacpp/b10991. Set, the node never + // asks the engine's publisher. + OfflineBase string `protobuf:"bytes,4,opt,name=offline_base,json=offlineBase,proto3" json:"offline_base,omitempty"` + // offline_sha256 is each of those files' checksum by name, from the + // bundle the control plane trusts — not from the mirror, which serves + // every node over plain HTTP. + OfflineSha256 map[string]string `protobuf:"bytes,5,rep,name=offline_sha256,json=offlineSha256,proto3" json:"offline_sha256,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2693,6 +2702,20 @@ func (x *EngineWant) GetVariant() string { return "" } +func (x *EngineWant) GetOfflineBase() string { + if x != nil { + return x.OfflineBase + } + return "" +} + +func (x *EngineWant) GetOfflineSha256() map[string]string { + if x != nil { + return x.OfflineSha256 + } + return nil +} + // ProvisionStatus is where one engine's installation stands on a node. type ProvisionStatus struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -4369,12 +4392,17 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" + "\tProvision\x12.\n" + - "\aengines\x18\x01 \x03(\v2\x14.llmgw.v1.EngineWantR\aengines\"X\n" + + "\aengines\x18\x01 \x03(\v2\x14.llmgw.v1.EngineWantR\aengines\"\x8d\x02\n" + "\n" + "EngineWant\x12\x16\n" + "\x06engine\x18\x01 \x01(\tR\x06engine\x12\x18\n" + "\aversion\x18\x02 \x01(\tR\aversion\x12\x18\n" + - "\avariant\x18\x03 \x01(\tR\avariant\"\x8b\x01\n" + + "\avariant\x18\x03 \x01(\tR\avariant\x12!\n" + + "\foffline_base\x18\x04 \x01(\tR\vofflineBase\x12N\n" + + "\x0eoffline_sha256\x18\x05 \x03(\v2'.llmgw.v1.EngineWant.OfflineSha256EntryR\rofflineSha256\x1a@\n" + + "\x12OfflineSha256Entry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8b\x01\n" + "\x0fProvisionStatus\x12\x16\n" + "\x06engine\x18\x01 \x01(\tR\x06engine\x12\x18\n" + "\aversion\x18\x02 \x01(\tR\aversion\x12\x18\n" + @@ -4532,7 +4560,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 44) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 45) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -4578,7 +4606,8 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ nil, // 41: llmgw.v1.GPU.NicPathsEntry nil, // 42: llmgw.v1.IBPort.CountersEntry nil, // 43: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 44: llmgw.v1.EngineProfile.EnvEntry + nil, // 44: llmgw.v1.EngineWant.OfflineSha256Entry + nil, // 45: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 4, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register @@ -4624,18 +4653,19 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 17, // 40: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState 16, // 41: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX 20, // 42: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 43: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 23, // 44: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 25, // 45: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 44, // 46: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 38, // 47: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter - 1, // 48: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 49: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 49, // [49:50] is the sub-list for method output_type - 48, // [48:49] is the sub-list for method input_type - 48, // [48:48] is the sub-list for extension type_name - 48, // [48:48] is the sub-list for extension extendee - 0, // [0:48] is the sub-list for field type_name + 44, // 43: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry + 0, // 44: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 23, // 45: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 25, // 46: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 45, // 47: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 38, // 48: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter + 1, // 49: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 50: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 50, // [50:51] is the sub-list for method output_type + 49, // [49:50] is the sub-list for method input_type + 49, // [49:49] is the sub-list for extension type_name + 49, // [49:49] is the sub-list for extension extendee + 0, // [0:49] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -4671,7 +4701,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 44, + NumMessages: 45, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 98caf13..28b4789 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -515,6 +515,15 @@ message EngineWant { // variant is the build to install, when one is chosen for the node rather // than left to its hardware: "cu13", "cu129". Empty is the agent's choice. string variant = 3; + // offline_base, on a cluster with no way to the internet, is where this + // engine version's files are on its mirror: + // http://mirror:8081/_llmgw/engines/llamacpp/b10991. Set, the node never + // asks the engine's publisher. + string offline_base = 4; + // offline_sha256 is each of those files' checksum by name, from the + // bundle the control plane trusts — not from the mirror, which serves + // every node over plain HTTP. + map offline_sha256 = 5; } // ProvisionStatus is where one engine's installation stands on a node. diff --git a/deploy/container/Dockerfile b/deploy/container/Dockerfile new file mode 100644 index 0000000..3f217c7 --- /dev/null +++ b/deploy/container/Dockerfile @@ -0,0 +1,40 @@ +# The llmgw control plane as a container image, for deploy/helm/llmgw. +# +# The project publishes binaries, not an image, so this is built where the +# image is to live: +# +# docker build -f deploy/container/Dockerfile --build-arg VERSION=v0.11.1 \ +# -t registry.internal/llmgw:v0.11.1 . +# docker push registry.internal/llmgw:v0.11.1 +# +# The binary is static (CGO_ENABLED=0, as every release is), so the image is +# that one file on a base with nothing else in it: no shell, no package +# manager. `kubectl exec ... -- llmgw ctl status` still works, because it +# runs the binary rather than a shell. +# +# Only the control plane runs from it. An agent installs engines on its +# machine, manages its GPUs and supervises processes there as root, which is +# what a container is built to prevent: docs/kubernetes.md. + +FROM golang:1.26 AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +ARG VERSION=dev +RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags "-s -w -X gitea.mynodes.xyz/ilker/llmgw/internal/version.Version=${VERSION}" \ + -o /out/llmgw ./cmd/llmgw && \ + mkdir -p /out/state + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/llmgw /usr/local/bin/llmgw +# The chart mounts server.yaml, the certificates and the token under +# /etc/llmgw, and the SQLite store, when it is one, under /var/lib/llmgw — +# owned by the user the server runs as, so a volume Docker fills from the +# image is writable without Kubernetes' fsGroup. +COPY --from=build --chown=65532:65532 /out/state /var/lib/llmgw +USER 65532:65532 +EXPOSE 8080 7443 9091 +ENTRYPOINT ["/usr/local/bin/llmgw"] +CMD ["server", "--config", "/etc/llmgw/server.yaml"] diff --git a/deploy/helm/llmgw/Chart.yaml b/deploy/helm/llmgw/Chart.yaml new file mode 100644 index 0000000..e2d899a --- /dev/null +++ b/deploy/helm/llmgw/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: llmgw +description: >- + The llmgw control plane — the OpenAI-compatible gateway, the router and the + scheduler that places models on GPU and CPU nodes. The nodes themselves run + outside Kubernetes; see docs/kubernetes.md. +type: application +# The chart's own version; appVersion is the llmgw release the image holds. +version: 0.1.0 +appVersion: "v0.11.1" +home: https://gitea.mynodes.xyz/ilker/llmgw +sources: + - https://gitea.mynodes.xyz/ilker/llmgw +keywords: + - llm + - gateway + - inference diff --git a/deploy/helm/llmgw/templates/NOTES.txt b/deploy/helm/llmgw/templates/NOTES.txt new file mode 100644 index 0000000..52912ca --- /dev/null +++ b/deploy/helm/llmgw/templates/NOTES.txt @@ -0,0 +1,27 @@ +The llmgw control plane is starting as {{ include "llmgw.fullname" . }}. + +The admin API listens on the pod's loopback only. To reach it from here: + + kubectl -n {{ .Release.Namespace }} port-forward statefulset/{{ include "llmgw.fullname" . }} 9090:9090 & + export LLMGW_ADMIN_TOKEN=$(kubectl -n {{ .Release.Namespace }} get secret {{ include "llmgw.adminSecret" . }} \ + -o jsonpath='{.data.{{ if .Values.adminToken.existingSecret }}{{ .Values.adminToken.secretKey }}{{ else }}admin-token{{ end }}}' | base64 -d) + llmgw ctl status + +The first administrator: + + kubectl -n {{ .Release.Namespace }} exec -it statefulset/{{ include "llmgw.fullname" . }} -- \ + llmgw users create you@example.com --password '' + kubectl -n {{ .Release.Namespace }} exec -it statefulset/{{ include "llmgw.fullname" . }} -- \ + llmgw users role you@example.com admin + +Agents connect to the {{ include "llmgw.fullname" . }}-agents Service on port {{ .Values.agentService.port }}: + + kubectl -n {{ .Release.Namespace }} get service {{ include "llmgw.fullname" . }}-agents + +Its address must be in the certificate in {{ .Values.agentTLS.existingSecret }}. Nodes join from +the machine holding llmgw's authority: + + sudo llmgw node add ubuntu@gpu01 --server https://:{{ .Values.agentService.port }} \ + --pki ./pki --admin 127.0.0.1:9090 + +docs/kubernetes.md has the rest, and why the nodes stay outside Kubernetes. diff --git a/deploy/helm/llmgw/templates/_helpers.tpl b/deploy/helm/llmgw/templates/_helpers.tpl new file mode 100644 index 0000000..388d248 --- /dev/null +++ b/deploy/helm/llmgw/templates/_helpers.tpl @@ -0,0 +1,74 @@ +{{/* The release's name for everything the chart makes. */}} +{{- define "llmgw.fullname" -}} +{{- if contains .Chart.Name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "llmgw.selectorLabels" -}} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "llmgw.labels" -}} +{{ include "llmgw.selectorLabels" . }} +app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +{{- end -}} + +{{- define "llmgw.image" -}} +{{- if not .Values.image.repository -}} +{{- fail "image.repository: the image built from deploy/container/Dockerfile, pushed where the cluster can pull it (the project publishes binaries, not an image)" -}} +{{- end -}} +{{- printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) -}} +{{- end -}} + +{{- define "llmgw.adminSecret" -}} +{{- .Values.adminToken.existingSecret | default (printf "%s-admin" (include "llmgw.fullname" .)) -}} +{{- end -}} + +{{/* +server.yaml: what the chart decides — listeners, where the certificates, +the token and the store are mounted — laid over .Values.config. The chart's +keys win, because each names a port or a path the pod is built around. +*/}} +{{- define "llmgw.serverConfig" -}} +{{- $store := "/var/lib/llmgw/llmgw.db" -}} +{{- if eq .Values.store.type "postgres" -}} +{{- $store = "@/etc/llmgw/store/dsn" -}} +{{- end -}} +{{- $base := dict + "listen" ":8080" + "admin_listen" "127.0.0.1:9090" + "metrics_listen" ":9091" + "agent_listen" ":7443" + "admin_token_file" "/etc/llmgw/admin/admin-token" + "agent_tls" (dict "ca" "/etc/llmgw/tls/ca.pem" "cert" "/etc/llmgw/tls/server.pem" "key" "/etc/llmgw/tls/server-key.pem") + "auth" (dict "store" $store) +-}} +{{- if .Values.offlineChecksums.configMap -}} +{{- $_ := set $base "offline" (dict "checksums" "/etc/llmgw/offline/offline.sha256") -}} +{{- end -}} +{{- $cfg := mustMergeOverwrite (deepCopy (.Values.config | default dict)) $base -}} +# Made by the llmgw Helm chart from its values. Every key: docs/reference.md +{{ toYaml $cfg }} +{{- end -}} + +{{/* Refuses values that would make a pod that cannot work. */}} +{{- define "llmgw.validate" -}} +{{- if not (has .Values.store.type (list "sqlite" "postgres")) -}} +{{- fail (printf "store.type %q: sqlite or postgres" .Values.store.type) -}} +{{- end -}} +{{- if ne (int .Values.replicas) 1 -}} +{{- fail "replicas: this chart runs one control plane; values.yaml says why" -}} +{{- end -}} +{{- if and (eq .Values.store.type "postgres") (not .Values.store.postgres.existingSecret) -}} +{{- fail "store.postgres.existingSecret: a Secret holding the PostgreSQL address" -}} +{{- end -}} +{{- if not .Values.agentTLS.existingSecret -}} +{{- fail "agentTLS.existingSecret: a Secret with ca.pem, server.pem and server-key.pem from llmgw's authority (see values.yaml)" -}} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/llmgw/templates/configmap.yaml b/deploy/helm/llmgw/templates/configmap.yaml new file mode 100644 index 0000000..f4d63dd --- /dev/null +++ b/deploy/helm/llmgw/templates/configmap.yaml @@ -0,0 +1,10 @@ +{{- include "llmgw.validate" . -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "llmgw.fullname" . }} + labels: + {{- include "llmgw.labels" . | nindent 4 }} +data: + server.yaml: | + {{- include "llmgw.serverConfig" . | nindent 4 }} diff --git a/deploy/helm/llmgw/templates/ingress.yaml b/deploy/helm/llmgw/templates/ingress.yaml new file mode 100644 index 0000000..9d0d09b --- /dev/null +++ b/deploy/helm/llmgw/templates/ingress.yaml @@ -0,0 +1,37 @@ +{{- if .Values.ingress.enabled }} +# The OpenAI-compatible API, the portal and the chat page on port 8080. Never +# the agent port: its mutual TLS cannot pass through an HTTP proxy. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "llmgw.fullname" . }} + labels: + {{- include "llmgw.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "llmgw.fullname" $ }} + port: + name: api + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/helm/llmgw/templates/secret-admin.yaml b/deploy/helm/llmgw/templates/secret-admin.yaml new file mode 100644 index 0000000..69f9f63 --- /dev/null +++ b/deploy/helm/llmgw/templates/secret-admin.yaml @@ -0,0 +1,19 @@ +{{- if not .Values.adminToken.existingSecret }} +{{- $name := include "llmgw.adminSecret" . }} +{{- $token := randAlphaNum 64 }} +{{- /* Kept across upgrades: a new token on every upgrade would lock out every console holding the old one. */}} +{{- with lookup "v1" "Secret" .Release.Namespace $name }} +{{- $token = index .data "admin-token" | b64dec }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + labels: + {{- include "llmgw.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + admin-token: {{ $token | b64enc | quote }} +{{- end }} diff --git a/deploy/helm/llmgw/templates/servicemonitor.yaml b/deploy/helm/llmgw/templates/servicemonitor.yaml new file mode 100644 index 0000000..a060466 --- /dev/null +++ b/deploy/helm/llmgw/templates/servicemonitor.yaml @@ -0,0 +1,21 @@ +{{- if .Values.serviceMonitor.enabled }} +# For the Prometheus Operator: the metrics port, which serves metrics and +# nothing else. deploy/monitoring holds the rules and dashboards that read them. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "llmgw.fullname" . }} + labels: + {{- include "llmgw.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "llmgw.selectorLabels" . | nindent 6 }} + endpoints: + - port: metrics + path: /metrics + interval: {{ .Values.serviceMonitor.interval }} +{{- end }} diff --git a/deploy/helm/llmgw/templates/services.yaml b/deploy/helm/llmgw/templates/services.yaml new file mode 100644 index 0000000..e599b58 --- /dev/null +++ b/deploy/helm/llmgw/templates/services.yaml @@ -0,0 +1,70 @@ +# The API and the metrics, for the cluster and an Ingress. +apiVersion: v1 +kind: Service +metadata: + name: {{ include "llmgw.fullname" . }} + labels: + {{- include "llmgw.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + selector: + {{- include "llmgw.selectorLabels" . | nindent 4 }} + ports: + - name: api + port: {{ .Values.service.port }} + targetPort: api + - name: metrics + port: {{ .Values.service.metricsPort }} + targetPort: metrics +--- +# Where agents on the nodes connect. Mutual TLS ends in the pod: whatever is +# in between must pass TCP through untouched. +apiVersion: v1 +kind: Service +metadata: + name: {{ include "llmgw.fullname" . }}-agents + labels: + {{- include "llmgw.labels" . | nindent 4 }} + {{- with .Values.agentService.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.agentService.type }} + {{- if and (eq .Values.agentService.type "LoadBalancer") .Values.agentService.loadBalancerIP }} + loadBalancerIP: {{ .Values.agentService.loadBalancerIP }} + {{- end }} + {{- with .Values.agentService.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if has .Values.agentService.type (list "LoadBalancer" "NodePort") }} + externalTrafficPolicy: {{ .Values.agentService.externalTrafficPolicy }} + {{- end }} + selector: + {{- include "llmgw.selectorLabels" . | nindent 4 }} + ports: + - name: agents + port: {{ .Values.agentService.port }} + targetPort: agents +--- +# The StatefulSet's own, giving each pod a stable name. +apiVersion: v1 +kind: Service +metadata: + name: {{ include "llmgw.fullname" . }}-headless + labels: + {{- include "llmgw.labels" . | nindent 4 }} +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + {{- include "llmgw.selectorLabels" . | nindent 4 }} + ports: + - name: api + port: 8080 + targetPort: api diff --git a/deploy/helm/llmgw/templates/statefulset.yaml b/deploy/helm/llmgw/templates/statefulset.yaml new file mode 100644 index 0000000..da11cd0 --- /dev/null +++ b/deploy/helm/llmgw/templates/statefulset.yaml @@ -0,0 +1,181 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "llmgw.fullname" . }} + labels: + {{- include "llmgw.labels" . | nindent 4 }} +spec: + # One control plane: Kubernetes replaces it within seconds when it goes, + # and the agents reconnect on their own. See replicas in values.yaml. + replicas: 1 + serviceName: {{ include "llmgw.fullname" . }}-headless + selector: + matchLabels: + {{- include "llmgw.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "llmgw.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + # server.yaml is read at start: a changed one restarts the pod. + checksum/config: {{ include "llmgw.serverConfig" . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + # Agents reconnect within seconds of a control plane going; a long + # grace period only delays the one that replaces it. + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: llmgw + image: {{ include "llmgw.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ["server", "--config", "/etc/llmgw/server.yaml"] + env: + # For `kubectl exec ... -- llmgw ctl`, which reads the token + # from here; the server reads admin_token_file. + - name: LLMGW_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "llmgw.adminSecret" . }} + key: {{ if .Values.adminToken.existingSecret }}{{ .Values.adminToken.secretKey }}{{ else }}admin-token{{ end }} + ports: + - name: api + containerPort: 8080 + - name: agents + containerPort: 7443 + - name: metrics + containerPort: 9091 + # Up is /health. Ready is /leader: a control plane holds the + # agents' connections and places models only once it leads — on + # PostgreSQL, once the lease of the one it replaced has run out. A + # cluster with no node yet is still ready: /ready would wait for a + # replica, and nodes could then never connect. + startupProbe: + httpGet: {path: /health, port: api} + periodSeconds: 5 + failureThreshold: 60 + livenessProbe: + httpGet: {path: /health, port: api} + periodSeconds: 10 + failureThreshold: 6 + readinessProbe: + httpGet: {path: /leader, port: api} + periodSeconds: 5 + failureThreshold: 2 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/llmgw/server.yaml + subPath: server.yaml + readOnly: true + - name: agent-tls + mountPath: /etc/llmgw/tls + readOnly: true + - name: admin-token + mountPath: /etc/llmgw/admin + readOnly: true + {{- if eq .Values.store.type "postgres" }} + - name: store + mountPath: /etc/llmgw/store + readOnly: true + {{- end }} + {{- if .Values.offlineChecksums.configMap }} + - name: offline + mountPath: /etc/llmgw/offline + readOnly: true + {{- end }} + - name: data + mountPath: /var/lib/llmgw + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: {{ include "llmgw.fullname" . }} + - name: agent-tls + secret: + secretName: {{ .Values.agentTLS.existingSecret }} + defaultMode: 0440 + items: + - {key: ca.pem, path: ca.pem} + - {key: server.pem, path: server.pem} + - {key: server-key.pem, path: server-key.pem} + - name: admin-token + secret: + secretName: {{ include "llmgw.adminSecret" . }} + defaultMode: 0440 + items: + - key: {{ if .Values.adminToken.existingSecret }}{{ .Values.adminToken.secretKey }}{{ else }}admin-token{{ end }} + path: admin-token + {{- if eq .Values.store.type "postgres" }} + - name: store + secret: + secretName: {{ .Values.store.postgres.existingSecret }} + defaultMode: 0440 + items: + - key: {{ .Values.store.postgres.secretKey }} + path: dsn + {{- end }} + {{- if .Values.offlineChecksums.configMap }} + - name: offline + configMap: + name: {{ .Values.offlineChecksums.configMap }} + items: + - key: {{ .Values.offlineChecksums.key }} + path: offline.sha256 + {{- end }} + - name: tmp + emptyDir: {} + {{- if eq .Values.store.type "postgres" }} + # The store is PostgreSQL; nothing here outlives the pod. + - name: data + emptyDir: {} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if eq .Values.store.type "sqlite" }} + volumeClaimTemplates: + - metadata: + name: data + labels: + {{- include "llmgw.selectorLabels" . | nindent 10 }} + spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.store.sqlite.storageClass }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.store.sqlite.size }} + {{- end }} diff --git a/deploy/helm/llmgw/values.yaml b/deploy/helm/llmgw/values.yaml new file mode 100644 index 0000000..b989ccc --- /dev/null +++ b/deploy/helm/llmgw/values.yaml @@ -0,0 +1,132 @@ +# The llmgw control plane in Kubernetes. docs/kubernetes.md says what this +# runs, what it does not, and how nodes outside the cluster join it. + +image: + # Where you pushed the image built from deploy/container/Dockerfile. The + # project publishes binaries, not an image, so this has no default. + repository: "" + # Empty takes the chart's appVersion. + tag: "" + pullPolicy: IfNotPresent +imagePullSecrets: [] + +# One control plane, always. Two against PostgreSQL would elect a leader and +# keep a standby, but a standby serves nothing and is never ready, so a +# rollout would wait on it for ever. Until the chart handles that, it runs +# one: Kubernetes replaces it within seconds when it goes, and the agents +# reconnect on their own. +replicas: 1 + +store: + # sqlite: a file on a PersistentVolumeClaim, for one replica. + # postgres: a PostgreSQL you run, its address in a Secret. + type: sqlite + sqlite: + size: 10Gi + storageClass: "" + postgres: + # A Secret holding the address, postgres://user:password@host:5432/llmgw?sslmode=verify-full + existingSecret: "" + secretKey: dsn + +# The agent port's certificate. It comes from llmgw's own authority, not a +# public one — agents present certificates from the same authority — so it +# is made outside the cluster, where the authority's key stays: +# +# llmgw ctl cert init-ca --dir ./pki +# llmgw ctl cert issue-server llmgw-ctl --host --dir ./pki --out ./pki +# kubectl create secret generic llmgw-agent-tls \ +# --from-file=ca.pem=pki/ca.pem \ +# --from-file=server.pem=pki/server.pem \ +# --from-file=server-key.pem=pki/server-key.pem +# +# --host is every address agents dial: agentService's load balancer address +# or DNS name. +agentTLS: + existingSecret: "" + +# The admin API's token (llmgw ctl, the console). Empty makes one, kept in a +# Secret across upgrades. +adminToken: + existingSecret: "" + secretKey: admin-token + +# server.yaml beyond what the chart sets (listeners, TLS, store, admin +# token): any key of docs/reference.md, laid over the chart's. An offline +# cluster sets offline.mirror here, and offlineChecksums below. +config: + control: + enabled: true + auth: + enabled: true + signup_open: false + default_rpm: 60 + default_tpm: 200000 + default_price: + prompt_per_1k: 1000 + completion_per_1k: 1000 + max_key_ttl: 2160h + # mirror: + # api: https://models.internal:8082 + # offline: + # mirror: http://models.internal:8081 + # firewall_networks: ["10.42.0.0/16"] # the pods' network, so nodes' engines answer the router + +# On an offline cluster, a ConfigMap or Secret holding the engine checksums +# 'llmgw bundle trust' writes (offline.sha256), mounted where offline.checksums +# points. Empty mounts none. +offlineChecksums: + configMap: "" + key: offline.sha256 + +# The OpenAI-compatible API and the metrics, inside the cluster. +service: + type: ClusterIP + port: 8080 + metricsPort: 9091 + annotations: {} + +# Where agents connect, with mutual TLS. Nodes run outside the cluster, so +# this is a load balancer by default; the TLS ends in the pod, never at a +# proxy in between. +agentService: + type: LoadBalancer + port: 7443 + annotations: {} + loadBalancerIP: "" + loadBalancerSourceRanges: [] + # Keeps the agents' addresses as they are, which the node's identity does + # not need but the logs and the audit read better with. + externalTrafficPolicy: Local + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: llm.example.com + paths: + - path: / + pathType: Prefix + tls: [] + # - secretName: llm-example-com + # hosts: [llm.example.com] + +# A ServiceMonitor for the Prometheus Operator, scraping the metrics port. +serviceMonitor: + enabled: false + interval: 30s + labels: {} + +resources: + requests: + cpu: 500m + memory: 512Mi + limits: + memory: 2Gi + +nodeSelector: {} +tolerations: [] +affinity: {} +podAnnotations: {} +podLabels: {} diff --git a/deploy/mirror/README.md b/deploy/mirror/README.md index 44d3bf6..9a5ee54 100644 --- a/deploy/mirror/README.md +++ b/deploy/mirror/README.md @@ -25,6 +25,13 @@ An HTTP server serving a directory. That is the whole design. model.safetensors ``` +At an offline site the mirror also holds what the nodes would otherwise fetch +from the internet — llmgw's releases and the llama.cpp builds — under +`_llmgw/releases/` and `_llmgw/engines/`, put there by `llmgw bundle import` +(docs/install.md, "Offline sites"). They are not models: `mirror list` and +`mirror unused` pass over them, and a node checks each against what the +control plane says it must be, not against anything the mirror says. + The date in the path is the point of the layout, not decoration. A model repository that changes upstream changes what you are running, and the first sign is a prompt that worked last week answering differently this week with diff --git a/docs/console.md b/docs/console.md index 8a8b047..3a6630e 100644 --- a/docs/console.md +++ b/docs/console.md @@ -52,6 +52,18 @@ cluster, a backup and adding a node change this machine or another over ssh; those close the screen and run in the terminal, so their output is there to read, and Enter brings the console back. +## Setting up + +On a machine with nothing installed, the control plane is set up from one +form (`sudo llmgw server init`): the administrator's email, other names nodes +reach the machine by, and ticks for PostgreSQL, the model mirror, this machine +as a node, the starter model and the guardrails' judge. **Offline bundle** +and **Its sha256** are for a site with no internet: the path of a bundle made +by `llmgw bundle create`, and the sha256 it printed. Given, the bundle goes +into the mirror, every file checked, and the cluster is set up to fetch +nothing from outside (install.md, "Offline sites"). A bundle with no sha256, +or with the mirror unticked, is said in the form and nothing starts. + ## Watch **Overview** is `llmgw ctl status`: nodes, GPUs by class, fabrics, replicas, diff --git a/docs/data.md b/docs/data.md index 77ed63a..b604b18 100644 --- a/docs/data.md +++ b/docs/data.md @@ -95,22 +95,62 @@ your own hardware, and the answer comes back. The router talks to engines, the engines talk to nothing, the agents talk to the control plane. All of it is inside the cluster. -Two things can reach outside, both started by an operator rather than by a -request: +Around the serving path, a cluster set up the ordinary way does reach out. +None of it carries a prompt, an answer, a key or a user — it is software and +public model files coming in — but a review should know each one: -**Fetching weights.** When a model's `source` is a URL, the agent downloads it. -Point that at an internal mirror and nothing leaves at all — which is also the +**Installing engines.** Each node's agent downloads the llama.cpp build for its +hardware from GitHub, checked against the checksum GitHub publishes, and vLLM +(SGLang when asked) from PyPI, with `uv` from GitHub. On every node, when it +joins and when the engine version changes. + +**Releases.** `llmgw node add`, `llmgw ctl upgrade` and `llmgw upgrade` fetch +llmgw's own release binaries from the project's release server, checked against +its signed checksums. + +**Fetching weights.** When a model's `source` is a URL, the agent downloads it; +the mirror fetches what it is asked to, from Hugging Face or another URL, and +`server init` has it fetch the starter model and the guardrails' judge. Point +sources at the mirror and the nodes fetch only from it — which is also the faster arrangement, because eight nodes pulling forty gigabytes from the internet is a day's work and from a local mirror it is minutes. A `source` that is a path on disk is never fetched. -**`llmgw ctl plan`.** Reads a model's architecture from huggingface.co to work -out whether it fits your hardware. An operator command run at planning time, -not part of serving, and not run by anything automatically. +**The registry.** With a mirror configured, the control plane asks Hugging Face +every six hours whether the models served from it have newer files +(`mirror.update_check`), and `/admin/mirror` searches it. `llmgw ctl plan` +reads a model's architecture from huggingface.co to work out whether it fits +your hardware. -There is no telemetry, no licence check, and no call home. The binary is -static, built with `CGO_ENABLED=0`, and runs in a network with no route to the -internet as long as the weights are already on disk. +There is no telemetry, no licence check, and no call home. + +### A cluster with no way out + +A site with no route to the internet installs from an **offline bundle** +instead, and then nothing at all leaves the cluster (docs/install.md, "Offline +sites"). The bundle is made on a machine that has internet — llmgw's release +binaries with their signed checksums, the llama.cpp builds with the checksums +GitHub publishes, the model files with the checksums Hugging Face publishes, +each checked as it is downloaded — carried in, and imported into the mirror. +With `offline.mirror` set in `server.yaml`, nodes install engines from the +mirror and never from GitHub, releases come from the mirror, and the registry +is not asked about newer weights. + +Verification is not weakened by any of this. A release is held to the project's +signing key wherever it is fetched from. An engine file is held to the checksum +recorded when the bundle was made, which the control plane keeps and hands each +node over the agent's authenticated connection: the mirror serves plain HTTP, +and nothing a node runs is taken on the mirror's word. The bundle itself is +checked against the sha256 its maker printed, carried to the site apart from +the bundle. + +What does not come in a bundle: vLLM and SGLang, which are Python environments +from PyPI — an offline node serves with llama.cpp unless one is installed on it +by hand — and the operating system's own packages (the OpenMP runtime the +engine needs, PostgreSQL when chosen), which come from the machines' package +sources: at an offline site, the distribution mirror it already has. Searching +the registry from `/admin/mirror` and `llmgw ctl plan` need the internet and +do not work offline; `mirror.registry` can point the first at an internal copy. ## What the software cannot do for you diff --git a/docs/install.md b/docs/install.md index 9059a9b..f70c554 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1245,6 +1245,139 @@ doctor` on a node says when the table is missing. `--no-firewall` on any of the four commands leaves the firewall alone; `sudo systemctl disable --now llmgw-firewall` removes it. +## Offline sites + +Everything above downloads: the nodes fetch llama.cpp from GitHub and vLLM from +PyPI, `node add` and upgrades fetch llmgw's releases, and the mirror fetches +models from Hugging Face. A site with no route to the internet gets all of that +in a **bundle**, made somewhere that has one and carried in. + +### Making the bundle + +On any machine with internet and llmgw — a laptop will do: + +``` +llmgw bundle create --out llmgw-bundle --arch amd64 --tar +``` + +That carries: + +- **llmgw's release** for each `--arch` (default: this llmgw's own version; + `--version` for another), with `checksums.txt` and its signature, checked + against the project's signing key as `ctl upgrade` checks it. +- **llama.cpp** at the version this llmgw installs, every build for each + architecture — CPU, CUDA 13.3, CUDA 12.8 (amd64), Vulkan — each checked + against the checksum GitHub publishes, the same check a node does online. + `--variant cpu --variant cuda-13.3` carries only those, for a site whose + hardware you know; `--engine llamacpp=b10991` names the version. +- **The starter model and the guardrails' judge** that `server init` serves + (about 1.3 GB; `--no-starter-models` leaves them out), and any + `--model name/version=hf:owner/repo/file` you add, pinned to the commit the + registry had and checked against the checksum it publishes. + +It ends by printing the manifest's **sha256**. Carry that apart from the bundle +— in the ticket, read out over the phone — because it is what says the bundle +that arrived is the one that was made. `llmgw bundle show llmgw-bundle.tar +--verify` reads every file back, worth doing once it is on the disk that +carries it. Run `create` again with the same `--out` (without `--tar`) to add +to a bundle; what is there already is not downloaded again. + +What a bundle does **not** carry: vLLM and SGLang, which are Python +environments of several gigabytes from PyPI. Offline nodes serve with llama.cpp; +on a GPU node that needs vLLM, install it by hand and name it with `node add +--vllm-bin` (the node reports vLLM as unsupported until then). Nor the +operating system's packages — the OpenMP runtime llama.cpp needs, PostgreSQL +with `--postgres` — which come from each machine's own package sources: at an +offline site, its distribution mirror. + +### Installing from it + +On the control plane, with the bundle on a local disk: + +``` +sudo llmgw server init --bundle /media/usb/llmgw-bundle.tar --bundle-sha256 3f2a… +``` + +(The console's setup form has the same two fields.) Init then does what it +always does, with these differences: + +1. Before the control plane starts, the bundle goes into the mirror beside it: + every file checked against the manifest, the release against the signing + key, and only then put in place. llmgw's releases and the engines go under + `/srv/models/_llmgw/`, the models where the mirror keeps models. +2. The engines' checksums are written to `/etc/llmgw/offline.sha256`. That + file, on the control plane, is what each node's engine files are held to. +3. `server.yaml` gets + + ```yaml + offline: + mirror: http://10.0.0.5:8081 # the mirror's file port, as nodes reach it + ``` + + and from then on the cluster asks nothing of the internet: nodes install + llama.cpp from the mirror, `releases` defaults to the mirror's copy (so + `node add` and `ctl upgrade` fetch from it), and the registry is not asked + about newer weights. +4. The starter model and the judge are served from the mirror when the bundle + carries them, and skipped — not waited for — when it does not. Without the + judge, the guard policy is written without one. + +With the mirror on another machine (`--mirror`), import the bundle there first +and then run init with `--bundle` as above: the control plane records only the +checksums. + +``` +# on the mirror machine +sudo llmgw bundle import /media/usb/llmgw-bundle.tar --sha256 3f2a… +``` + +Nodes join as usual with `sudo llmgw node add`: it fetches the node's binary +from the mirror and checks its signature, and the agent installs llama.cpp +from the mirror. Nothing on the node needs the internet. + +### How it stays checked + +Online, a node trusts GitHub, over TLS, for the checksum of the llama.cpp it +installs, and the project's signing key for llmgw. Offline it is the same +strength by a different route: the checksum GitHub published is recorded when +the bundle is made, the bundle is held to its sha256 when it is imported, and +the control plane keeps the checksums and tells each node, over its mutually +authenticated connection, what every file it fetches must be. The mirror serves +plain HTTP and is never asked for a checksum: a file changed on it, or on the +way from it, is refused by the node. Releases carry the project's signature, +which holds whatever server they come from — so plain HTTP from the mirror is +allowed for releases only because this build trusts a signing key. + +### Later + +A new llmgw release or llama.cpp version is another bundle. Make it the same +way — `--version v0.12.0`, `--engine llamacpp=b11200` — and on the control plane +(and the mirror, if separate): + +``` +sudo llmgw bundle import new-bundle.tar --sha256 … +``` + +The control plane sees the new checksums without a restart. Then set +`engines.llamacpp` in `server.yaml` to the new version and restart +`llmgw-server`, or run `sudo llmgw ctl upgrade v0.12.0` for the release. A +mirror on its own machine, which is not an agent, is upgraded from its own +copy: `sudo llmgw upgrade v0.12.0 --service llmgw-mirror --releases +http://127.0.0.1:8081/_llmgw/releases`. A control plane whose bundle lacks the +llama.cpp version it asks for says so in its log at start, and the node says +which `bundle create` would carry it. More models: `--model` in a bundle, or +copy the files into the mirror by hand — it is a directory. + +A cluster set up online becomes offline the same way: `bundle import` on the +mirror and control plane, then the `offline:` block in `server.yaml` and a +restart. + +## Kubernetes + +The control plane runs in Kubernetes from the Helm chart in +`deploy/helm/llmgw`; GPU nodes are best kept outside it, as ordinary hosts +joined with `node add`. Why, and how, is in [kubernetes.md](kubernetes.md). + ## Upgrading On the control plane: @@ -1488,6 +1621,8 @@ requests. On a slow model, set `timeouts.generation` higher. wrote about - `deploy/mirror/README.md` — the mirror in more detail, including the layout and what belongs on which port +- `docs/kubernetes.md` — the control plane in Kubernetes from the Helm chart, + and why GPU nodes stay outside it - `deploy/monitoring/README.md` — Prometheus, Grafana and the alerts, with what each one assumes about this system - `docs/fabrics.md` — InfiniBand fabrics and NVLink: what llmgw needs of them, diff --git a/docs/kubernetes.md b/docs/kubernetes.md new file mode 100644 index 0000000..d89b5a8 --- /dev/null +++ b/docs/kubernetes.md @@ -0,0 +1,228 @@ +# llmgw and Kubernetes + +The control plane runs in Kubernetes, from the Helm chart in +`deploy/helm/llmgw`. The GPU nodes do not: they stay ordinary machines, +joined with `llmgw node add` as on any other cluster. This page says why, +and how the two meet. + +## What goes where + +| Part | Where | Why | +|---|---|---| +| Control plane: the API on 8080, the agents' port 7443, the scheduler | A StatefulSet, from the chart | A Go binary with a store and three ports: what Kubernetes runs well | +| Store | A file on a PersistentVolumeClaim (SQLite), or your PostgreSQL | Either, as on a host | +| GPU and CPU nodes | Outside, as hosts | See below | +| Model mirror | Outside, on a machine with a large disk | Every node pulls tens of gigabytes from it over plain HTTP; a pod with a volume that size is a host in all but name | + +### Why the nodes stay outside + +The agent on a node is not a service that happens to use a GPU. It installs +the engines on the machine — the llama.cpp build for its driver, vLLM in a +Python environment — checks them against their publisher's checksums and +runs them; it chooses which GPUs each replica gets, by index and by NVLink +topology; it starts engines as processes that outlive its own restarts, so a +node upgrades while it serves; it writes the machine's firewall; and it +replaces its own binary when the cluster upgrades. All of that is root on +the host, which is exactly what a container is built to prevent. + +Run as a DaemonSet it would need to be privileged, with the host's network, +process namespace and filesystem, and the NVIDIA device plugin would have to +be kept off its GPUs — or the two would hand the same card to two owners. +What is left is the host agent with a container's indirection around it, and +none of the scheduling, isolation or restart guarantees that make a pod +worth having. So the chart does not ship one, and this is the +recommendation: + +- **Keep GPU machines out of Kubernetes**, and add them with `node add`. +- **If they are already Kubernetes workers**, dedicate them: taint them so + no other pod lands there, keep the NVIDIA device plugin off them (most + installs select nodes by label — leave the label off), and run the llmgw + agent on the host, where `node add` puts it. llmgw then owns the GPUs, and + Kubernetes schedules nothing that would want them. + +## The image + +The project publishes binaries, not an image. Build one from the repository +and push it where the cluster pulls from — at an offline site, the internal +registry: + +``` +docker build -f deploy/container/Dockerfile --build-arg VERSION=v0.11.1 \ + -t registry.internal/llmgw:v0.11.1 . +docker push registry.internal/llmgw:v0.11.1 +``` + +It is the static binary on a distroless base: no shell, running as user +65532, with a read-only root filesystem in the pod. + +## The agent port's certificate + +Agents and the control plane authenticate each other with certificates from +llmgw's own authority, and the TLS ends in the pod — so the authority's key +stays outside the cluster, on the machine you will add nodes from, and the +pod is given only its own certificate. + +Decide first the address agents will dial: the `-agents` Service is a +LoadBalancer, so give it a fixed `agentService.loadBalancerIP` or a DNS name +that points at it. That address has to be in the certificate: + +``` +llmgw ctl cert init-ca --dir ./pki +llmgw ctl cert issue-server llmgw-ctl --host 10.0.0.200 --host llmgw-agents.example.com \ + --dir ./pki --out ./pki +kubectl -n llmgw create secret generic llmgw-agent-tls \ + --from-file=ca.pem=pki/ca.pem \ + --from-file=server.pem=pki/server.pem \ + --from-file=server-key.pem=pki/server-key.pem +``` + +Keep `pki/ca-key.pem` as you would on a host: whoever holds it can make any +node's identity. + +## Installing + +```yaml +# values.yaml +image: + repository: registry.internal/llmgw + tag: v0.11.1 +agentTLS: + existingSecret: llmgw-agent-tls +agentService: + loadBalancerIP: 10.0.0.200 +ingress: + enabled: true + className: nginx + hosts: [{host: llm.example.com, paths: [{path: /, pathType: Prefix}]}] + tls: [{secretName: llm-example-com, hosts: [llm.example.com]}] +config: + mirror: + api: http://models.internal:8082 + # The address the router reaches nodes' engines from is a Kubernetes + # node's, once the pod's traffic leaves the cluster: the nodes' firewalls + # must let it in. + firewall_networks: ["10.0.1.0/24"] +``` + +``` +helm install llmgw deploy/helm/llmgw -n llmgw -f values.yaml +``` + +`config` is `server.yaml`: any key of [reference.md](reference.md), laid +under what the chart sets itself — the listeners, where the certificates, +the admin token and the store are mounted. A changed `config` restarts the +pod. + +**The store.** By default SQLite, on a 10 GiB claim. For PostgreSQL, put its +address in a Secret and point the chart at it: + +``` +kubectl -n llmgw create secret generic llmgw-store \ + --from-literal=dsn='postgres://llmgw:…@pg.internal:5432/llmgw?sslmode=verify-full' +``` + +```yaml +store: + type: postgres + postgres: + existingSecret: llmgw-store +``` + +**One control plane.** The chart runs one replica. Two against PostgreSQL +would elect a leader and keep a standby, but a standby serves nothing and is +never ready, so a StatefulSet rollout would wait on it for ever. Kubernetes +replaces the one pod within seconds when it goes, and agents reconnect on +their own; the replicas on the nodes keep serving meanwhile. + +**Probes.** Alive is `/health`. Ready is `/leader`, which on PostgreSQL waits +for the lease of the pod it replaced to run out — not `/ready`, which waits +for a model replica, and would keep a cluster with no node yet from ever +accepting one. + +**Monitoring.** The metrics port, 9091, serves metrics and nothing else; +`serviceMonitor.enabled: true` makes a ServiceMonitor for the Prometheus +Operator, and `deploy/monitoring` holds the rules and dashboards. + +## Using it + +The admin API listens on the pod's loopback, as on a host — it can dump keys +from the heap when profiling is on. Reach it through the pod: + +``` +kubectl -n llmgw exec -it statefulset/llmgw -- llmgw users create you@example.com --password '…' +kubectl -n llmgw exec -it statefulset/llmgw -- llmgw users role you@example.com admin +kubectl -n llmgw exec -it statefulset/llmgw -- llmgw ctl status +``` + +or from your machine, through a port-forward: + +``` +kubectl -n llmgw port-forward statefulset/llmgw 9090:9090 & +export LLMGW_ADMIN_TOKEN=$(kubectl -n llmgw get secret llmgw-admin -o jsonpath='{.data.admin-token}' | base64 -d) +llmgw ctl status +``` + +The token is made by the chart and kept across upgrades +(`adminToken.existingSecret` brings your own). + +## Adding nodes + +From the machine holding `pki/`, with the port-forward above running and +`LLMGW_ADMIN_TOKEN` set: + +``` +sudo -E llmgw node add ubuntu@gpu01 \ + --server https://10.0.0.200:7443 --pki ./pki --admin 127.0.0.1:9090 +``` + +`--server` is the address in the certificate. The node is installed, checked +and started over ssh exactly as on any cluster ([node-onboarding.md](node-onboarding.md)), +and dials the control plane in the pod. Its firewall comes from +`firewall_networks` in the server configuration `node add` reads (`--config`, +`/etc/llmgw/server.yaml` by default): copy the chart's out of the cluster +first, or give `--no-firewall` and keep the engines' ports open to the +Kubernetes nodes by hand. + +``` +kubectl -n llmgw get configmap llmgw -o jsonpath='{.data.server\.yaml}' > server.yaml +sudo -E llmgw node add ubuntu@gpu01 --config ./server.yaml --server https://10.0.0.200:7443 \ + --pki ./pki --admin 127.0.0.1:9090 +``` + +## Offline + +An offline site ([install.md](install.md), "Offline sites") works the same +way with the control plane in a pod. Import the bundle on the mirror's +machine with `llmgw bundle import`, record its engines' checksums in a +file, and give that file to the chart as a ConfigMap: + +``` +llmgw bundle trust llmgw-bundle.tar --sha256 3f2a… --file ./offline.sha256 +kubectl -n llmgw create configmap llmgw-offline --from-file=offline.sha256 +``` + +```yaml +offlineChecksums: + configMap: llmgw-offline +config: + offline: + mirror: http://models.internal:8081 +``` + +A later bundle: `bundle trust` into the same file, and replace the ConfigMap +(`kubectl create configmap … --dry-run=client -o yaml | kubectl apply -f -`). +The control plane reads the file again when it changes; there is no restart. + +## What has and has not been tried + +The chart is checked with `helm lint` and `helm template` for SQLite, +PostgreSQL, an Ingress, a ServiceMonitor and an offline cluster, and refuses +what would not work — no image, no certificate, a PostgreSQL store without +its address, more than one replica. The image has been run the way the +chart runs it — read-only root, user 65532, the same files mounted at the +same paths, SQLite on a volume — and answered `/health`, `/leader` and +`/metrics`, with `llmgw ctl` and `llmgw users` working through `exec`. + +It has **not** been installed on a Kubernetes cluster, and no node has yet +joined a control plane running in one. Until it has, a control plane on a +host is the well-trodden path. diff --git a/docs/reference.md b/docs/reference.md index df96ec4..6742233 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -120,6 +120,8 @@ rather than repeated. | `roles` | `map[string][]string` | [OIDC](#oidc) | | `memberships` | `map[string]string` | [OIDC](#oidc) | | `no_passwords` | `bool` | [OIDC](#oidc) | +| `mirror` | `string` | [Offline](#offline) | +| `checksums` | `string` | [Offline](#offline) | | `nvidia_smi` | `string` | [Options](#options) | | `ib_root` | `string` | [Options](#options) | | `prefer_nvml` | `bool` | [Options](#options) | @@ -226,6 +228,7 @@ rather than repeated. | `monitoring` | `Monitoring` | [Server](#server) | | `tracing` | `Tracing` | [Server](#server) | | `mirror` | `Mirror` | [Server](#server) | +| `offline` | `Offline` | [Server](#server) | | `engines` | `map[string]string` | [Server](#server) | | `engine_overrides` | `[]EngineOverride` | [Server](#server) | | `backup_dir` | `string` | [Server](#server) | @@ -1084,6 +1087,29 @@ PasswordSignIn off refuses passwords on the sign-in form: people come in through the provider only. Administrators still can, so a broken provider does not lock everybody out. +## Offline + +### `mirror` + +`string` + +Mirror is the mirror's file address, as nodes reach it: +http://10.0.0.5:8081 — the port model files are served on, not the +management API. Set, the cluster is offline: nodes install engines +from it and never from their publishers, releases come from it unless +releases names another place, and no registry is asked whether a +model has newer weights. Python engines (vLLM, SGLang) come from PyPI, +which no bundle carries; on an offline node they are installed by hand. + +### `checksums` + +`string` + +Checksums is the file of engine checksums this control plane trusts, +kept by llmgw bundle import and llmgw bundle trust; every engine file +a node fetches from the mirror is checked against it. Read again +whenever it changes. Empty is /etc/llmgw/offline.sha256. + ## Options ### `nvidia_smi` @@ -1834,6 +1860,16 @@ Empty means there is none, and /admin/mirror stays unmounted. Takes a [Mirror](#mirror) block. +### `offline` + +`Offline` + +Offline is a cluster that reaches nothing outside itself: its nodes +install engines, and upgrades fetch releases, from the mirror, out of +a bundle carried in (llmgw bundle create, llmgw bundle import). + +Takes a [Offline](#offline) block. + ### `engines` `map[string]string` diff --git a/internal/agent/provision.go b/internal/agent/provision.go index fbd3d28..d49557f 100644 --- a/internal/agent/provision.go +++ b/internal/agent/provision.go @@ -88,7 +88,13 @@ func (a *Agent) setProvision(p *llmgwv1.Provision) { // the installed build is used first — and quietly replace theirs. continue } - wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion(), Variant: e.GetVariant()}) + w := provision.Want{Engine: e.GetEngine(), Version: e.GetVersion(), Variant: e.GetVariant()} + if base := e.GetOfflineBase(); base != "" { + // An offline cluster: from its mirror, checked against what the + // control plane holds, never the publisher. + w.Offline = &provision.Offline{Base: base, SHA256: maps.Clone(e.GetOfflineSha256())} + } + wants = append(wants, w) } a.mu.Lock() a.provWants = wants diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go index 740759b..da4c764 100644 --- a/internal/agent/provision/install.go +++ b/internal/agent/provision/install.go @@ -30,6 +30,24 @@ type Want struct { // Variant is a build chosen for this node ("cu129"); empty leaves it to // the node's hardware. Variant string + // Offline, when set, is where the engine comes from instead of its + // publisher: a cluster with no way to the internet installs from its + // mirror's copy of an offline bundle. + Offline *Offline +} + +// Offline is an engine version's files on a cluster's own mirror. +// +// The files come from the mirror and their checksums from the control plane, +// which read them from the bundle they were imported from. The mirror serves +// plain HTTP to every node; a checksum taken from beside the file there would +// prove only that the file and the checksum came from the same place. +type Offline struct { + // Base is where the version's files are: /_llmgw/engines//. + Base string + // SHA256 is each file's checksum, by name, as the publisher gave it + // when the bundle was made. + SHA256 map[string]string } // The states an engine's installation can be in. @@ -152,6 +170,16 @@ func (in *Installer) one(ctx context.Context, w Want) Status { } return st } + if w.Offline != nil && recipe.Repo == "" { + // Said, not attempted: uv would spend minutes failing to reach PyPI, + // and the failure would read like a network fault rather than the + // design of an offline site. + st.State = Unsupported + st.Detail = "this cluster installs offline, and " + w.Engine + " comes from PyPI, which an offline bundle " + + "does not carry: install it on the node by hand and name it in agent.yaml, or set engines." + + w.Engine + ` to "" in server.yaml` + return st + } // Installed before: whichever candidate is, best first, checked that it // still runs and trusted for nothing else. @@ -259,10 +287,15 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir return err } - in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version) - sums, err := in.checksums(ctx, r.Repo, w.Version) - if err != nil { - return err + var sums map[string]string + if w.Offline != nil { + sums = w.Offline.SHA256 + } else { + in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version) + var err error + if sums, err = in.checksums(ctx, r.Repo, w.Version); err != nil { + return err + } } tmp := dir + ".partial" @@ -275,14 +308,22 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir got := map[string]string{} for _, asset := range b.Assets { want, ok := sums[asset] - if !ok { + switch { + case !ok && w.Offline != nil: + return fmt.Errorf("the offline bundle has no %s for %s %s: make one that has "+ + "(llmgw bundle create --engine %s=%s) and import it", asset, w.Engine, w.Version, w.Engine, w.Version) + case !ok: // Refused rather than installed unchecked: this is a binary that // will run as root on every node. return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified", w.Engine, w.Version, asset) } + url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + r.Repo + "/releases/download/" + w.Version + "/" + asset + if w.Offline != nil { + url = strings.TrimSuffix(w.Offline.Base, "/") + "/" + asset + } archive := filepath.Join(tmp, asset+".download") - if err := in.download(ctx, r.Repo, w.Version, asset, archive, want, st); err != nil { + if err := in.download(ctx, url, asset, archive, want, st); err != nil { return err } in.progress(st, "unpacking %s", asset) @@ -539,6 +580,26 @@ func (in *Installer) has(tool string) bool { return err == nil } +// Published is the sha256 an engine's publisher gives for each file of a +// version, and where each is downloaded from: what an offline bundle is made +// of, read the way a node online reads it. +func (in *Installer) Published(ctx context.Context, engine, version string) (sums map[string]string, url func(asset string) string, err error) { + r, ok := Recipes[engine] + if !ok || r.Repo == "" { + return nil, nil, fmt.Errorf("%s is not published as release files", engine) + } + if !validVersion.MatchString(version) { + return nil, nil, fmt.Errorf("%q is not a version of %s", version, engine) + } + sums, err = in.checksums(ctx, r.Repo, version) + if err != nil { + return nil, nil, err + } + return sums, func(asset string) string { + return strings.TrimSuffix(in.downloadBase(), "/") + "/" + r.Repo + "/releases/download/" + version + "/" + asset + }, nil +} + // checksums reads the sha256 GitHub publishes for each asset of a release. func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) { url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag @@ -580,8 +641,7 @@ func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[strin return out, nil } -func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want string, st *Status) error { - url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + repo + "/releases/download/" + tag + "/" + asset +func (in *Installer) download(ctx context.Context, url, asset, dst, want string, st *Status) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return err diff --git a/internal/agent/provision/offline_test.go b/internal/agent/provision/offline_test.go new file mode 100644 index 0000000..978a631 --- /dev/null +++ b/internal/agent/provision/offline_test.go @@ -0,0 +1,147 @@ +package provision + +import ( + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +// offlineMirror is a cluster's mirror holding a bundle's engine files, and +// nothing that answers for GitHub. +type offlineMirror struct { + files map[string][]byte + mu sync.Mutex + paths []string +} + +func (m *offlineMirror) serve(t *testing.T) *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m.mu.Lock() + m.paths = append(m.paths, r.URL.Path) + m.mu.Unlock() + b, ok := m.files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(b) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestAnOfflineNodeInstallsFromTheMirrorWithChecksumsFromTheControlPlane(t *testing.T) { + rel := llamaRelease(t) + asset := "llama-b1-bin-ubuntu-x64.tar.gz" + m := &offlineMirror{files: map[string][]byte{"/_llmgw/engines/llamacpp/b1/" + asset: rel.assets[asset]}} + srv := m.serve(t) + sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}} + // GitHub is the mirror too, so a request for it would be seen and + // answered 404: an offline install must not ask. + in, _ := installer(t, srv, sys, ubuntu) + + want := Want{Engine: "llamacpp", Version: "b1", Offline: &Offline{ + Base: srv.URL + "/_llmgw/engines/llamacpp/b1", + SHA256: map[string]string{asset: strings.TrimPrefix(rel.digests[asset], "sha256:")}, + }} + got := in.Converge(t.Context(), []Want{want}) + if got[0].State != Ready { + t.Fatalf("not installed from the mirror: %+v", got[0]) + } + for _, p := range m.paths { + if strings.HasPrefix(p, "/repos/") || strings.Contains(p, "/releases/download/") { + t.Errorf("an offline install asked for %s", p) + } + } + + // A file on the mirror that is not what the bundle recorded is refused, + // whatever the mirror says of it. + m.files["/_llmgw/engines/llamacpp/b2/llama-b2-bin-ubuntu-x64.tar.gz"] = []byte("replaced on the mirror") + got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b2", Offline: &Offline{ + Base: srv.URL + "/_llmgw/engines/llamacpp/b2", + SHA256: map[string]string{"llama-b2-bin-ubuntu-x64.tar.gz": strings.Repeat("a", 64)}, + }}}) + if got[0].State != Failed || !strings.Contains(got[0].Detail, "checksum") { + t.Errorf("a file not matching the bundle's checksum was not refused: %+v", got[0]) + } + + // One the bundle has no checksum for is not fetched at all, and the + // operator is told which bundle would have it. + got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b3", Offline: &Offline{ + Base: srv.URL + "/_llmgw/engines/llamacpp/b3", + }}}) + if got[0].State != Failed || !strings.Contains(got[0].Detail, "bundle create --engine llamacpp=b3") { + t.Errorf("a version missing from the bundle did not say how to add it: %+v", got[0]) + } +} + +func TestAnOfflineNodeIsToldPythonEnginesComeFromPyPI(t *testing.T) { + gpu := ubuntu + gpu.NVIDIADriver, gpu.MinComputeCapability = "580.65", 8.9 + in := &Installer{Root: t.TempDir(), Host: gpu, Run: (&fakeSystem{}).run} + got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0", Offline: &Offline{Base: "http://mirror"}}}) + if got[0].State != Unsupported || !strings.Contains(got[0].Detail, "PyPI") { + t.Fatalf("vLLM offline: %+v", got[0]) + } + + // A node it is not meant for still says nothing is wrong. + in.Host = ubuntu + got = in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0", Offline: &Offline{Base: "http://mirror"}}}) + if got[0].State != Skipped { + t.Errorf("vLLM offline on a CPU node: %+v", got[0]) + } +} + +func TestABundleCarriesEveryBuildTheNodesMightChoose(t *testing.T) { + variants := func(arch string) []string { + bs, err := Builds("llamacpp", "b1", arch) + if err != nil { + t.Fatal(err) + } + var out []string + for _, b := range bs { + out = append(out, b.Variant) + } + return out + } + if got := variants("amd64"); !slices.Equal(got, []string{"cpu", "cuda-13.3", "cuda-12.8", "vulkan"}) { + t.Errorf("amd64 builds: %v", got) + } + if got := variants("arm64"); !slices.Equal(got, []string{"cpu", "cuda-13.3", "vulkan"}) { + t.Errorf("arm64 builds: %v", got) + } + + // Each one a node would choose is among them, file for file. + bundled := map[string][]string{} + bs, _ := Builds("llamacpp", "b1", "amd64") + for _, b := range bs { + bundled[b.Variant] = b.Assets + } + for _, h := range []discovery.Host{ + ubuntu, + {Arch: "amd64", NVIDIADriver: "550.1", MinComputeCapability: 8.6}, + {Arch: "amd64", NVIDIADriver: "470.1", MinComputeCapability: 6.1}, + } { + chosen, err := llamaCppCandidates("b1", "", h) + if err != nil { + t.Fatal(err) + } + for _, b := range chosen { + if !slices.Equal(bundled[b.Variant], b.Assets) { + t.Errorf("%+v chooses %s %v, which the bundle carries as %v", h, b.Variant, b.Assets, bundled[b.Variant]) + } + } + } + + if _, err := Builds("vllm", "0.30.0", "amd64"); err == nil || !strings.Contains(err.Error(), "PyPI") { + t.Errorf("a Python engine was offered for a bundle: %v", err) + } + if _, err := Builds("llamacpp", "../b1", "amd64"); err == nil { + t.Error("a version that is a path was accepted") + } +} diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index c76ca16..a816e70 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -120,6 +120,49 @@ func Installable() []string { return out } +// Builds are every build of an engine version published for an +// architecture: what an offline bundle carries, for nodes it has not seen. +// Read from the same table the nodes choose from, asked as a machine with +// no GPU and as one with any NVIDIA GPU, so a bundle and the nodes that +// install from it cannot disagree about which files a build is. Archive +// engines only: a Python engine is a PyPI environment, which no bundle +// carries. +func Builds(engine, version, arch string) ([]Build, error) { + r, ok := Recipes[engine] + switch { + case !ok: + return nil, fmt.Errorf("this agent does not know how to install %s", engine) + case r.Repo == "": + return nil, fmt.Errorf("%s is installed from PyPI, which an offline bundle does not carry", engine) + case !validVersion.MatchString(version): + return nil, fmt.Errorf("%q is not a version of %s", version, engine) + } + var out []Build + seen := map[string]bool{} + var errs []string + for _, h := range []discovery.Host{ + {Arch: arch}, + // Any card and the newest driver: every GPU build is a candidate. + {Arch: arch, NVIDIADriver: "9999"}, + } { + bs, err := r.Candidates(version, "", h) + if err != nil { + errs = append(errs, err.Error()) + continue + } + for _, b := range bs { + if !seen[b.Variant] { + seen[b.Variant] = true + out = append(out, b) + } + } + } + if len(out) == 0 { + return nil, fmt.Errorf("no build of %s %s for %s: %s", engine, version, arch, strings.Join(errs, "; ")) + } + return out, nil +} + // UVVersion is the uv that installs the Python engines. const UVVersion = "0.12.20" diff --git a/internal/bundle/bundle.go b/internal/bundle/bundle.go new file mode 100644 index 0000000..c579070 --- /dev/null +++ b/internal/bundle/bundle.go @@ -0,0 +1,213 @@ +// Package bundle carries what a cluster would otherwise fetch from the +// internet to a site that has no way to it. +// +// A bundle is made on a machine with internet access: llmgw's own release +// binaries with their signed checksums, the engine builds the nodes install +// with the checksums their publisher gave, and model files with the +// checksums their registry gave — each checked as it arrives, exactly as a +// node online would check it. Its manifest lists every file with its sha256, +// and the manifest's own sha256 is what the person carrying the bundle +// compares at the other end. +// +// Imported, the files go into the cluster's mirror: releases and engines +// under _llmgw/, which no model can be called, models where the mirror keeps +// models. The engines' checksums go to the control plane, which tells each +// node what every file it fetches from the mirror must be. The mirror serves +// plain HTTP to every node, so nothing a node runs is taken on the mirror's +// word: releases carry the project's signature, engines the checksum the +// control plane holds. +package bundle + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" +) + +// ManifestName is the manifest's file name in a bundle, and the first entry +// of a bundle packed as a tar. +const ManifestName = "manifest.json" + +// Format is the manifest's layout version. A bundle of a later format is +// refused rather than half understood. +const Format = 1 + +// Prefix is where a mirror keeps what is not a model: _llmgw/releases and +// _llmgw/engines. A model name may not start with an underscore here, so +// none can be called this. +const Prefix = "_llmgw" + +// Manifest says what a bundle holds. +type Manifest struct { + Format int `json:"format"` + Created time.Time `json:"created"` + // MadeBy is the llmgw version that made it. + MadeBy string `json:"made_by"` + Releases []Release `json:"releases,omitempty"` + Engines []Engine `json:"engines,omitempty"` + Models []Model `json:"models,omitempty"` + // Files are every file in the bundle but the manifest, with their + // checksums: what an import checks each one against. + Files []File `json:"files"` +} + +// Release is llmgw's own release, for some architectures. +type Release struct { + Version string `json:"version"` + Arches []string `json:"arches"` +} + +// Engine is an engine version's builds, for some architectures. +type Engine struct { + Engine string `json:"engine"` + Version string `json:"version"` + Arches []string `json:"arches"` + Variants []string `json:"variants"` +} + +// Model is a model's files, as the mirror will hold them. +type Model struct { + Name string `json:"name"` + Version string `json:"version"` + // Source is what it was asked for as: hf:owner/repo/file, or a URL. + Source string `json:"source"` + // Revision is the registry's commit it was taken at, when it has one. + Revision string `json:"revision,omitempty"` +} + +// File is one file of a bundle. +type File struct { + // Path is where it is in the bundle: releases//, + // engines/// or models///. + Path string `json:"path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + // Source is where it was downloaded from. + Source string `json:"source,omitempty"` +} + +// Kinds of file, by the first element of their path. +const ( + kindRelease = "releases" + kindEngine = "engines" + kindModel = "models" +) + +// Digest is a manifest's sha256: what `bundle create` prints and an import +// is given to check the bundle by. +func Digest(raw []byte) string { + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) +} + +// parseManifest reads a manifest and checks every path in it, before any +// file is read: a path is where the file is written. +func parseManifest(raw []byte) (Manifest, error) { + var m Manifest + if err := json.Unmarshal(raw, &m); err != nil { + return Manifest{}, fmt.Errorf("the bundle's manifest cannot be read: %w", err) + } + if m.Format != Format { + return Manifest{}, fmt.Errorf("the bundle is of format %d and this llmgw reads format %d: "+ + "import it with the llmgw that made it", m.Format, Format) + } + seen := map[string]bool{} + for _, f := range m.Files { + if _, err := Destination(f.Path); err != nil { + return Manifest{}, err + } + if len(f.SHA256) != 64 { + return Manifest{}, fmt.Errorf("the manifest gives %s no sha256", f.Path) + } + if _, err := hex.DecodeString(f.SHA256); err != nil { + return Manifest{}, fmt.Errorf("the manifest's sha256 of %s is not one", f.Path) + } + if seen[f.Path] { + return Manifest{}, fmt.Errorf("the manifest lists %s twice", f.Path) + } + seen[f.Path] = true + } + return m, nil +} + +// Destination is where a bundle's file goes in a mirror, relative to its +// root, or why it goes nowhere. +// +// Every path in a bundle has a fixed shape — three or four plain elements +// under one of three directories — because a path is where the import +// writes, as root. Anything else is refused, not cleaned. +func Destination(p string) (string, error) { + parts := strings.Split(p, "/") + bad := func(why string) (string, error) { + return "", fmt.Errorf("the bundle names a file %q, which %s", p, why) + } + if path.Clean(p) != p || strings.HasPrefix(p, "/") { + return bad("is not a plain path") + } + for _, e := range parts { + if err := mirror.ValidName(e); err != nil { + return bad("has an element that is not a plain name") + } + } + switch parts[0] { + case kindRelease: + if len(parts) != 3 || !release.Tag.MatchString(parts[1]) { + return bad("is not releases//") + } + return Prefix + "/" + p, nil + case kindEngine: + if len(parts) != 4 { + return bad("is not engines///") + } + return Prefix + "/" + p, nil + case kindModel: + if len(parts) != 4 { + return bad("is not models///") + } + if strings.HasPrefix(parts[1], "_") { + return bad("names a model starting with an underscore, which the mirror keeps for itself") + } + return strings.Join(parts[1:], "/"), nil + } + return bad("is not under releases/, engines/ or models/") +} + +// EngineBase is where an engine version's files are on a mirror whose files +// are served at mirrorURL. +func EngineBase(mirrorURL, engine, version string) string { + return strings.TrimSuffix(mirrorURL, "/") + "/" + Prefix + "/" + kindEngine + "/" + engine + "/" + version +} + +// ReleasesBase is where llmgw's releases are on a mirror whose files are +// served at mirrorURL: what `releases` is on an offline cluster. +func ReleasesBase(mirrorURL string) string { + return strings.TrimSuffix(mirrorURL, "/") + "/" + Prefix + "/" + kindRelease +} + +// ErrNoDigest is an import or trust given no sha256 to check the bundle by. +var ErrNoDigest = errors.New("give the bundle's sha256, as 'llmgw bundle create' printed it, " + + "so what was carried is known to be what was made; or say it is to be taken unchecked") + +// checkDigest compares a manifest with the digest it was given. Unchecked +// takes it as it is, said by the caller. +func checkDigest(raw []byte, want string, unchecked bool) error { + want = strings.ToLower(strings.TrimSpace(want)) + switch { + case want == "" && unchecked: + return nil + case want == "": + return ErrNoDigest + case Digest(raw) != want: + return fmt.Errorf("the bundle's manifest is %s, not the %s it was given as: "+ + "it is not the bundle that was made, or the digest was mistyped", Digest(raw)[:16], want) + } + return nil +} diff --git a/internal/bundle/bundle_test.go b/internal/bundle/bundle_test.go new file mode 100644 index 0000000..dc6e3a1 --- /dev/null +++ b/internal/bundle/bundle_test.go @@ -0,0 +1,365 @@ +package bundle + +import ( + "archive/tar" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" +) + +func hexSum(b []byte) string { + s := sha256.Sum256(b) + return hex.EncodeToString(s[:]) +} + +// internet is everything a bundle is made from: a release server, GitHub +// and Hugging Face, all fakes. +type internet struct { + srv *httptest.Server + files map[string][]byte + digests map[string]string // GitHub's, by asset + priv string + weights []byte + hfDigest string +} + +func newInternet(t *testing.T) *internet { + t.Helper() + pub, priv, err := release.GenerateKey() + if err != nil { + t.Fatal(err) + } + t.Setenv("LLMGW_RELEASE_KEY", pub) + in := &internet{files: map[string][]byte{}, digests: map[string]string{}, priv: priv} + + // llmgw v1.2.3 for both architectures, signed. + var sums strings.Builder + for _, arch := range []string{"amd64", "arm64"} { + name := release.AssetName("v1.2.3", "linux", arch) + body := []byte("llmgw for " + arch) + in.files["/releases/v1.2.3/"+name] = body + sums.WriteString(hexSum(body) + " " + name + "\n") + } + in.files["/releases/v1.2.3/checksums.txt"] = []byte(sums.String()) + sig, _ := release.Sign(priv, []byte(sums.String())) + in.files["/releases/v1.2.3/"+release.SignatureName] = []byte(sig) + + // llama.cpp b1: every build there is for amd64. + builds, err := provision.Builds("llamacpp", "b1", "amd64") + if err != nil { + t.Fatal(err) + } + for _, b := range builds { + for _, a := range b.Assets { + body := []byte("archive " + a) + in.files["/gh/ggml-org/llama.cpp/releases/download/b1/"+a] = body + in.digests[a] = "sha256:" + hexSum(body) + } + } + + // A model on Hugging Face, with the checksum the registry publishes. + in.weights = []byte("GGUF weights") + in.hfDigest = hexSum(in.weights) + in.files["/Org/Model-GGUF/resolve/abc123/model-q4.gguf"] = in.weights + + in.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/api/repos/ggml-org/llama.cpp/releases/tags/b1"): + var out struct { + Assets []map[string]string `json:"assets"` + } + for name, d := range in.digests { + out.Assets = append(out.Assets, map[string]string{"name": name, "digest": d}) + } + _ = json.NewEncoder(w).Encode(out) + case r.URL.Path == "/api/models/Org/Model-GGUF": + _, _ = w.Write([]byte(`{"id":"Org/Model-GGUF","sha":"abc123"}`)) + case r.URL.Path == "/api/models/Org/Model-GGUF/tree/abc123": + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"type": "file", "path": "model-q4.gguf", "size": 133, + "lfs": map[string]any{"oid": in.hfDigest, "size": len(in.weights)}}, + }) + default: + b, ok := in.files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(b) + } + })) + t.Cleanup(in.srv.Close) + return in +} + +func (in *internet) options(out string) CreateOptions { + return CreateOptions{ + Out: out, Release: "v1.2.3", Arches: []string{"amd64"}, + Releases: in.srv.URL + "/releases", + Engines: []EngineSpec{{Engine: "llamacpp", Version: "b1", Variants: []string{"cpu", "cuda-12.8"}}}, + Models: []ModelSpec{{Name: "starter", Version: "2026-10", Source: "hf:Org/Model-GGUF/model-q4.gguf"}}, + HTTP: in.srv.Client(), + GitHubAPI: in.srv.URL + "/api", GitHubDownload: in.srv.URL + "/gh", Registry: in.srv.URL, + } +} + +func TestABundleCarriesWhatTheClusterWouldFetchAndAMirrorTakesItChecked(t *testing.T) { + in := newInternet(t) + out := filepath.Join(t.TempDir(), "bundle") + m, digest, err := Create(t.Context(), in.options(out)) + if err != nil { + t.Fatal(err) + } + var paths []string + for _, f := range m.Files { + paths = append(paths, f.Path) + } + for _, want := range []string{ + "releases/v1.2.3/checksums.txt", "releases/v1.2.3/" + release.SignatureName, + "releases/v1.2.3/llmgw-v1.2.3-linux-amd64", + "engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz", + "engines/llamacpp/b1/llama-b1-bin-ubuntu-cuda-12.8-x64.tar.gz", + "engines/llamacpp/b1/cudart-llama-b1-bin-ubuntu-cuda-12.8-x64.tar.gz", + "models/starter/2026-10/model-q4.gguf", + } { + if !strings.Contains(strings.Join(paths, "\n"), want) { + t.Errorf("the bundle lacks %s:\n%s", want, strings.Join(paths, "\n")) + } + } + if strings.Contains(strings.Join(paths, "\n"), "vulkan") { + t.Error("a build not asked for was carried") + } + + // Carried as one file, and imported from it as from the directory. + tarball := out + ".tar" + if err := Pack(out, tarball); err != nil { + t.Fatal(err) + } + for _, from := range []string{tarball, out} { + root := t.TempDir() + res, err := Import(from, ImportOptions{Mirror: root, SHA256: digest}) + if err != nil { + t.Fatalf("importing %s: %v", filepath.Base(from), err) + } + if res.Written != len(m.Files) || res.Digest != digest { + t.Errorf("wrote %d of %d files", res.Written, len(m.Files)) + } + bin, _ := os.ReadFile(filepath.Join(root, "_llmgw/releases/v1.2.3/llmgw-v1.2.3-linux-amd64")) + if string(bin) != "llmgw for amd64" { + t.Errorf("the release is not where releases are fetched from: %q", bin) + } + if _, err := os.Stat(filepath.Join(root, "_llmgw/engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz")); err != nil { + t.Error(err) + } + // The model is the mirror's like any other, with its checksum. + man, err := mirror.ManifestFor(root, "starter", "2026-10") + if err != nil || len(man.Files) != 1 || man.Files[0].SHA256 != in.hfDigest { + t.Errorf("the model is not offered with its checksum: %+v %v", man, err) + } + // And the mirror lists nothing of llmgw's own as a model. + entries, _ := mirror.List(root) + if len(entries) != 1 { + t.Errorf("the mirror lists %v", entries) + } + + // Again: everything is there and the same, so nothing is written. + res, err = Import(from, ImportOptions{Mirror: root, SHA256: digest}) + if err != nil || res.Written != 0 || res.Kept != len(m.Files) { + t.Errorf("a second import: %+v %v", res, err) + } + } + + // Made again into the same directory, a file already there and right is + // not downloaded: only the checksums are asked for again. + for p := range in.files { + if !strings.HasSuffix(p, "checksums.txt") && !strings.HasSuffix(p, release.SignatureName) { + delete(in.files, p) + } + } + _, again, err := Create(t.Context(), in.options(out)) + if err != nil { + t.Fatalf("made again from what was there: %v", err) + } + var m2 Manifest + raw, _ := os.ReadFile(filepath.Join(out, ManifestName)) + _ = json.Unmarshal(raw, &m2) + if again != Digest(raw) || len(m2.Files) != len(m.Files) { + t.Errorf("made again: %d files, digest %s", len(m2.Files), again) + } +} + +func TestTheControlPlaneTrustsTheBundlesEngineChecksumsAndNodesAreToldThem(t *testing.T) { + in := newInternet(t) + out := filepath.Join(t.TempDir(), "bundle") + _, digest, err := Create(t.Context(), in.options(out)) + if err != nil { + t.Fatal(err) + } + file := filepath.Join(t.TempDir(), "offline.sha256") + if _, _, err := Trust(out, file, "", false); err == nil { + t.Fatal("a bundle was trusted with no digest to check it by") + } + if _, _, err := Trust(out, file, strings.Repeat("0", 64), false); err == nil { + t.Fatal("a bundle was trusted against the wrong digest") + } + if _, n, err := Trust(out, file, digest, false); err != nil || n != 3 { + t.Fatalf("trusted %d: %v", n, err) + } + + tr := NewTrusted(file) + got := tr.For("llamacpp", "b1") + if got["llama-b1-bin-ubuntu-x64.tar.gz"] != strings.TrimPrefix(in.digests["llama-b1-bin-ubuntu-x64.tar.gz"], "sha256:") { + t.Errorf("the control plane holds %v", got) + } + if len(tr.For("llamacpp", "b2")) != 0 { + t.Error("checksums were given for a version no bundle carried") + } + + // A second bundle adds to what is trusted, and the control plane sees + // it without a restart. + m2 := Manifest{Format: Format, Files: []File{{Path: "engines/llamacpp/b2/llama-b2-bin-ubuntu-x64.tar.gz", Bytes: 1, SHA256: strings.Repeat("b", 64)}}} + if n, err := TrustManifest(m2, file); err != nil || n != 4 { + t.Fatalf("trusted %d: %v", n, err) + } + if tr.For("llamacpp", "b2")["llama-b2-bin-ubuntu-x64.tar.gz"] != strings.Repeat("b", 64) { + t.Error("a bundle trusted while running was not seen") + } + if len(tr.For("llamacpp", "b1")) != 3 { + t.Error("the first bundle's checksums were lost") + } +} + +func TestADamagedBundlePutsNothingInTheMirror(t *testing.T) { + in := newInternet(t) + out := filepath.Join(t.TempDir(), "bundle") + _, digest, err := Create(t.Context(), in.options(out)) + if err != nil { + t.Fatal(err) + } + // The wrong digest, and none at all, are refused before anything is read. + root := t.TempDir() + if _, err := Import(out, ImportOptions{Mirror: root, SHA256: strings.Repeat("0", 64)}); err == nil { + t.Error("a bundle not matching its digest was imported") + } + if _, err := Import(out, ImportOptions{Mirror: root}); err == nil { + t.Error("a bundle was imported with no digest") + } + if _, err := Import(out, ImportOptions{Mirror: root, Unchecked: true}); err != nil { + t.Errorf("a bundle taken unchecked, as asked: %v", err) + } + + // An engine changed on the way. + engine := filepath.Join(out, "engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz") + good, _ := os.ReadFile(engine) + _ = os.WriteFile(engine, []byte("swapped"), 0o644) + root = t.TempDir() + if _, err := Import(out, ImportOptions{Mirror: root, SHA256: digest}); err == nil || !strings.Contains(err.Error(), "damaged") { + t.Errorf("a changed engine: %v", err) + } + if entries := walk(t, root); len(entries) > 0 { + t.Errorf("a damaged bundle left %v in the mirror", entries) + } + _ = os.WriteFile(engine, good, 0o644) + + // A release whose checksums the project did not sign, with a manifest + // made to agree: what the manifest says is not enough for a binary. + _, other, _ := release.GenerateKey() + sums, _ := os.ReadFile(filepath.Join(out, "releases/v1.2.3/checksums.txt")) + forged, _ := release.Sign(other, sums) + _ = os.WriteFile(filepath.Join(out, "releases/v1.2.3/"+release.SignatureName), []byte(forged), 0o644) + raw := remanifest(t, out) + root = t.TempDir() + if _, err := Import(out, ImportOptions{Mirror: root, SHA256: Digest(raw)}); err == nil || !strings.Contains(err.Error(), "not signed") { + t.Errorf("a release not signed by the project: %v", err) + } + if entries := walk(t, root); len(entries) > 0 { + t.Errorf("an unsigned release left %v in the mirror", entries) + } +} + +func TestAVersionAlreadyInTheMirrorIsNeverReplaced(t *testing.T) { + in := newInternet(t) + out := filepath.Join(t.TempDir(), "bundle") + _, digest, err := Create(t.Context(), in.options(out)) + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + _ = os.MkdirAll(filepath.Join(root, "starter", "2026-10"), 0o755) + _ = os.WriteFile(filepath.Join(root, "starter", "2026-10", "model-q4.gguf"), []byte("other weights"), 0o644) + if _, err := Import(out, ImportOptions{Mirror: root, SHA256: digest}); err == nil || !strings.Contains(err.Error(), "never replaced") { + t.Fatalf("a different file under the same version: %v", err) + } + if b, _ := os.ReadFile(filepath.Join(root, "starter", "2026-10", "model-q4.gguf")); string(b) != "other weights" { + t.Error("the mirror's file was replaced") + } +} + +func TestABundleCannotWriteOutsideTheMirror(t *testing.T) { + for _, p := range []string{ + "../etc/cron.d/x", "models/../../x/y/z", "models/a/b/../c", "/abs/olute/path/x", + "engines/llamacpp/b1", "releases/latest/llmgw", "models/_llmgw/releases/x", + "models/.hidden/v/f", "other/a/b/c", + } { + if _, err := Destination(p); err == nil { + t.Errorf("%q was given a destination", p) + } + } + if d, err := Destination("models/m/v/f.gguf"); err != nil || d != "m/v/f.gguf" { + t.Errorf("a model: %q %v", d, err) + } + + // A tar carrying a file its manifest does not list is refused. + dir := t.TempDir() + raw, _ := json.Marshal(Manifest{Format: Format, Files: []File{}}) + tarball := filepath.Join(dir, "b.tar") + f, _ := os.Create(tarball) + tw := tar.NewWriter(f) + _ = tw.WriteHeader(&tar.Header{Name: ManifestName, Mode: 0o644, Size: int64(len(raw)), Typeflag: tar.TypeReg}) + _, _ = tw.Write(raw) + _ = tw.WriteHeader(&tar.Header{Name: "models/m/v/extra", Mode: 0o644, Size: 1, Typeflag: tar.TypeReg}) + _, _ = tw.Write([]byte("x")) + _ = tw.Close() + _ = f.Close() + if _, err := Import(tarball, ImportOptions{Mirror: t.TempDir(), SHA256: Digest(raw)}); err == nil { + t.Error("a file the manifest does not list was imported") + } +} + +// remanifest rewrites a bundle's manifest to agree with its files, as +// someone changing a bundle on the way would. +func remanifest(t *testing.T, dir string) []byte { + t.Helper() + raw, _ := os.ReadFile(filepath.Join(dir, ManifestName)) + var m Manifest + _ = json.Unmarshal(raw, &m) + for i, f := range m.Files { + b, _ := os.ReadFile(filepath.Join(dir, f.Path)) + m.Files[i].SHA256, m.Files[i].Bytes = hexSum(b), int64(len(b)) + } + raw, _ = json.MarshalIndent(m, "", " ") + _ = os.WriteFile(filepath.Join(dir, ManifestName), raw, 0o644) + return raw +} + +func walk(t *testing.T, root string) []string { + t.Helper() + var out []string + _ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error { + if err == nil && !info.IsDir() { + out = append(out, p) + } + return nil + }) + return out +} diff --git a/internal/bundle/chown_other.go b/internal/bundle/chown_other.go new file mode 100644 index 0000000..7eba72a --- /dev/null +++ b/internal/bundle/chown_other.go @@ -0,0 +1,6 @@ +//go:build !unix + +package bundle + +// chownLike does nothing where files have no Unix owner. +func chownLike(string, string) {} diff --git a/internal/bundle/chown_unix.go b/internal/bundle/chown_unix.go new file mode 100644 index 0000000..390a5fe --- /dev/null +++ b/internal/bundle/chown_unix.go @@ -0,0 +1,26 @@ +//go:build unix + +package bundle + +import ( + "os" + "syscall" +) + +// chownLike gives p the owner of root, when this runs as root: the mirror +// runs as its own user, and a version it cannot remove is one `mirror +// remove` and the admin page fail on. +func chownLike(root, p string) { + if os.Geteuid() != 0 { + return + } + info, err := os.Stat(root) + if err != nil { + return + } + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return + } + _ = os.Lchown(p, int(st.Uid), int(st.Gid)) +} diff --git a/internal/bundle/create.go b/internal/bundle/create.go new file mode 100644 index 0000000..a377c64 --- /dev/null +++ b/internal/bundle/create.go @@ -0,0 +1,439 @@ +package bundle + +import ( + "archive/tar" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "slices" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" +) + +// EngineSpec is an engine version to carry, and which of its builds. +type EngineSpec struct { + Engine, Version string + // Variants are the builds to carry ("cpu", "cuda-13.3"); empty is every + // one, which is what a site whose nodes are not all known wants. + Variants []string +} + +// ModelSpec is a model file to carry, and where the mirror will hold it. +type ModelSpec struct { + Name, Version string + // Source is hf:owner/repo/path/in/repo, or an http(s) URL. + Source string +} + +// CreateOptions say what goes into a bundle. +type CreateOptions struct { + // Out is the directory the bundle is made in. A bundle made there + // before is added to: a file already there with the right checksum is + // not downloaded again. + Out string + // Release is llmgw's release to carry, v0.10.4; empty carries none. + Release string + // Arches are the architectures, of the releases and of the engines: + // amd64, arm64. + Arches []string + // Releases is where release files are published. + Releases string + Engines []EngineSpec + Models []ModelSpec + + // HTTP fetches everything; nil is a client with no overall timeout, + // because a model file is tens of gigabytes. + HTTP *http.Client + // GitHubAPI, GitHubDownload and Registry default to GitHub and Hugging + // Face; tests point them at fakes. + GitHubAPI, GitHubDownload, Registry string + // Log hears what is happening, a line at a time. + Log func(format string, args ...any) +} + +// Create downloads everything a bundle is asked to carry, checking each +// file as a node online would, and writes its manifest. It returns the +// manifest and its digest: the value to carry to the site separately and +// give to the import. +func Create(ctx context.Context, o CreateOptions) (Manifest, string, error) { + c := &creator{o: o, client: o.HTTP} + if c.client == nil { + c.client = &http.Client{} + } + if len(o.Arches) == 0 { + return Manifest{}, "", errors.New("give at least one architecture: amd64, arm64") + } + for _, a := range o.Arches { + if a != "amd64" && a != "arm64" { + return Manifest{}, "", fmt.Errorf("%q is not an architecture llmgw is built for: amd64 or arm64", a) + } + } + if err := os.MkdirAll(o.Out, 0o755); err != nil { + return Manifest{}, "", err + } + c.m = Manifest{Format: Format, Created: time.Now().UTC().Truncate(time.Second), MadeBy: version.Version} + + if o.Release != "" { + if err := c.release(ctx); err != nil { + return Manifest{}, "", err + } + } + for _, e := range o.Engines { + if err := c.engine(ctx, e); err != nil { + return Manifest{}, "", err + } + } + for _, m := range o.Models { + if err := c.model(ctx, m); err != nil { + return Manifest{}, "", err + } + } + raw, err := json.MarshalIndent(c.m, "", " ") + if err != nil { + return Manifest{}, "", err + } + raw = append(raw, '\n') + if err := os.WriteFile(filepath.Join(o.Out, ManifestName), raw, 0o644); err != nil { //nolint:gosec // a list of checksums + return Manifest{}, "", err + } + return c.m, Digest(raw), nil +} + +type creator struct { + o CreateOptions + client *http.Client + m Manifest +} + +func (c *creator) log(format string, args ...any) { + if c.o.Log != nil { + c.o.Log(format, args...) + } +} + +// release carries llmgw's release: its checksums and their signature, +// checked against the key this build trusts, and a binary per +// architecture checked against them. +func (c *creator) release(ctx context.Context) error { + v := c.o.Release + base := c.o.Releases + if base == "" { + base = release.DefaultBase + } + c.log("llmgw %s: its checksums and their signature", v) + sums, err := release.FetchChecksums(ctx, c.client, base, v) + if err != nil { + return fmt.Errorf("llmgw %s: %w", v, err) + } + dir := kindRelease + "/" + v + "/" + if err := c.writeSmall(dir+"checksums.txt", sums.Text); err != nil { + return err + } + if sums.Signature != "" { + if err := c.writeSmall(dir+release.SignatureName, []byte(sums.Signature)); err != nil { + return err + } + } + root := strings.TrimSuffix(base, "/") + "/" + v + "/" + for _, arch := range c.o.Arches { + asset := release.AssetName(v, "linux", arch) + want, err := sums.For(asset) + if err != nil { + return fmt.Errorf("llmgw %s: %w", v, err) + } + if err := c.fetch(ctx, root+asset, dir+asset, want); err != nil { + return err + } + } + c.m.Releases = append(c.m.Releases, Release{Version: v, Arches: slices.Clone(c.o.Arches)}) + return nil +} + +// engine carries an engine version's builds for each architecture, checked +// against the checksums its publisher gives — the same the nodes ask for +// online. +func (c *creator) engine(ctx context.Context, e EngineSpec) error { + in := &provision.Installer{HTTP: c.client, APIBase: c.o.GitHubAPI, DownloadBase: c.o.GitHubDownload} + c.log("%s %s: the checksums its publisher gives", e.Engine, e.Version) + sums, url, err := in.Published(ctx, e.Engine, e.Version) + if err != nil { + return fmt.Errorf("%s %s: %w", e.Engine, e.Version, err) + } + var variants []string + done := map[string]bool{} + for _, arch := range c.o.Arches { + builds, err := provision.Builds(e.Engine, e.Version, arch) + if err != nil { + return err + } + var names []string + for _, b := range builds { + names = append(names, b.Variant) + if len(e.Variants) > 0 && !slices.Contains(e.Variants, b.Variant) { + continue + } + if !slices.Contains(variants, b.Variant) { + variants = append(variants, b.Variant) + } + for _, asset := range b.Assets { + if done[asset] { + continue + } + want, ok := sums[asset] + if !ok { + return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be carried checked", e.Engine, e.Version, asset) + } + if err := c.fetch(ctx, url(asset), kindEngine+"/"+e.Engine+"/"+e.Version+"/"+asset, want); err != nil { + return err + } + done[asset] = true + } + } + for _, v := range e.Variants { + if !slices.Contains(names, v) && !anyArchHas(e, v, c.o.Arches) { + return fmt.Errorf("%s has no build %q; it has %s", e.Engine, v, strings.Join(names, ", ")) + } + } + } + c.m.Engines = append(c.m.Engines, Engine{Engine: e.Engine, Version: e.Version, + Arches: slices.Clone(c.o.Arches), Variants: variants}) + return nil +} + +// anyArchHas says whether a variant is a build on any of the architectures: +// CUDA 12.8 is published for x64 alone, and asking for it with arm64 too is +// not a mistake. +func anyArchHas(e EngineSpec, variant string, arches []string) bool { + for _, a := range arches { + bs, _ := provision.Builds(e.Engine, e.Version, a) + for _, b := range bs { + if b.Variant == variant { + return true + } + } + } + return false +} + +// model carries a model file, checked against the checksum its registry +// publishes when it publishes one, at the commit the registry had when it +// was asked — so the bundle says exactly which weights it carries. +func (c *creator) model(ctx context.Context, s ModelSpec) error { + for _, n := range []string{s.Name, s.Version} { + if err := mirror.ValidName(n); err != nil { + return fmt.Errorf("model %s/%s: %w", s.Name, s.Version, err) + } + } + if strings.HasPrefix(s.Name, "_") { + return fmt.Errorf("model %q: a name starting with an underscore is the mirror's own", s.Name) + } + var url, want, rev string + switch { + case strings.HasPrefix(s.Source, "hf:"): + parts := strings.SplitN(strings.TrimPrefix(s.Source, "hf:"), "/", 3) + if len(parts) != 3 || parts[2] == "" { + return fmt.Errorf("%s names a repository, not a file: hf:owner/repo/file", s.Source) + } + repoID, file := parts[0]+"/"+parts[1], parts[2] + reg := hf.New(c.o.Registry, time.Minute) + repo, err := reg.Repo(ctx, repoID) + if err != nil { + return fmt.Errorf("%s: %w", s.Source, err) + } + found := false + for _, f := range repo.Files { + if f.Path == file { + want, found = f.SHA256, true + } + } + if !found { + return fmt.Errorf("%s has no file %s at its latest commit", repoID, file) + } + url, rev = repo.DownloadURL(reg.Base(), file), repo.Revision + case strings.HasPrefix(s.Source, "https://"), strings.HasPrefix(s.Source, "http://"): + url = s.Source + default: + return fmt.Errorf("model source %q: hf:owner/repo/file or an http(s) URL", s.Source) + } + name := path.Base(strings.SplitN(url, "?", 2)[0]) + if err := mirror.ValidName(name); err != nil { + return fmt.Errorf("%s: %w", s.Source, err) + } + if err := c.fetch(ctx, url, kindModel+"/"+s.Name+"/"+s.Version+"/"+name, want); err != nil { + return err + } + c.m.Models = append(c.m.Models, Model{Name: s.Name, Version: s.Version, Source: s.Source, Revision: rev}) + return nil +} + +// writeSmall puts a file the bundle makes itself, rather than downloads. +func (c *creator) writeSmall(rel string, b []byte) error { + dst := filepath.Join(c.o.Out, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + if err := os.WriteFile(dst, b, 0o644); err != nil { //nolint:gosec // published files + return err + } + sum := sha256.Sum256(b) + c.add(File{Path: rel, Bytes: int64(len(b)), SHA256: hex.EncodeToString(sum[:])}) + return nil +} + +func (c *creator) add(f File) { + for i := range c.m.Files { + if c.m.Files[i].Path == f.Path { + c.m.Files[i] = f + return + } + } + c.m.Files = append(c.m.Files, f) +} + +// fetch downloads url into the bundle at rel, checked against want when +// there is one; a file already there that matches is kept. +func (c *creator) fetch(ctx context.Context, url, rel, want string) error { + dst := filepath.Join(c.o.Out, filepath.FromSlash(rel)) + if want != "" { + if got, err := fileSHA256(dst); err == nil && got == want { + info, _ := os.Stat(dst) + c.log(" have %s", rel) + c.add(File{Path: rel, Bytes: info.Size(), SHA256: want, Source: url}) + return nil + } + } + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("fetching %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("fetching %s: %s", url, resp.Status) + } + if resp.ContentLength > 0 { + c.log(" fetching %s (%d MB)", rel, resp.ContentLength>>20) + } else { + c.log(" fetching %s", rel) + } + part := dst + partSuffix + f, err := os.Create(part) //nolint:gosec // inside the bundle + if err != nil { + return err + } + h := sha256.New() + n, err := io.Copy(io.MultiWriter(f, h), &progressReader{r: resp.Body, total: resp.ContentLength, log: c.log, name: rel}) + if cerr := f.Close(); err == nil { + err = cerr + } + if err == nil && resp.ContentLength > 0 && n != resp.ContentLength { + err = fmt.Errorf("got %d bytes of %d", n, resp.ContentLength) + } + if err != nil { + _ = os.Remove(part) + return fmt.Errorf("fetching %s: %w", url, err) + } + got := hex.EncodeToString(h.Sum(nil)) + if want != "" && got != want { + _ = os.Remove(part) + return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)", url, got[:12], want[:12]) + } + if err := os.Rename(part, dst); err != nil { + return err + } + c.add(File{Path: rel, Bytes: n, SHA256: got, Source: url}) + return nil +} + +// progressReader says how far a long download has got, every few seconds. +type progressReader struct { + r io.Reader + total int64 + done int64 + last time.Time + log func(string, ...any) + name string +} + +func (p *progressReader) Read(b []byte) (int, error) { + n, err := p.r.Read(b) + p.done += int64(n) + if p.total > 0 && time.Since(p.last) > 10*time.Second { + if !p.last.IsZero() { + p.log(" %d%% of %s", p.done*100/p.total, p.name) + } + p.last = time.Now() + } + return n, err +} + +// Pack writes a bundle directory as one tar file, manifest first, so it can +// be carried as one file and imported as a stream. Weights do not compress, +// so it is not compressed. +func Pack(dir, out string) error { + src, m, err := open(dir) + if err != nil { + return err + } + defer src.close() //nolint:errcheck // read only + f, err := os.Create(out + partSuffix) //nolint:gosec // the operator's path + if err != nil { + return err + } + defer os.Remove(out + partSuffix) //nolint:errcheck // gone once renamed + tw := tar.NewWriter(f) + now := time.Now() + raw := src.manifest() + if err := tw.WriteHeader(&tar.Header{Name: ManifestName, Mode: 0o644, Size: int64(len(raw)), ModTime: now, Typeflag: tar.TypeReg}); err != nil { + f.Close() + return err + } + if _, err := tw.Write(raw); err != nil { + f.Close() + return err + } + for _, mf := range m.Files { + in, err := os.Open(filepath.Join(dir, filepath.FromSlash(mf.Path))) //nolint:gosec // a checked path in the bundle + if err != nil { + f.Close() + return err + } + err = tw.WriteHeader(&tar.Header{Name: mf.Path, Mode: 0o644, Size: mf.Bytes, ModTime: now, Typeflag: tar.TypeReg}) + if err == nil { + _, err = io.Copy(tw, in) + } + in.Close() + if err != nil { + f.Close() + return fmt.Errorf("packing %s: %w", mf.Path, err) + } + } + if err := tw.Close(); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(out+partSuffix, out) +} diff --git a/internal/bundle/import.go b/internal/bundle/import.go new file mode 100644 index 0000000..a3d9932 --- /dev/null +++ b/internal/bundle/import.go @@ -0,0 +1,335 @@ +package bundle + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "maps" + "os" + "path" + "path/filepath" + "slices" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" +) + +// ImportOptions say where a bundle goes and how it is known to be the one +// that was made. +type ImportOptions struct { + // Mirror is the mirror's root directory, /srv/models. Empty checks the + // bundle and writes nothing. + Mirror string + // SHA256 is the manifest's digest as `bundle create` printed it. + // Unchecked takes the bundle without one, said by whoever runs it. + SHA256 string + Unchecked bool + // Progress hears each file as it is done: kept is one the mirror + // already had, the same. + Progress func(f File, kept bool) +} + +// Result is what an import did. +type Result struct { + Manifest Manifest + // Digest is the manifest's sha256. + Digest string + // Written and Kept count files: put in the mirror, and already there. + Written, Kept int + Bytes int64 +} + +// partSuffix marks a file still being written; the mirror lists none. +const partSuffix = ".part" + +// Import checks every file of a bundle and puts it in a mirror. +// +// Nothing is in place until everything is checked. Each file is written +// beside where it goes, under a name the mirror does not serve, and checked +// against the manifest as it is written; each release's checksums are +// checked against the project's signing key and its binaries against them. +// Only then is each renamed into place — so a bundle damaged anywhere +// leaves the mirror as it was. +// +// A file the mirror already has is kept when it is the same, and refused +// when it is not: a version in the mirror is never replaced, because that +// is what makes going back to it possible. +func Import(p string, o ImportOptions) (Result, error) { + src, m, err := open(p) + if err != nil { + return Result{}, err + } + defer src.close() //nolint:errcheck // read only + res := Result{Manifest: m, Digest: Digest(src.manifest())} + if err := checkDigest(src.manifest(), o.SHA256, o.Unchecked); err != nil { + return res, err + } + + byPath := map[string]File{} + dest := map[string]string{} + keep := map[string]bool{} + for _, f := range m.Files { + byPath[f.Path] = f + if o.Mirror == "" { + continue + } + rel, _ := Destination(f.Path) // checked when the manifest was read + dest[f.Path] = filepath.Join(o.Mirror, filepath.FromSlash(rel)) + same, err := alreadyHas(dest[f.Path], f) + if err != nil { + return res, err + } + keep[f.Path] = same + } + + // Each release's checksums are kept as they pass, to be checked against + // the signing key once the whole release has been read. + small := map[string][]byte{} + var written []string + defer func() { + // Whatever was staged and not put in place goes. + for _, p := range written { + _ = os.Remove(dest[p] + partSuffix) + } + }() + seen := map[string]bool{} + err = src.each(func(name string, r io.Reader) error { + f, ok := byPath[name] + if !ok { + return fmt.Errorf("the bundle holds %s, which its manifest does not list", name) + } + if seen[name] { + return fmt.Errorf("the bundle holds %s twice", name) + } + seen[name] = true + var w io.Writer = io.Discard + var out *os.File + if o.Mirror != "" && !keep[name] { + if err := mkdirsLike(o.Mirror, filepath.Dir(dest[name])); err != nil { + return err + } + out, err = os.OpenFile(dest[name]+partSuffix, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) //nolint:gosec // a checked path under the mirror + if err != nil { + return err + } + written = append(written, name) + w = out + } + if keep[name] { + r = eofReader{} + } + var buf bytes.Buffer + if isReleaseSums(name) { + w = io.MultiWriter(w, &limited{w: &buf, n: 1 << 20}) + } + h := sha256.New() + n, err := io.Copy(io.MultiWriter(w, h), r) + if out != nil { + if cerr := out.Close(); err == nil { + err = cerr + } + } + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + if keep[name] { + if isReleaseSums(name) { + b, err := os.ReadFile(dest[name]) + if err != nil { + return err + } + small[name] = b + } + res.Kept++ + if o.Progress != nil { + o.Progress(f, true) + } + return nil + } + if got := hex.EncodeToString(h.Sum(nil)); got != f.SHA256 || n != f.Bytes { + return fmt.Errorf("%s is not what the bundle was made with (sha256 %s, expected %s): "+ + "the bundle was damaged on the way; nothing was put in the mirror", name, got[:12], f.SHA256[:12]) + } + if isReleaseSums(name) { + small[name] = buf.Bytes() + } + res.Written++ + res.Bytes += n + if o.Progress != nil { + o.Progress(f, false) + } + return nil + }) + if err != nil { + return res, err + } + for _, f := range m.Files { + if !seen[f.Path] { + return res, fmt.Errorf("the bundle lacks %s, which its manifest lists", f.Path) + } + } + if err := checkReleases(m, small); err != nil { + return res, err + } + if o.Mirror == "" { + return res, nil + } + + for _, name := range written { + d := dest[name] + f := byPath[name] + if strings.HasPrefix(name, kindModel+"/") { + // Beside the file before it is in place, as a download into the + // mirror does, so a node is never offered it without its checksum. + if err := mirror.Record(d, f.SHA256, f.Source); err != nil { + return res, err + } + } + if err := os.Rename(d+partSuffix, d); err != nil { + return res, err + } + chownLike(o.Mirror, d) + if strings.HasPrefix(name, kindModel+"/") { + dir, base := filepath.Split(d) + chownLike(o.Mirror, filepath.Join(dir, "."+base+".sha256")) + chownLike(o.Mirror, filepath.Join(dir, "."+base+".source")) + } + } + written = nil + return res, nil +} + +// alreadyHas says whether the mirror has this file already, the same; an +// error when it has a different one there. +func alreadyHas(dst string, f File) (bool, error) { + info, err := os.Stat(dst) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + sum := mirror.Recorded(dst) + if sum == "" || info.Size() != f.Bytes { + if sum, err = fileSHA256(dst); err != nil { + return false, err + } + } + if sum != f.SHA256 || info.Size() != f.Bytes { + return false, fmt.Errorf("the mirror already has %s, and it is not the bundle's: "+ + "a version in the mirror is never replaced; give the model another version in a new bundle", dst) + } + return true, nil +} + +func isReleaseSums(name string) bool { + if !strings.HasPrefix(name, kindRelease+"/") { + return false + } + b := path.Base(name) + return b == "checksums.txt" || b == release.SignatureName +} + +// checkReleases holds each release's binaries to its checksums, and the +// checksums to the project's signing key — as `node add` and an upgrade +// would online. The manifest's own checksums say the bundle arrived as it +// was made; these say what was made is the project's release. +func checkReleases(m Manifest, small map[string][]byte) error { + byVersion := map[string][]File{} + for _, f := range m.Files { + if strings.HasPrefix(f.Path, kindRelease+"/") { + v := strings.Split(f.Path, "/")[1] + byVersion[v] = append(byVersion[v], f) + } + } + for _, v := range slices.Sorted(maps.Keys(byVersion)) { + root := kindRelease + "/" + v + "/" + text, ok := small[root+"checksums.txt"] + if !ok { + return fmt.Errorf("the bundle's llmgw %s has no checksums.txt", v) + } + c := release.Checksums{Text: text, Signature: string(small[root+release.SignatureName])} + if err := c.Verify(); err != nil { + return fmt.Errorf("llmgw %s in the bundle: %w", v, err) + } + for _, f := range byVersion[v] { + name := path.Base(f.Path) + if name == "checksums.txt" || name == release.SignatureName { + continue + } + want, err := c.For(name) + if err != nil { + return fmt.Errorf("llmgw %s in the bundle: %w", v, err) + } + if want != f.SHA256 { + return fmt.Errorf("%s in the bundle is not the release's (the release gives sha256 %s)", f.Path, want[:12]) + } + } + } + return nil +} + +// mkdirsLike makes dir and every directory above it up to root, each owned +// as root is: the mirror runs as its own user, and removes what it holds. +func mkdirsLike(root, dir string) error { + rel, err := filepath.Rel(root, dir) + if err != nil || strings.HasPrefix(rel, "..") { + return fmt.Errorf("%s is outside the mirror at %s", dir, root) + } + cur := root + for _, e := range strings.Split(rel, string(filepath.Separator)) { + if e == "." { + continue + } + cur = filepath.Join(cur, e) + if _, err := os.Stat(cur); err == nil { + continue + } + if err := os.Mkdir(cur, 0o755); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + chownLike(root, cur) + } + return nil +} + +func fileSHA256(p string) (string, error) { + f, err := os.Open(p) //nolint:gosec // a file in the mirror + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// eofReader is a file passed over: a tar moves past it on its own. +type eofReader struct{} + +func (eofReader) Read([]byte) (int, error) { return 0, io.EOF } + +// limited keeps at most n bytes and quietly drops the rest: a checksums +// file larger than that is refused by its signature, not by a write error. +type limited struct { + w io.Writer + n int64 +} + +func (l *limited) Write(p []byte) (int, error) { + if l.n <= 0 { + return len(p), nil + } + k := min(int64(len(p)), l.n) + l.n -= k + if _, err := l.w.Write(p[:k]); err != nil { + return 0, err + } + return len(p), nil +} diff --git a/internal/bundle/read.go b/internal/bundle/read.go new file mode 100644 index 0000000..b7b7eb8 --- /dev/null +++ b/internal/bundle/read.go @@ -0,0 +1,157 @@ +package bundle + +import ( + "archive/tar" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// maxManifest bounds the manifest read into memory. +const maxManifest = 16 << 20 + +// source is a bundle being read: a directory, or the same packed as a tar +// with the manifest first, read as a stream so tens of gigabytes of weights +// are never unpacked twice. +type source interface { + // manifest is the manifest's bytes, as they are in the bundle. + manifest() []byte + // each hands over every file, in the order the bundle has them. A file + // fn does not read is passed over. + each(fn func(path string, r io.Reader) error) error + close() error +} + +// open reads a bundle's manifest, from a directory or a tar. +func open(p string) (source, Manifest, error) { + info, err := os.Stat(p) + if err != nil { + return nil, Manifest{}, fmt.Errorf("the bundle: %w", err) + } + var src source + if info.IsDir() { + src, err = openDir(p) + } else { + src, err = openTar(p) + } + if err != nil { + return nil, Manifest{}, err + } + m, err := parseManifest(src.manifest()) + if err != nil { + _ = src.close() + return nil, Manifest{}, err + } + if d, ok := src.(*dirSource); ok { + d.files = m.Files + } + return src, m, nil +} + +// ReadManifest reads a bundle's manifest and its digest, without reading +// its files. +func ReadManifest(p string) (Manifest, string, error) { + src, m, err := open(p) + if err != nil { + return Manifest{}, "", err + } + defer src.close() //nolint:errcheck // read only + return m, Digest(src.manifest()), nil +} + +type dirSource struct { + dir string + raw []byte + files []File +} + +func openDir(dir string) (*dirSource, error) { + raw, err := readLimited(filepath.Join(dir, ManifestName)) + if err != nil { + return nil, fmt.Errorf("%s is not a bundle: %w", dir, err) + } + return &dirSource{dir: dir, raw: raw}, nil +} + +func (d *dirSource) manifest() []byte { return d.raw } +func (d *dirSource) close() error { return nil } + +func (d *dirSource) each(fn func(string, io.Reader) error) error { + for _, f := range d.files { + // The path was checked when the manifest was read: plain names + // under the bundle's three directories. + in, err := os.Open(filepath.Join(d.dir, filepath.FromSlash(f.Path))) + if err != nil { + return fmt.Errorf("the bundle lists %s and does not have it: %w", f.Path, err) + } + err = fn(f.Path, in) + in.Close() + if err != nil { + return err + } + } + return nil +} + +type tarSource struct { + f *os.File + tr *tar.Reader + raw []byte +} + +func openTar(p string) (*tarSource, error) { + f, err := os.Open(p) //nolint:gosec // the operator's bundle + if err != nil { + return nil, err + } + tr := tar.NewReader(f) + h, err := tr.Next() + if err != nil || h.Name != ManifestName || h.Typeflag != tar.TypeReg { + f.Close() + return nil, fmt.Errorf("%s is not a bundle: its first entry is not %s", p, ManifestName) + } + raw, err := io.ReadAll(io.LimitReader(tr, maxManifest)) + if err != nil { + f.Close() + return nil, err + } + return &tarSource{f: f, tr: tr, raw: raw}, nil +} + +func (t *tarSource) manifest() []byte { return t.raw } +func (t *tarSource) close() error { return t.f.Close() } + +func (t *tarSource) each(fn func(string, io.Reader) error) error { + for { + h, err := t.tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("reading the bundle: %w", err) + } + if h.Typeflag == tar.TypeDir { + continue + } + if h.Typeflag != tar.TypeReg { + // Links and devices have no place in a bundle; the files it + // lists are all it holds. + return fmt.Errorf("the bundle holds %s, which is not a plain file", h.Name) + } + if err := fn(strings.TrimPrefix(h.Name, "./"), t.tr); err != nil { + return err + } + } +} + +func readLimited(p string) ([]byte, error) { + f, err := os.Open(p) //nolint:gosec // the operator's bundle + if err != nil { + return nil, err + } + defer f.Close() + return io.ReadAll(io.LimitReader(f, maxManifest)) +} diff --git a/internal/bundle/trust.go b/internal/bundle/trust.go new file mode 100644 index 0000000..aa8db1b --- /dev/null +++ b/internal/bundle/trust.go @@ -0,0 +1,132 @@ +package bundle + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "maps" + "os" + "path" + "path/filepath" + "slices" + "strings" + "sync" + "time" +) + +// Trust records a bundle's engine checksums in the file the control plane +// reads them from, beside any recorded before, and says how many it holds +// now. Only the manifest is read: the checksums are what each node holds +// the files it fetches to, so the manifest must be the one that was made — +// which is what the digest says. +func Trust(p, file, digest string, unchecked bool) (Manifest, int, error) { + src, m, err := open(p) + if err != nil { + return Manifest{}, 0, err + } + defer src.close() //nolint:errcheck // read only + if err := checkDigest(src.manifest(), digest, unchecked); err != nil { + return m, 0, err + } + n, err := TrustManifest(m, file) + return m, n, err +} + +// TrustManifest records a manifest's engine checksums, for one already +// checked by an import. +func TrustManifest(m Manifest, file string) (int, error) { + sums, err := readTrusted(file) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 0, err + } + if sums == nil { + sums = map[string]string{} + } + for _, f := range m.Files { + if strings.HasPrefix(f.Path, kindEngine+"/") { + sums[Prefix+"/"+f.Path] = f.SHA256 + } + } + var b strings.Builder + b.WriteString("# Engine files the nodes may install, as the offline bundles they came in\n" + + "# gave them. Kept by 'llmgw bundle import' and 'llmgw bundle trust'.\n") + for _, k := range slices.Sorted(maps.Keys(sums)) { + fmt.Fprintf(&b, "%s %s\n", sums[k], k) + } + if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil { + return 0, err + } + tmp := file + partSuffix + if err := os.WriteFile(tmp, []byte(b.String()), 0o644); err != nil { //nolint:gosec // checksums, for anyone to read + return 0, err + } + return len(sums), os.Rename(tmp, file) +} + +// readTrusted reads a checksums file: " " a line, as +// sha256sum writes them, with paths as the mirror has the files. +func readTrusted(file string) (map[string]string, error) { + b, err := os.ReadFile(file) //nolint:gosec // the control plane's own file + if err != nil { + return nil, err + } + out := map[string]string{} + sc := bufio.NewScanner(bytes.NewReader(b)) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + f := strings.Fields(line) + if len(f) != 2 || len(f[0]) != 64 { + return nil, fmt.Errorf("%s: %q is not a checksum and a file", file, line) + } + out[strings.TrimPrefix(f[1], "*")] = f[0] + } + return out, sc.Err() +} + +// Trusted is the engine checksums a control plane holds, read from their +// file again whenever it changes — so a bundle imported while the control +// plane runs is offered to the next node that asks. +type Trusted struct { + file string + + mu sync.Mutex + mod time.Time + size int64 + sums map[string]string + err error +} + +// NewTrusted reads checksums from file as they are asked for. +func NewTrusted(file string) *Trusted { return &Trusted{file: file} } + +// For is an engine version's files and their checksums, by file name; none +// when no bundle imported here carried it. +func (t *Trusted) For(engine, version string) map[string]string { + t.mu.Lock() + defer t.mu.Unlock() + if info, err := os.Stat(t.file); err != nil { + t.sums, t.err, t.mod, t.size = nil, err, time.Time{}, 0 + } else if !info.ModTime().Equal(t.mod) || info.Size() != t.size { + t.sums, t.err = readTrusted(t.file) + t.mod, t.size = info.ModTime(), info.Size() + } + prefix := Prefix + "/" + kindEngine + "/" + engine + "/" + version + "/" + out := map[string]string{} + for p, sum := range t.sums { + if rest, ok := strings.CutPrefix(p, prefix); ok && path.Base(rest) == rest { + out[rest] = sum + } + } + return out +} + +// Err is why the file could not be read, the last time it was. +func (t *Trusted) Err() error { + t.mu.Lock() + defer t.mu.Unlock() + return t.err +} diff --git a/internal/cli/bundle.go b/internal/cli/bundle.go new file mode 100644 index 0000000..dc13391 --- /dev/null +++ b/internal/cli/bundle.go @@ -0,0 +1,341 @@ +package cli + +import ( + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/bundle" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" +) + +// newBundleCmd carries a cluster's downloads to a site with no internet. +func newBundleCmd() *cobra.Command { + c := &cobra.Command{ + Use: "bundle", + Short: "Carry releases, engines and models to a site with no internet", + Long: "A cluster fetches llmgw's releases, the llama.cpp builds its nodes install,\n" + + "and model files from the internet. A site that has none gets them in a\n" + + "bundle: made where there is internet with 'bundle create', carried in, and\n" + + "put into the cluster's mirror with 'bundle import'. Every file is checked\n" + + "as a node online would check it — releases against the project's signing\n" + + "key, engines against their publisher's checksums — when the bundle is made,\n" + + "when it is imported, and again on the node that installs it.", + } + c.AddCommand(newBundleCreateCmd(), newBundleImportCmd(), newBundleTrustCmd(), newBundleShowCmd()) + return c +} + +func newBundleCreateCmd() *cobra.Command { + var ( + out, rel, releases, registry string + arches, engines, variants []string + models []string + noRelease, noStarter, pack bool + ) + c := &cobra.Command{ + Use: "create", + Short: "Make a bundle, on a machine with internet", + Long: "Downloads into --out, checking each file as it arrives: llmgw's release for\n" + + "each --arch with its signed checksums, every build of each --engine for those\n" + + "architectures (or the --variant builds), and each --model's file from Hugging\n" + + "Face, pinned to the commit it was taken at. The starter model and the\n" + + "guardrails' judge that 'server init' serves are carried unless\n" + + "--no-starter-models.\n\n" + + "Run again with the same --out to add to a bundle: what is there and right is\n" + + "not downloaded again.\n\n" + + "It prints the manifest's sha256. Carry that apart from the bundle — read it\n" + + "out over the phone, write it in the ticket — and give it to 'bundle import':\n" + + "it is what says the bundle that arrived is the one that was made.\n\n" + + "vLLM and SGLang are Python environments from PyPI, and no bundle carries\n" + + "them; offline nodes serve with llama.cpp, or have them installed by hand.", + Example: ` llmgw bundle create --out llmgw-bundle --arch amd64 --tar + llmgw bundle create --out site-b --engine llamacpp=b10991 --variant cpu --variant cuda-13.3 \ + --model qwen2.5-14b-instruct/2026-10=hf:Qwen/Qwen2.5-14B-Instruct-GGUF/qwen2.5-14b-instruct-q4_k_m.gguf`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if out == "" { + return errors.New("--out: the directory to make the bundle in") + } + o := bundle.CreateOptions{Out: out, Arches: arches, Releases: releases, Registry: registry} + switch { + case noRelease: + case rel != "": + o.Release = rel + case release.Tag.MatchString(version.Version): + o.Release = version.Version + default: + return fmt.Errorf("this llmgw is %s, which has no release to carry: give --version v0.x.y, "+ + "or --no-release to carry none", version.Version) + } + if len(engines) == 0 { + engines = []string{"llamacpp=" + config.DefaultEngines["llamacpp"]} + } + for _, e := range engines { + name, v, ok := strings.Cut(e, "=") + if !ok || name == "" || v == "" { + return fmt.Errorf("--engine %q: engine=version, llamacpp=%s", e, config.DefaultEngines["llamacpp"]) + } + o.Engines = append(o.Engines, bundle.EngineSpec{Engine: name, Version: v, Variants: variants}) + } + if !noStarter { + o.Models = append(o.Models, + bundle.ModelSpec{Name: starterName, Version: starterVersion, Source: starterSource}, + bundle.ModelSpec{Name: guardModelName, Version: guardModelVersion, Source: guardModelSource}) + } + for _, m := range models { + spec, err := parseModelSpec(m) + if err != nil { + return err + } + o.Models = append(o.Models, spec) + } + w := cmd.OutOrStdout() + o.Log = func(format string, args ...any) { fmt.Fprintf(w, format+"\n", args...) } + + m, digest, err := bundle.Create(cmd.Context(), o) + if err != nil { + return err + } + where := out + if pack { + where = strings.TrimSuffix(out, "/") + ".tar" + fmt.Fprintf(w, "packing it into %s\n", where) + if err := bundle.Pack(out, where); err != nil { + return err + } + if err := os.RemoveAll(out); err != nil { + return err + } + } + printManifest(w, m) + fmt.Fprintf(w, "\nThe bundle is %s. Its sha256, to carry apart from it:\n\n %s\n\n", where, digest) + fmt.Fprintf(w, "At the site, on the machine with the mirror (and the control plane):\n\n"+ + " sudo llmgw bundle import %s --sha256 %s\n\n"+ + "or, installing a control plane from it:\n\n"+ + " sudo llmgw server init --bundle %s --bundle-sha256 %s\n", where, digest, where, digest) + return nil + }, + } + f := c.Flags() + f.StringVar(&out, "out", "", "the directory to make the bundle in; added to when it holds one already") + f.BoolVar(&pack, "tar", false, "pack the bundle into .tar, one file to carry, and remove the directory") + f.StringVar(&rel, "version", "", "llmgw's release to carry (default: this llmgw's own)") + f.BoolVar(&noRelease, "no-release", false, "carry no llmgw release") + f.StringVar(&releases, "releases", release.DefaultBase, "where llmgw's releases are published") + f.StringArrayVar(&arches, "arch", []string{"amd64"}, "an architecture of the site's machines, amd64 or arm64; repeatable") + f.StringArrayVar(&engines, "engine", nil, "an engine version to carry, engine=version; repeatable (default: llama.cpp at the version this llmgw installs)") + f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3, cuda-12.8, vulkan; repeatable (default: every build)") + f.StringArrayVar(&models, "model", nil, "a model file to carry, name/version=hf:owner/repo/file or name/version=https://...; repeatable") + f.BoolVar(&noStarter, "no-starter-models", false, "leave out the starter model and the guardrails' judge that server init serves") + f.StringVar(®istry, "registry", "", "the model registry, when not the public Hugging Face") + return c +} + +// parseModelSpec reads name/version=source. +func parseModelSpec(s string) (bundle.ModelSpec, error) { + where, source, ok := strings.Cut(s, "=") + name, ver, ok2 := strings.Cut(where, "/") + if !ok || !ok2 || name == "" || ver == "" || source == "" { + return bundle.ModelSpec{}, fmt.Errorf("--model %q: name/version=hf:owner/repo/file", s) + } + return bundle.ModelSpec{Name: name, Version: ver, Source: source}, nil +} + +func newBundleImportCmd() *cobra.Command { + var ( + dir, digest, trust, serverCfg string + unchecked, noTrust bool + ) + c := &cobra.Command{ + Use: "import PATH", + Short: "Put a bundle into this machine's mirror", + Long: "Checks every file of the bundle — against its manifest, and llmgw's release\n" + + "against the project's signing key — and only then puts them into the mirror:\n" + + "releases and engines under _llmgw/, models where the mirror keeps models.\n" + + "A file the mirror has already, the same, is kept; a different one under the\n" + + "same version is refused, because a version is never replaced.\n\n" + + "On a control plane (server.yaml is there) it also records the engines'\n" + + "checksums in offline.checksums, which the nodes check each engine file they\n" + + "fetch against. When the mirror is on another machine, import there, and\n" + + "run 'llmgw bundle trust' with the same bundle on the control plane.\n\n" + + "PATH is the bundle's directory or its .tar.", + Example: ` sudo llmgw bundle import /media/usb/llmgw-bundle.tar --sha256 3f2a…`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + w := cmd.OutOrStdout() + if st, err := os.Stat(dir); err != nil || !st.IsDir() { + return fmt.Errorf("%s is not the mirror's directory: give --dir, or set the mirror up first "+ + "(sudo llmgw mirror init, or server init)", dir) + } + fmt.Fprintf(w, "checking %s and putting it into the mirror at %s\n", args[0], dir) + res, err := bundle.Import(args[0], bundle.ImportOptions{ + Mirror: dir, SHA256: digest, Unchecked: unchecked, + Progress: func(f bundle.File, kept bool) { + if kept { + fmt.Fprintf(w, " kept %s\n", f.Path) + } else { + fmt.Fprintf(w, " ok %s (%s)\n", f.Path, fileSize(f.Bytes)) + } + }, + }) + if err != nil { + return err + } + if unchecked { + fmt.Fprintf(w, "\nTaken unchecked: its manifest's sha256 is %s; compare it with what was made.\n", res.Digest) + } + fmt.Fprintf(w, "\n%d files put in the mirror (%s), %d already there.\n", res.Written, fileSize(res.Bytes), res.Kept) + if noTrust { + return nil + } + cfg, haveCfg := loadServerConfig(serverCfg) + if trust == "" && !haveCfg { + fmt.Fprintf(w, "\nThis machine is not a control plane. On the control plane:\n\n"+ + " sudo llmgw bundle trust %s --sha256 %s\n", args[0], res.Digest) + return nil + } + if trust == "" { + trust = cfg.Offline.ChecksumsFile() + } + n, err := bundle.TrustManifest(res.Manifest, trust) + if err != nil { + return fmt.Errorf("recording the engines' checksums in %s: %w", trust, err) + } + fmt.Fprintf(w, "The control plane trusts %d engine files (%s).\n", n, trust) + offlineHint(w, cfg, haveCfg) + return nil + }, + } + f := c.Flags() + f.StringVar(&dir, "dir", mirrorDir, "the mirror's directory") + f.StringVar(&digest, "sha256", "", "the bundle's sha256, as 'bundle create' printed it") + f.BoolVar(&unchecked, "unchecked", false, "take the bundle without its sha256 (its files are still checked against its manifest)") + f.StringVar(&trust, "trust", "", "where to record the engines' checksums (default: offline.checksums, on a control plane)") + f.BoolVar(&noTrust, "no-trust", false, "do not record the engines' checksums here") + f.StringVar(&serverCfg, "config", "/etc/llmgw/server.yaml", "the control plane's configuration, when this is one") + return c +} + +func newBundleTrustCmd() *cobra.Command { + var digest, file, serverCfg string + var unchecked bool + c := &cobra.Command{ + Use: "trust PATH", + Short: "Have this control plane trust a bundle's engine checksums", + Long: "Records the checksum of every engine file in the bundle in offline.checksums,\n" + + "where the control plane reads what it tells each node a file from the\n" + + "mirror must be. Only the manifest is read, so this is the step for a control\n" + + "plane whose mirror is on another machine: import the bundle there, and run\n" + + "this here with the same bundle and sha256. The control plane sees the new\n" + + "checksums without a restart.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + w := cmd.OutOrStdout() + cfg, haveCfg := loadServerConfig(serverCfg) + if file == "" { + file = cfg.Offline.ChecksumsFile() + } + m, n, err := bundle.Trust(args[0], file, digest, unchecked) + if err != nil { + return err + } + printManifest(w, m) + fmt.Fprintf(w, "\nThe control plane trusts %d engine files (%s).\n", n, file) + offlineHint(w, cfg, haveCfg) + return nil + }, + } + f := c.Flags() + f.StringVar(&digest, "sha256", "", "the bundle's sha256, as 'bundle create' printed it") + f.BoolVar(&unchecked, "unchecked", false, "take the manifest without its sha256") + f.StringVar(&file, "file", "", "where to record them (default: offline.checksums from server.yaml, else "+config.DefaultOfflineChecksums+")") + f.StringVar(&serverCfg, "config", "/etc/llmgw/server.yaml", "the control plane's configuration") + return c +} + +func newBundleShowCmd() *cobra.Command { + var verify bool + var digest string + c := &cobra.Command{ + Use: "show PATH", + Short: "Say what a bundle holds, and check it", + Long: "Prints what the bundle carries and its sha256. With --verify every file is\n" + + "read and checked, as an import would, and nothing is written: worth doing\n" + + "after copying a bundle onto the disk that carries it.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + w := cmd.OutOrStdout() + m, d, err := bundle.ReadManifest(args[0]) + if err != nil { + return err + } + printManifest(w, m) + fmt.Fprintf(w, "\nsha256 %s\n", d) + if !verify { + return nil + } + res, err := bundle.Import(args[0], bundle.ImportOptions{SHA256: digest, Unchecked: digest == ""}) + if err != nil { + return err + } + fmt.Fprintf(w, "Every file is as it was made (%d files, %s).\n", res.Written, fileSize(res.Bytes)) + return nil + }, + } + c.Flags().BoolVar(&verify, "verify", false, "read and check every file") + c.Flags().StringVar(&digest, "sha256", "", "with --verify, the sha256 the bundle should have") + return c +} + +// printManifest says what a bundle carries. +func printManifest(w io.Writer, m bundle.Manifest) { + var total int64 + for _, f := range m.Files { + total += f.Bytes + } + fmt.Fprintf(w, "\nA bundle made %s by llmgw %s: %d files, %s.\n", + m.Created.Format("2006-01-02 15:04 UTC"), m.MadeBy, len(m.Files), fileSize(total)) + for _, r := range m.Releases { + fmt.Fprintf(w, " llmgw %s for %s\n", r.Version, strings.Join(r.Arches, ", ")) + } + for _, e := range m.Engines { + fmt.Fprintf(w, " %-9s %s for %s: %s\n", e.Engine, e.Version, strings.Join(e.Arches, ", "), strings.Join(e.Variants, ", ")) + } + for _, md := range m.Models { + at := "" + if md.Revision != "" { + at = " at " + md.Revision[:min(len(md.Revision), 8)] + } + fmt.Fprintf(w, " model %s/%s from %s%s\n", md.Name, md.Version, md.Source, at) + } +} + +// loadServerConfig reads a control plane's configuration, when this +// machine is one. +func loadServerConfig(path string) (config.Server, bool) { + if _, err := os.Stat(path); err != nil { + return config.Server{}, false + } + cfg, err := config.Load(path) + if err != nil { + return config.Server{}, false + } + return cfg, true +} + +// offlineHint says what is left once a bundle is trusted. +func offlineHint(w io.Writer, cfg config.Server, haveCfg bool) { + if haveCfg && !cfg.Offline.Enabled() { + fmt.Fprintf(w, "\nserver.yaml does not yet say the cluster is offline. Add the mirror's file\n"+ + "address, as nodes reach it, and restart the control plane:\n\n"+ + " offline:\n mirror: http://:8081\n\n"+ + " sudo systemctl restart llmgw-server\n") + } +} diff --git a/internal/cli/ctl_upgrade.go b/internal/cli/ctl_upgrade.go index 150dff6..c119e0c 100644 --- a/internal/cli/ctl_upgrade.go +++ b/internal/cli/ctl_upgrade.go @@ -50,6 +50,16 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command { } out := cmd.OutOrStdout() ctx := cmd.Context() + if releases == "" { + // Where the control plane sends its nodes: an offline + // cluster's mirror, so this machine needs no internet either. + if cfg, ok := loadServerConfig("/etc/llmgw/server.yaml"); ok { + releases = cfg.ReleasesBase() + } + if releases == "" { + releases = release.DefaultBase + } + } if !skipCP { if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil { return err @@ -60,7 +70,8 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command { } c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)") c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes") - c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from, for this machine") + c.Flags().StringVar(&releases, "releases", "", "where release binaries are fetched from, for this machine "+ + "(default: as server.yaml says, the mirror's copy on an offline cluster, else the project's)") c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back") c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit") return c diff --git a/internal/cli/join.go b/internal/cli/join.go index f7dc8f9..925b8a3 100644 --- a/internal/cli/join.go +++ b/internal/cli/join.go @@ -53,9 +53,15 @@ func newJoinCmd() *cobra.Command { // screen that cuts long ones off. func joinInstall() []string { v := version.Version + // From where this cluster takes its releases: on an offline one, its + // mirror, which the new machine can reach when the internet is not. + base := release.DefaultBase + if cfg, ok := loadServerConfig("/etc/llmgw/server.yaml"); ok && cfg.ReleasesBase() != "" { + base = cfg.ReleasesBase() + } return []string{ `a=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')`, - fmt.Sprintf("sudo curl -fsSL -o /usr/local/bin/llmgw \\\n %s/%s/llmgw-%s-linux-$a", release.DefaultBase, v, v), + fmt.Sprintf("sudo curl -fsSL -o /usr/local/bin/llmgw \\\n %s/%s/llmgw-%s-linux-$a", base, v, v), "sudo chmod 755 /usr/local/bin/llmgw", } } diff --git a/internal/cli/node.go b/internal/cli/node.go index 02bea3c..e00f9f3 100644 --- a/internal/cli/node.go +++ b/internal/cli/node.go @@ -17,6 +17,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/firewall" "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/version" ) @@ -120,7 +121,7 @@ engines are kept.`, f.StringVar(&na.opts.NvidiaSMI, "nvidia-smi", "", "where nvidia-smi is, when it is not on PATH") f.StringVar(&na.opts.IBRoot, "ib-root", "", "the sysfs InfiniBand directory, when not /sys/class/infiniband") f.StringVar(&na.opts.PKI, "pki", defaultPKIDir, "directory holding the certificate authority") - f.StringVar(&na.opts.Releases, "releases", "https://gitea.mynodes.xyz/ilker/llmgw/releases/download", "where release files are downloaded from") + f.StringVar(&na.opts.Releases, "releases", "", "where release files are downloaded from (default: releases in server.yaml, the mirror's copy on an offline cluster, else the project's)") f.BoolVar(&na.opts.IgnoreChecks, "ignore-checks", false, "add the node even when doctor finds a problem") f.BoolVar(&na.opts.NoNodeExporter, "no-node-exporter", false, "leave Prometheus' node exporter off the machine") f.BoolVar(&na.opts.NoFirewall, "no-firewall", false, "leave the machine's firewall alone, instead of keeping llmgw's ports to the cluster's networks") @@ -176,6 +177,16 @@ func (na *nodeAdd) prepare() error { o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter} o.FirewallAllow, _ = cfg.FirewallAllow() } + // Where the control plane upgrades nodes from, so a node joins on the + // same release from the same place: an offline cluster's mirror. + if o.Releases == "" { + if cfg, err := config.Load(na.serverCfg); err == nil { + o.Releases = cfg.ReleasesBase() + } + if o.Releases == "" { + o.Releases = release.DefaultBase + } + } o.Version = version.Version if o.Binary == "" { exe, err := os.Executable() diff --git a/internal/cli/root.go b/internal/cli/root.go index e6f61fd..1f45211 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -52,6 +52,7 @@ func Root() *cobra.Command { newUsersCmd(), newOrgCmd(), newMirrorCmd(), + newBundleCmd(), newSimEngineCmd(), newDoctorCmd(), newNodeCmd(), diff --git a/internal/cli/server.go b/internal/cli/server.go index aa783ce..38c1e98 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -32,6 +32,7 @@ import ( llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" "gitea.mynodes.xyz/ilker/llmgw/internal/audit" + "gitea.mynodes.xyz/ilker/llmgw/internal/bundle" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" @@ -100,6 +101,12 @@ func newServerCmd() *cobra.Command { return errors.New("nothing to serve: declare models with profiles, " + "or give --endpoint for a router-only run") } + // Offline, nothing is asked of the internet: releases come from + // the mirror's copy, and no registry is asked about newer weights. + if cfg.Offline.Enabled() { + cfg.Releases = cfg.ReleasesBase() + cfg.Mirror.UpdateCheck = 0 + } // Before the router, so its tracer is the configured one. shutdownTracing, err := obs.SetupTracing(cmd.Context(), obs.TracingOptions{ @@ -654,6 +661,21 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live return cfg.EnginesFor(n.ID, slices.Sorted(maps.Keys(n.GPUsByClass())), n.Labels) }) log.Info("nodes install these engines", "engines", cfg.WantedEngines(), "overrides", len(cfg.EngineOverrides)) + if cfg.Offline.Enabled() { + // From the mirror, each file held to the checksum the bundle gave, + // which this control plane keeps and the mirror has no say in. + trusted := bundle.NewTrusted(cfg.Offline.ChecksumsFile()) + svc.SetOffline(func(engine, version string) (string, map[string]string) { + return bundle.EngineBase(cfg.Offline.Mirror, engine, version), trusted.For(engine, version) + }) + if v, ok := cfg.WantedEngines()["llamacpp"]; ok && len(trusted.For("llamacpp", v)) == 0 { + log.Warn("offline, and no bundle imported here carries llama.cpp at this version: nodes cannot "+ + "install it until one does (llmgw bundle create --engine llamacpp="+v+", then bundle import or trust)", + "version", v, "checksums", cfg.Offline.ChecksumsFile(), "err", trusted.Err()) + } + log.Info("offline: nodes install engines from the mirror", "mirror", cfg.Offline.Mirror, + "releases", cfg.Releases) + } if ch := ctl.EngineChoices(); ch != nil { svc.SetEngineChoices(ch.For) ch.OnChange(svc.Reprovision) diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index 03e6f5b..b3bf488 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -89,6 +89,12 @@ type serverInit struct { noFirewall bool // mirrorURL is a mirror on another machine to use instead of running one. mirrorURL string + // bundle is an offline bundle to install from, and bundleSHA its + // manifest's sha256: the cluster is then set up offline. + bundle, bundleSHA string + // mirrorPath is the mirror's directory here; "" is /srv/models. A seam + // for tests. + mirrorPath string // pgSetup prepares PostgreSQL and returns the store address; a seam for // tests. Nil does it for real. pgSetup func(context.Context) (string, error) @@ -165,6 +171,9 @@ func newServerInitCmd() *cobra.Command { f.BoolVar(&in.noMirror, "no-mirror", false, "do not run a model mirror on this machine") f.StringVar(&in.mirrorURL, "mirror", "", "use the mirror on another machine, set up there by 'llmgw mirror init': its management API, http://host:8082") + f.StringVar(&in.bundle, "bundle", "", + "install offline from this bundle ('llmgw bundle create'): into the mirror, with the cluster set to reach nothing outside") + f.StringVar(&in.bundleSHA, "bundle-sha256", "", "the bundle's sha256, as 'llmgw bundle create' printed it") f.BoolVar(&in.noNode, "no-node", false, "do not make this machine a node (and serve no model on it)") f.BoolVar(&in.noModel, "no-model", false, "do not fetch and serve the starter model ("+starterName+")") f.BoolVar(&in.noGuardModel, "no-guard-model", false, @@ -180,6 +189,9 @@ func newServerInitCmd() *cobra.Command { } func (in *serverInit) run(ctx context.Context) error { + if err := in.checkBundle(); err != nil { + return fmt.Errorf("offline bundle: %w", err) + } steps := []struct { name string fn func(context.Context) error @@ -214,6 +226,13 @@ func (in *serverInit) run(ctx context.Context) error { return fmt.Errorf("mirror: %w", err) } } + // Into the mirror, and server.yaml told, before the service starts: + // the control plane reads at start that the cluster is offline. + if in.bundle != "" { + if err := in.ensureBundle(ctx); err != nil { + return fmt.Errorf("offline bundle: %w", err) + } + } if err := in.ensureService(ctx); err != nil { return fmt.Errorf("service: %w", err) } @@ -256,6 +275,11 @@ func (in *serverInit) judgeWanted() bool { if in.noGuardModel || (in.noMirror && in.mirrorURL == "") || in.unitPath == "" { return false } + // Offline, only a judge the bundle carries: a policy asking a judge + // that never comes refuses every request. + if in.bundle != "" && !in.bundleCarries(guardModelName) { + return false + } _, err := exec.LookPath("systemctl") return err == nil } diff --git a/internal/cli/serverinit_bundle.go b/internal/cli/serverinit_bundle.go new file mode 100644 index 0000000..be782b2 --- /dev/null +++ b/internal/cli/serverinit_bundle.go @@ -0,0 +1,158 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/bundle" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" +) + +// --- installing from an offline bundle --- + +// checkBundle refuses, before anything is done, a bundle init could not +// install from: one that is not there, or not the one that was made. +func (in *serverInit) checkBundle() error { + if in.bundle == "" { + return nil + } + if in.noMirror && in.mirrorURL == "" { + return errors.New("a bundle goes into the cluster's mirror: leave out --no-mirror, or give --mirror") + } + if in.bundleSHA == "" { + return errors.New("give --bundle-sha256, the sha256 'llmgw bundle create' printed: " + + "it is what says the bundle that arrived is the one that was made") + } + _, digest, err := bundle.ReadManifest(in.bundle) + if err != nil { + return err + } + if digest != in.bundleSHA { + return fmt.Errorf("the bundle's sha256 is %s, not the %s given: it is not the bundle that was made, "+ + "or the digest was mistyped", digest, in.bundleSHA) + } + return nil +} + +// bundleCarries says whether the bundle holds a model of this name. +func (in *serverInit) bundleCarries(model string) bool { + m, _, err := bundle.ReadManifest(in.bundle) + if err != nil { + return false + } + for _, md := range m.Models { + if md.Name == model { + return true + } + } + return false +} + +// offline says whether this cluster is set up to reach nothing outside. +func (in *serverInit) offline() bool { + if in.bundle != "" { + return true + } + cfg, err := config.Load(in.configPath()) + return err == nil && cfg.Offline.Enabled() +} + +func (in *serverInit) mirrorRoot() string { + if in.mirrorPath != "" { + return in.mirrorPath + } + return mirrorDir +} + +// ensureBundle puts the bundle into the mirror beside this control plane, +// has the control plane trust its engines' checksums, and says in +// server.yaml that the cluster is offline. With a mirror on another +// machine, the files are imported there; here only the checksums are kept. +func (in *serverInit) ensureBundle(ctx context.Context) error { + cfg, err := config.Load(in.configPath()) + if err != nil { + return err + } + var ( + m bundle.Manifest + files string + ) + if in.mirrorURL == "" { + res, err := bundle.Import(in.bundle, bundle.ImportOptions{Mirror: in.mirrorRoot(), SHA256: in.bundleSHA}) + if err != nil { + return err + } + if res.Written > 0 { + in.done("offline bundle in the mirror: %d files, %s, each checked", res.Written, fileSize(res.Bytes)) + } else { + in.kept("offline bundle in the mirror (%d files)", res.Kept) + } + m = res.Manifest + files = "http://" + net.JoinHostPort(in.reachableAddr(), "8081") + } else { + mc, err := mirror.NewClient(in.mirrorURL, tlsconf.Config{}, 10*time.Second) + if err != nil { + return err + } + list, err := mc.List(ctx) + if err != nil { + return fmt.Errorf("asking the mirror at %s where nodes reach it: %w", in.mirrorURL, err) + } + files = list.Base + if m, _, err = bundle.ReadManifest(in.bundle); err != nil { + return err + } + fmt.Fprintf(in.out, " !! the mirror is on another machine: import the bundle there too, "+ + "sudo llmgw bundle import --sha256 %s\n", in.bundleSHA) + } + sums := cfg.Offline.Checksums + if sums == "" { + sums = filepath.Join(in.etcDir, "offline.sha256") + } + n, err := bundle.TrustManifest(m, sums) + if err != nil { + return err + } + in.done("the control plane trusts %d engine files from the bundle (%s)", n, sums) + + if cfg.Offline.Enabled() { + in.kept("offline, installing from %s, as server.yaml says", cfg.Offline.Mirror) + return nil + } + if sums == config.DefaultOfflineChecksums { + sums = "" + } + if err := appendOfflineBlock(in.configPath(), files, sums); err != nil { + return err + } + in.done("server.yaml: offline, nodes install from the mirror at %s", files) + return nil +} + +// appendOfflineBlock says in server.yaml that the cluster is offline, +// without touching the rest. +func appendOfflineBlock(path, files, sums string) error { + b, err := os.ReadFile(path) //nolint:gosec // server.yaml + if err != nil { + return err + } + info, err := os.Stat(path) + if err != nil { + return err + } + block := "\n# Offline: nothing in the cluster reaches the internet. Nodes install\n" + + "# engines, and upgrades fetch releases, from the mirror, out of the bundle\n" + + "# 'llmgw server init --bundle' imported; 'llmgw bundle import' adds more.\n" + + "offline:\n mirror: " + files + "\n" + if sums != "" { + block += " checksums: " + sums + "\n" + } + return os.WriteFile(path, append(b, []byte(block)...), info.Mode().Perm()) +} diff --git a/internal/cli/serverinit_bundle_test.go b/internal/cli/serverinit_bundle_test.go new file mode 100644 index 0000000..a89fe62 --- /dev/null +++ b/internal/cli/serverinit_bundle_test.go @@ -0,0 +1,86 @@ +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/bundle" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" +) + +// handMadeBundle is a bundle as 'bundle create' leaves it: an engine build +// and the starter model, with their checksums in the manifest. +func handMadeBundle(t *testing.T) (dir, digest string) { + t.Helper() + dir = t.TempDir() + m := bundle.Manifest{Format: bundle.Format, MadeBy: "test", + Models: []bundle.Model{{Name: starterName, Version: starterVersion, Source: starterSource}}} + for p, body := range map[string]string{ + "engines/llamacpp/b1/llama-b1-bin-ubuntu-x64.tar.gz": "an engine", + "models/" + starterName + "/" + starterVersion + "/qwen-q4_k_m.gguf": "weights", + } { + full := filepath.Join(dir, p) + _ = os.MkdirAll(filepath.Dir(full), 0o755) + _ = os.WriteFile(full, []byte(body), 0o644) + sum := sha256.Sum256([]byte(body)) + m.Files = append(m.Files, bundle.File{Path: p, Bytes: int64(len(body)), SHA256: hex.EncodeToString(sum[:])}) + } + raw, _ := json.Marshal(m) + _ = os.WriteFile(filepath.Join(dir, bundle.ManifestName), raw, 0o644) + return dir, bundle.Digest(raw) +} + +func TestInitFromABundleLeavesAnOfflineClusterThatTrustsItsEngines(t *testing.T) { + in, out := newTestInit(t) + dir, digest := handMadeBundle(t) + in.bundle, in.hosts, in.mirrorPath = dir, []string{"10.9.8.7"}, t.TempDir() + + in.bundleSHA = "" + if err := in.checkBundle(); err == nil || !strings.Contains(err.Error(), "--bundle-sha256") { + t.Errorf("a bundle with no digest: %v", err) + } + in.bundleSHA = strings.Repeat("0", 64) + if err := in.checkBundle(); err == nil { + t.Error("a bundle not matching its digest was taken") + } + in.bundleSHA = digest + if err := in.run(context.Background()); err != nil { + t.Fatalf("%v\n%s", err, out) + } + if err := in.ensureBundle(context.Background()); err != nil { + t.Fatalf("%v\n%s", err, out) + } + + cfg, err := config.Load(in.configPath()) + if err != nil { + t.Fatal(err) + } + if cfg.Offline.Mirror != "http://10.9.8.7:8081" || cfg.ReleasesBase() != "http://10.9.8.7:8081/_llmgw/releases" { + t.Errorf("server.yaml: offline %+v, releases %s", cfg.Offline, cfg.ReleasesBase()) + } + got := bundle.NewTrusted(cfg.Offline.ChecksumsFile()).For("llamacpp", "b1") + if len(got) != 1 { + t.Errorf("the control plane trusts %v", got) + } + if _, err := os.Stat(filepath.Join(in.mirrorPath, starterName, starterVersion, "qwen-q4_k_m.gguf")); err != nil { + t.Errorf("the starter model is not in the mirror: %v", err) + } + if !in.bundleCarries(starterName) || in.bundleCarries(guardModelName) { + t.Error("what the bundle carries was misread") + } + + // Again, and nothing changes: not the mirror, not server.yaml. + before, _ := os.ReadFile(in.configPath()) + if err := in.ensureBundle(context.Background()); err != nil { + t.Fatal(err) + } + if after, _ := os.ReadFile(in.configPath()); string(after) != string(before) { + t.Error("init from the same bundle changed server.yaml again") + } +} diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index 9b162b8..efaef21 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -516,6 +516,11 @@ func (in *serverInit) ensureServed(ctx context.Context, name, source, version st var apiErr *mirror.APIError switch { case err == nil: + case errors.As(err, &apiErr) && apiErr.Status == 404 && in.offline(): + // Asked of the internet, it would fail after a timeout and stop init + // over a model nobody carried in. + fmt.Fprintf(in.out, " skip %s: not in the mirror, and an offline cluster does not download it\n", name) + return nil case errors.As(err, &apiErr) && apiErr.Status == 404: if st, err = m.Fetch(ctx, mirror.FetchRequest{ Source: upstream, Model: name, Version: version, diff --git a/internal/cli/tui_init.go b/internal/cli/tui_init.go index c256902..fda49a2 100644 --- a/internal/cli/tui_init.go +++ b/internal/cli/tui_init.go @@ -55,6 +55,8 @@ const ( fNode fModel fGuard + fBundle + fBundleSHA fStart fCount ) @@ -68,6 +70,8 @@ type initModel struct { focus int admin textField hosts textField + bundle textField + bundleSHA textField toggles [fCount]bool formError string @@ -99,8 +103,19 @@ func newInitModel(in *serverInit, found []string, askForm bool) initModel { hosts.SetValue(strings.Join(in.hosts, ", ")) hosts.CharLimit = 400 hosts.Width = 40 + bundle := newTextField() + bundle.Placeholder = "/media/usb/llmgw-bundle.tar" + bundle.SetValue(in.bundle) + bundle.CharLimit = 400 + bundle.Width = 40 + bundleSHA := newTextField() + bundleSHA.Placeholder = "the sha256 'llmgw bundle create' printed" + bundleSHA.SetValue(in.bundleSHA) + bundleSHA.CharLimit = 64 + bundleSHA.Width = 64 sp := newSpinner() - m := initModel{in: in, found: found, form: askForm, admin: admin, hosts: hosts, spin: sp, width: 100} + m := initModel{in: in, found: found, form: askForm, admin: admin, hosts: hosts, + bundle: bundle, bundleSHA: bundleSHA, spin: sp, width: 100} m.toggles[fPostgres] = in.postgres m.toggles[fMirror] = !in.noMirror m.toggles[fNode] = !in.noNode @@ -207,6 +222,20 @@ func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { return m, nil } } + bundle, sum := strings.TrimSpace(m.bundle.Value()), strings.ToLower(strings.TrimSpace(m.bundleSHA.Value())) + switch { + case bundle != "" && sum == "": + m.formError = "give the bundle's sha256, as 'llmgw bundle create' printed it" + m.focus = fBundleSHA + m.moveFocus(0) + return m, nil + case bundle != "" && !m.toggles[fMirror] && m.in.mirrorURL == "": + m.formError = "an offline bundle goes into the model mirror: tick it" + m.focus = fMirror + m.moveFocus(0) + return m, nil + } + m.in.bundle, m.in.bundleSHA = bundle, sum m.in.admin = admin m.in.hosts = nil for _, h := range strings.Split(m.hosts.Value(), ",") { @@ -231,6 +260,14 @@ func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { var cmd tea.Cmd m.hosts, cmd = m.hosts.Update(k) return m, cmd + case fBundle: + var cmd tea.Cmd + m.bundle, cmd = m.bundle.Update(k) + return m, cmd + case fBundleSHA: + var cmd tea.Cmd + m.bundleSHA, cmd = m.bundleSHA.Update(k) + return m, cmd case fPostgres, fMirror, fNode, fModel, fGuard: if k.String() == " " || k.String() == "x" { m.toggles[m.focus] = !m.toggles[m.focus] @@ -247,11 +284,17 @@ func (m *initModel) moveFocus(d int) { } m.admin.Blur() m.hosts.Blur() + m.bundle.Blur() + m.bundleSHA.Blur() switch m.focus { case fAdmin: m.admin.Focus() case fHosts: m.hosts.Focus() + case fBundle: + m.bundle.Focus() + case fBundleSHA: + m.bundleSHA.Focus() } } @@ -375,6 +418,9 @@ func (m initModel) formView() string { b.WriteString(check(fNode, "This machine as a node", "serves models here too, on its GPUs or CPU")) b.WriteString(check(fModel, "Starter model", "a small Qwen, so there is something to ask at once")) b.WriteString(check(fGuard, "Guardrails' judge", "Llama Guard 3 1B on a CPU, refusing violent, criminal and abusive requests")) + fmt.Fprintf(&b, "\n%s\n %s\n", label(fBundle, "Offline bundle"), m.bundle.View()) + b.WriteString(" " + sDim.Render("For a site with no internet: engines, releases and models from 'llmgw bundle create'. Empty downloads them.") + "\n") + fmt.Fprintf(&b, "%s\n %s\n", label(fBundleSHA, "Its sha256"), m.bundleSHA.View()) b.WriteString("\n" + label(fStart, lipgloss.NewStyle().Bold(true).Render("Set up")) + "\n") if m.formError != "" { b.WriteString("\n" + sBad.Render(m.formError) + "\n") diff --git a/internal/cli/tui_init_test.go b/internal/cli/tui_init_test.go index 025ae45..f3ea6ef 100644 --- a/internal/cli/tui_init_test.go +++ b/internal/cli/tui_init_test.go @@ -23,8 +23,9 @@ func TestTheInitFormBecomesInitsSettings(t *testing.T) { m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // PostgreSQL on m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // starter model off - m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // judge off - m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter}) + m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // judge off + m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}) // no bundle + m = send(m, tea.KeyMsg{Type: tea.KeyEnter}) if in.admin != "ops@example.com" || strings.Join(in.hosts, "|") != "llm.example.com|203.0.113.10" { t.Errorf("admin %q hosts %q", in.admin, in.hosts) @@ -37,6 +38,27 @@ func TestTheInitFormBecomesInitsSettings(t *testing.T) { } } +func TestAnOfflineBundleIsGivenInTheFormWithItsDigest(t *testing.T) { + in := &serverInit{} + var m tea.Model = newInitModel(in, nil, true) + if v := m.View(); !strings.Contains(v, "Offline bundle") { + t.Fatalf("the form offers no bundle:\n%s", v) + } + for range fBundle { + m = send(m, tea.KeyMsg{Type: tea.KeyTab}) + } + m = typeIn(m, "/media/usb/b.tar") + m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter}) + if !m.(initModel).form || !strings.Contains(m.View(), "sha256") || m.(initModel).focus != fBundleSHA { + t.Fatalf("a bundle without its digest was accepted:\n%s", m.View()) + } + m = typeIn(m, strings.Repeat("AB", 32)) + m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter}) + if m.(initModel).form || in.bundle != "/media/usb/b.tar" || in.bundleSHA != strings.Repeat("ab", 32) { + t.Errorf("bundle %q sha %q, form up %v", in.bundle, in.bundleSHA, m.(initModel).form) + } +} + func TestAnEmailThatIsNotOneIsSaidSo(t *testing.T) { in := &serverInit{} var m tea.Model = newInitModel(in, nil, true) diff --git a/internal/config/config.go b/internal/config/config.go index 5d7050c..0073692 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -102,6 +102,11 @@ type Server struct { // Empty means there is none, and /admin/mirror stays unmounted. Mirror Mirror `koanf:"mirror"` + // Offline is a cluster that reaches nothing outside itself: its nodes + // install engines, and upgrades fetch releases, from the mirror, out of + // a bundle carried in (llmgw bundle create, llmgw bundle import). + Offline Offline `koanf:"offline"` + // Engines are the engines every node installs, and at which version. The // agents install them themselves — the right build for their hardware, // checked against its publisher's checksum and seen to run — so a node is @@ -674,6 +679,9 @@ func (s Server) Validate() error { if err := s.AgentTLS.Validate(); err != nil { return err } + if err := s.Offline.Validate(); err != nil { + return err + } if s.Auth.Enabled && s.Auth.Store == "" { return fmt.Errorf("auth.store is required when auth is enabled") @@ -822,3 +830,75 @@ func (s Server) WantedEngines() map[string]string { } return out } + +// Offline is a cluster with no way to the internet. +// +// Everything it would otherwise fetch from outside — llmgw's releases, the +// engine builds its nodes install, model files — is carried in as a bundle, +// made where there is internet (llmgw bundle create) and imported into the +// mirror here (llmgw bundle import). Nodes then fetch from the mirror, and +// check what they fetch as strictly as they would online: releases against +// the project's signing key, engines against the checksums the control plane +// holds from the bundle. +type Offline struct { + // Mirror is the mirror's file address, as nodes reach it: + // http://10.0.0.5:8081 — the port model files are served on, not the + // management API. Set, the cluster is offline: nodes install engines + // from it and never from their publishers, releases come from it unless + // releases names another place, and no registry is asked whether a + // model has newer weights. Python engines (vLLM, SGLang) come from PyPI, + // which no bundle carries; on an offline node they are installed by hand. + Mirror string `koanf:"mirror"` + // Checksums is the file of engine checksums this control plane trusts, + // kept by llmgw bundle import and llmgw bundle trust; every engine file + // a node fetches from the mirror is checked against it. Read again + // whenever it changes. Empty is /etc/llmgw/offline.sha256. + Checksums string `koanf:"checksums"` +} + +// DefaultOfflineChecksums is where the engine checksums are kept when +// offline.checksums does not say. +const DefaultOfflineChecksums = "/etc/llmgw/offline.sha256" + +// Enabled reports whether the cluster is offline. +func (o Offline) Enabled() bool { return o.Mirror != "" } + +// ChecksumsFile is the engine checksums' file. +func (o Offline) ChecksumsFile() string { + if o.Checksums != "" { + return o.Checksums + } + return DefaultOfflineChecksums +} + +// ReleasesBase is where an offline cluster's releases are on its mirror. +func (o Offline) ReleasesBase() string { + return strings.TrimSuffix(o.Mirror, "/") + "/_llmgw/releases" +} + +// Validate refuses a mirror address nodes could not fetch from. +func (o Offline) Validate() error { + if o.Mirror == "" { + return nil + } + if !strings.HasPrefix(o.Mirror, "http://") && !strings.HasPrefix(o.Mirror, "https://") { + return fmt.Errorf("offline.mirror %q: give the mirror's file address, http://host:8081", o.Mirror) + } + if strings.HasSuffix(strings.TrimSuffix(o.Mirror, "/"), ":8082") { + return fmt.Errorf("offline.mirror %q is the mirror's management port: give the one nodes fetch files from, :8081", o.Mirror) + } + return nil +} + +// ReleasesBase is where release binaries are fetched from: releases when +// set, the mirror's copy on an offline cluster, and empty — the caller's +// default, the project's own releases — otherwise. +func (s Server) ReleasesBase() string { + if s.Releases != "" { + return s.Releases + } + if s.Offline.Enabled() { + return s.Offline.ReleasesBase() + } + return "" +} diff --git a/internal/control/agentsvc/installable_test.go b/internal/control/agentsvc/installable_test.go index d8675e0..21e0ace 100644 --- a/internal/control/agentsvc/installable_test.go +++ b/internal/control/agentsvc/installable_test.go @@ -55,6 +55,23 @@ func TestEachNodeIsToldTheEnginesDecidedForIt(t *testing.T) { } } +func TestAnOfflineClusterTellsNodesWhereOnTheMirrorAndWhatEachFileMustBe(t *testing.T) { + s := &Service{} + s.SetEngines(map[string]string{"llamacpp": "b1"}) + if w := s.provisionFor(inventory.Node{ID: "n"}, []string{"llamacpp"}).GetEngines()[0]; w.GetOfflineBase() != "" { + t.Fatalf("an online cluster sent a mirror: %v", w) + } + s.SetOffline(func(engine, version string) (string, map[string]string) { + return "http://mirror:8081/_llmgw/engines/" + engine + "/" + version, + map[string]string{"llama-" + version + "-bin-ubuntu-x64.tar.gz": "abc"} + }) + w := s.provisionFor(inventory.Node{ID: "n"}, []string{"llamacpp"}).GetEngines()[0] + if w.GetOfflineBase() != "http://mirror:8081/_llmgw/engines/llamacpp/b1" || + w.GetOfflineSha256()["llama-b1-bin-ubuntu-x64.tar.gz"] != "abc" { + t.Errorf("the node was told %v", w) + } +} + func TestABuildChosenForANodeIsWhatItIsTold(t *testing.T) { s := &Service{} s.SetEngines(map[string]string{"llamacpp": "b1", "vllm": "0.30.0"}) diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 3771e2a..8075cc0 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -63,6 +63,9 @@ type Service struct { // each connected node said it can install, for telling it again. choices func(nodeID string) map[string]string installableBy map[string][]string + // offline, when set, is where engines come from on a cluster with no + // way to the internet; see SetOffline. + offline func(engine, version string) (string, map[string]string) // controlPlanes lists what agents may dial; see SetControlPlanes. controlPlanes func(context.Context) ([]string, error) @@ -962,11 +965,24 @@ func (s *Service) provisionFor(node inventory.Node, can []string) *llmgwv1.Provi if !slices.Contains(can, e) || chosen[e] == "none" { continue } - p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: wants[e], Variant: chosen[e]}) + w := &llmgwv1.EngineWant{Engine: e, Version: wants[e], Variant: chosen[e]} + if s.offline != nil { + w.OfflineBase, w.OfflineSha256 = s.offline(e, wants[e]) + } + p.Engines = append(p.Engines, w) } return p } +// SetOffline makes nodes install engines from the cluster's mirror rather +// than from their publishers: f gives where an engine version's files are +// there, and each file's checksum from the bundle the control plane trusts. +// A version no bundle carried has no checksums, and the node says which +// bundle to import rather than reaching for the internet. +func (s *Service) SetOffline(f func(engine, version string) (base string, sha256 map[string]string)) { + s.offline = f +} + // installable are the engines a node can be asked to install. An agent older // than the list said nothing, and could install llama.cpp alone: asked for // vLLM too, it reported it unsupported, and a cluster half way through an diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 839dea7..e802e09 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -141,6 +141,10 @@ var Reference = []Block{ {Key: "memberships", Type: "map[string]string", Doc: "Memberships maps a group to a place in the tree — org:NAME,\nteam:ORG/TEAM, project:ORG/TEAM/PROJECT — joined at sign-in.", Nested: ""}, {Key: "no_passwords", Type: "bool", Doc: "PasswordSignIn off refuses passwords on the sign-in form: people come\nin through the provider only. Administrators still can, so a broken\nprovider does not lock everybody out.", Nested: ""}, }}, + {Package: "config", Name: "Offline", Title: "Offline", Doc: "", Fields: []Field{ + {Key: "mirror", Type: "string", Doc: "Mirror is the mirror's file address, as nodes reach it:\nhttp://10.0.0.5:8081 — the port model files are served on, not the\nmanagement API. Set, the cluster is offline: nodes install engines\nfrom it and never from their publishers, releases come from it unless\nreleases names another place, and no registry is asked whether a\nmodel has newer weights. Python engines (vLLM, SGLang) come from PyPI,\nwhich no bundle carries; on an offline node they are installed by hand.", Nested: ""}, + {Key: "checksums", Type: "string", Doc: "Checksums is the file of engine checksums this control plane trusts,\nkept by llmgw bundle import and llmgw bundle trust; every engine file\na node fetches from the mirror is checked against it. Read again\nwhenever it changes. Empty is /etc/llmgw/offline.sha256.", Nested: ""}, + }}, {Package: "discovery", Name: "Options", Title: "Options", Doc: "", Fields: []Field{ {Key: "nvidia_smi", Type: "string", Doc: "NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.\nWorth configuring on nodes where the driver lives outside the default\npaths, which containerized driver installs often do.", Nested: ""}, {Key: "ib_root", Type: "string", Doc: "IBRoot is the sysfs InfiniBand directory.", Nested: ""}, @@ -280,6 +284,7 @@ var Reference = []Block{ {Key: "monitoring", Type: "Monitoring", Doc: "Monitoring is where the dashboard reads its history from.", Nested: "config.Monitoring"}, {Key: "tracing", Type: "Tracing", Doc: "Tracing sends a trace of each inference request to an OpenTelemetry\ncollector. Off unless an endpoint is set here or in\nOTEL_EXPORTER_OTLP_ENDPOINT.", Nested: "config.Tracing"}, {Key: "mirror", Type: "Mirror", Doc: "Mirror points the control plane at a model mirror it can administer.\nEmpty means there is none, and /admin/mirror stays unmounted.", Nested: "config.Mirror"}, + {Key: "offline", Type: "Offline", Doc: "Offline is a cluster that reaches nothing outside itself: its nodes\ninstall engines, and upgrades fetch releases, from the mirror, out of\na bundle carried in (llmgw bundle create, llmgw bundle import).", Nested: "config.Offline"}, {Key: "engines", Type: "map[string]string", Doc: "Engines are the engines every node installs, and at which version. The\nagents install them themselves — the right build for their hardware,\nchecked against its publisher's checksum and seen to run — so a node is\nnever trusted to already have what somebody once put there.\n\nUnset keeps DefaultEngines. A version replaces the default; an empty\none leaves that engine to be installed by hand.", Nested: ""}, {Key: "engine_overrides", Type: "[]EngineOverride", Doc: "EngineOverrides give some nodes different engines from the rest: a\nvLLM built for CUDA 12 on the Volta cards CUDA 13 dropped, or none at\nall on nodes meant only for llama.cpp. Each picks nodes and lays its\nengines over the cluster's, in order, later ones last; an empty version\nremoves that engine from the nodes it picks.", Nested: ""}, {Key: "backup_dir", Type: "string", Doc: "BackupDir is where `llmgw server backup` writes, and where the newest\narchive's age is read for llmgw_backup_last_success_timestamp_seconds.\nEmpty means /var/backups/llmgw-server.", Nested: ""}, diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index c2c3b4f..703391b 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -202,6 +202,22 @@ func writeSource(file, source string) error { return os.Rename(tmp, sourcePath(file)) } +// Record keeps a file's checksum and source beside it, as a download into +// the mirror does: for a file put in place another way — an offline bundle's +// — that should be offered to nodes with its checksum all the same. +func Record(file, sha256, source string) error { + if err := writeChecksum(file, sha256); err != nil { + return err + } + if source == "" { + return nil + } + return writeSource(file, source) +} + +// Recorded is a file's checksum as kept beside it, "" for none. +func Recorded(file string) string { return readChecksum(file) } + // readSource is where a file was fetched from, or "" for one fetched before // this was kept, or put in place by hand. func readSource(file string) string { diff --git a/internal/release/release.go b/internal/release/release.go index 1cd656a..d8ba298 100644 --- a/internal/release/release.go +++ b/internal/release/release.go @@ -95,6 +95,11 @@ func Install(ctx context.Context, client *http.Client, base, version, goos, goar // over plain HTTP, where anyone on the path can swap the bytes and, without a // signing key, the checksums with them. Loopback is allowed, for a mirror on // this machine and for tests. +// +// With a signing key built in, plain HTTP is allowed too: the checksums are +// then checked against a key no server holds, so what carries them adds +// nothing, and an offline site's mirror serves releases over the plain HTTP +// it serves weights on. func CheckBase(base string) error { u, err := url.Parse(base) if err != nil || u.Host == "" { @@ -107,6 +112,9 @@ func CheckBase(base string) error { if ip := net.ParseIP(u.Hostname()); (ip != nil && ip.IsLoopback()) || u.Hostname() == "localhost" { return nil } + if TrustedKey() != "" { + return nil + } } return fmt.Errorf("releases are fetched over https, and %s is not", base) } @@ -161,34 +169,86 @@ func (l *limitWriter) Write(p []byte) (int, error) { } // checksum reads the release's checksum for asset — after checking the -// checksums' signature, when this build trusts a signing key. A release with -// no signature is then refused rather than taken on its checksums: a missing -// signature is exactly what replacing a release would look like. +// checksums' signature, when this build trusts a signing key. func checksum(ctx context.Context, client *http.Client, root, asset string) (string, error) { - body, err := get(ctx, client, root+"checksums.txt") + c, err := fetchChecksums(ctx, client, root) if err != nil { return "", err } + return c.For(asset) +} + +// Checksums are a release's checksums.txt and its signature, as published. +type Checksums struct { + Text []byte + // Signature is checksums.txt.sig; empty for a release read by a build + // that trusts no key. + Signature string +} + +// FetchChecksums reads a release's checksums and their signature, checked +// as Install checks them: for a machine that keeps them — in an offline +// bundle — to be checked again where they are used. +func FetchChecksums(ctx context.Context, client *http.Client, base, version string) (Checksums, error) { + if !Tag.MatchString(version) { + return Checksums{}, fmt.Errorf("%q is not a release version", version) + } + if err := CheckBase(base); err != nil { + return Checksums{}, err + } + if client == nil { + client = &http.Client{Timeout: 10 * time.Minute} + } + return fetchChecksums(ctx, client, strings.TrimSuffix(base, "/")+"/"+version+"/") +} + +// fetchChecksums reads checksums.txt under root, and its signature when this +// build trusts a signing key. A release with no signature is then refused +// rather than taken on its checksums: a missing signature is exactly what +// replacing a release would look like. +func fetchChecksums(ctx context.Context, client *http.Client, root string) (Checksums, error) { + body, err := get(ctx, client, root+"checksums.txt") + if err != nil { + return Checksums{}, err + } defer body.Close() sums, err := io.ReadAll(io.LimitReader(body, 1<<20)) if err != nil { - return "", err + return Checksums{}, err } - if key := TrustedKey(); key != "" { - sigBody, err := get(ctx, client, root+SignatureName) - if err != nil { - return "", fmt.Errorf("this build only installs signed releases, and %s could not be read: %w", SignatureName, err) - } - sig, err := io.ReadAll(io.LimitReader(sigBody, 4096)) - sigBody.Close() - if err != nil { - return "", err - } - if err := Verify(key, sums, string(sig)); err != nil { - return "", err - } + c := Checksums{Text: sums} + if TrustedKey() == "" { + return c, nil } - sc := bufio.NewScanner(bytes.NewReader(sums)) + sigBody, err := get(ctx, client, root+SignatureName) + if err != nil { + return Checksums{}, fmt.Errorf("this build only installs signed releases, and %s could not be read: %w", SignatureName, err) + } + sig, err := io.ReadAll(io.LimitReader(sigBody, 4096)) + sigBody.Close() + if err != nil { + return Checksums{}, err + } + c.Signature = string(sig) + return c, c.Verify() +} + +// Verify checks the checksums against the key this build trusts. With none +// they are taken as they are, as every release before signing was. +func (c Checksums) Verify() error { + key := TrustedKey() + if key == "" { + return nil + } + if strings.TrimSpace(c.Signature) == "" { + return fmt.Errorf("this build only installs signed releases, and these checksums carry no %s", SignatureName) + } + return Verify(key, c.Text, c.Signature) +} + +// For is the checksum the release gives for one asset. +func (c Checksums) For(asset string) (string, error) { + sc := bufio.NewScanner(bytes.NewReader(c.Text)) for sc.Scan() { f := strings.Fields(sc.Text()) if len(f) == 2 && f[1] == asset && len(f[0]) == 64 { diff --git a/internal/release/release_test.go b/internal/release/release_test.go index ac2d895..8ca9a64 100644 --- a/internal/release/release_test.go +++ b/internal/release/release_test.go @@ -71,6 +71,7 @@ func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) { } func TestReleasesAreFetchedOverHTTPSOrFromThisMachine(t *testing.T) { + unsigned(t) for base, ok := range map[string]bool{ "https://gitea.example/releases/download": true, "http://127.0.0.1:8080/r": true, @@ -89,6 +90,60 @@ func TestReleasesAreFetchedOverHTTPSOrFromThisMachine(t *testing.T) { } } +func TestASignedReleaseMayComeOverPlainHTTPFromAnOfflineMirror(t *testing.T) { + pub, priv, err := GenerateKey() + if err != nil { + t.Fatal(err) + } + t.Setenv("LLMGW_RELEASE_KEY", pub) + if err := CheckBase("http://10.0.0.5:8081/_llmgw/releases"); err != nil { + t.Fatalf("a mirror's plain HTTP was refused with a signing key trusted: %v", err) + } + + // And what comes over it is still held to the signature. + body := []byte("the release") + sum := sha256.Sum256(body) + asset := AssetName("v1.2.3", "linux", "amd64") + sums := []byte(hex.EncodeToString(sum[:]) + " " + asset + "\n") + sig, _ := Sign(priv, sums) + _, other, _ := GenerateKey() + forged, _ := Sign(other, sums) + for _, tc := range []struct { + sig string + ok bool + }{{sig, true}, {forged, false}, {"", false}} { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1.2.3/checksums.txt": + _, _ = w.Write(sums) + case "/v1.2.3/" + SignatureName: + if tc.sig == "" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(tc.sig)) + case "/v1.2.3/" + asset: + _, _ = w.Write(body) + default: + http.NotFound(w, r) + } + })) + c, err := FetchChecksums(t.Context(), srv.Client(), srv.URL, "v1.2.3") + if (err == nil) != tc.ok { + t.Errorf("signature %q: %v", tc.sig, err) + } + if err == nil { + if got, _ := c.For(asset); got != hex.EncodeToString(sum[:]) { + t.Errorf("the checksum read is %s", got) + } + if err := c.Verify(); err != nil { + t.Errorf("kept checksums do not verify again: %v", err) + } + } + srv.Close() + } +} + func TestFetchGivesOnlyACheckedBinary(t *testing.T) { unsigned(t) body := []byte("for another machine") diff --git a/notes/c18.md b/notes/c18.md new file mode 100644 index 0000000..addab1c --- /dev/null +++ b/notes/c18.md @@ -0,0 +1,78 @@ +# C18 — Kubernetes/Helm and an air-gapped bundle + +README's documentation list should gain a line for `docs/kubernetes.md` +(the control plane in Kubernetes from the Helm chart, and why GPU nodes stay +outside it) — README.md was not edited here. + +## CHANGELOG + +### Offline sites and Kubernetes + +- **Offline bundles.** `llmgw bundle create` (on any machine with internet) + collects what a cluster would otherwise download — llmgw's release for + each `--arch` with its signed checksums, every llama.cpp build for those + architectures (or the `--variant` ones) with the checksums GitHub + publishes, the starter model and guardrails' judge, and any `--model + name/version=hf:owner/repo/file` pinned to the registry's commit and its + checksum — each checked as it arrives, into a directory or one `.tar` + (`--tar`), with a manifest whose sha256 it prints to carry apart from the + bundle. Run again into the same directory, it downloads only what is + missing. `llmgw bundle show --verify` reads one back. +- **`llmgw bundle import`** checks every file against the manifest and + llmgw's release against the project's signing key, and only then puts them + into the mirror — releases and engines under `_llmgw/`, models where the + mirror keeps models, with their checksums — so a damaged bundle leaves the + mirror as it was; a different file under a version already there is + refused. On a control plane it also records the engines' checksums in + `offline.checksums` (`/etc/llmgw/offline.sha256`); `llmgw bundle trust` + does that alone, for a mirror on another machine. +- **`offline.mirror` in server.yaml** makes the cluster reach nothing + outside: nodes install llama.cpp from the mirror, each file held to the + checksum the control plane sends them over the agent's authenticated + connection — never one from the mirror — and `releases` defaults to the + mirror's copy, so `node add`, `ctl upgrade` and agents upgrading fetch + from it; no registry is asked about newer weights. A release may now come + over plain HTTP when the build trusts a signing key, since the signature + holds whatever carries it. vLLM and SGLang come from PyPI and are said to + be unsupported on an offline node rather than attempted. +- **`llmgw server init --bundle PATH --bundle-sha256 …`**, and the same two + fields on the setup form: installs a control plane offline — the bundle + into the mirror, the checksums trusted, `offline:` written to server.yaml, + the starter model and judge served from the bundle or skipped when it does + not carry them. +- `docs/data.md` no longer says only weights and `ctl plan` reach outside: + it lists everything that does, and what an offline cluster changes. +- **A Helm chart for the control plane** (`deploy/helm/llmgw`): one replica + in a StatefulSet, SQLite on a claim or an external PostgreSQL, the agent + port's certificate and the admin token from Secrets, the API and metrics + Service, a LoadBalancer for the agents' mutual TLS, an optional Ingress + and ServiceMonitor, `config` laid into server.yaml, and the offline + checksums from a ConfigMap; read-only root, non-root, ready on `/leader`. + `deploy/container/Dockerfile` builds the image it runs. GPU nodes stay + outside Kubernetes and join with `node add` (docs/kubernetes.md says why). + +## BACKLOG + +**Done (branch `features-c2`):** offline bundles — `llmgw bundle create` +(releases with signed checksums, every llama.cpp build per arch/variant +with GitHub's checksums, starter/judge and named Hugging Face model files +pinned and checked; directory or tar; manifest sha256), `bundle import` +(checked, all-or-nothing, into the mirror under `_llmgw/`), `bundle trust`, +`bundle show --verify`; `offline.mirror`/`offline.checksums` in server.yaml, +with nodes installing llama.cpp from the mirror against checksums the +control plane sends over the agent stream (proto `EngineWant.offline_*`), +releases defaulting to the mirror for `node add`, `ctl upgrade` and agent +upgrades, update checks off; `server init --bundle` and the setup form's +bundle fields; docs/data.md's offline claim corrected; install.md "Offline +sites". A Helm chart for the control plane (`deploy/helm/llmgw`, one replica, +SQLite PVC or external PostgreSQL, Secrets, Services, Ingress, +ServiceMonitor) with `deploy/container/Dockerfile` and docs/kubernetes.md; +GPU nodes stay outside Kubernetes by design. Left: vLLM/SGLang offline (a +wheelhouse of several GB per CUDA build and a uv `--find-links` install +path); the NVML release binary and uv in bundles; the setup wizard's +preflight still fails a machine without internet; `join`'s printed `curl` +from the mirror checks nothing; models in a bundle are single files +(no whole-repository safetensors sets yet); the chart runs one replica (a +standby is never ready, so HA in Kubernetes needs a different readiness +design), has not been installed on a real cluster, and ships no agent +DaemonSet; no published container image.