A bundle carries what a cluster would download — llmgw's release with its signed checksums, every llama.cpp build per architecture with GitHub's checksums, model files pinned and checked against Hugging Face's — into a directory or a tar whose manifest sha256 is carried apart. Import checks everything, the release against the signing key, before anything is put in the mirror. With offline.mirror set, nodes install llama.cpp from the mirror against checksums the control plane holds and sends over the agent stream; releases default to the mirror for node add, ctl upgrade and agent upgrades. server init and the setup form install from a bundle. The Helm chart runs one control plane with SQLite on a claim or an external PostgreSQL, Secrets for the agent certificate and admin token, Services, Ingress and ServiceMonitor; deploy/container/Dockerfile builds its image. GPU nodes stay outside Kubernetes, as docs/kubernetes.md explains. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
133 lines
3.8 KiB
YAML
133 lines
3.8 KiB
YAML
# The llmgw control plane in Kubernetes. docs/kubernetes.md says what this
|
|
# runs, what it does not, and how nodes outside the cluster join it.
|
|
|
|
image:
|
|
# Where you pushed the image built from deploy/container/Dockerfile. The
|
|
# project publishes binaries, not an image, so this has no default.
|
|
repository: ""
|
|
# Empty takes the chart's appVersion.
|
|
tag: ""
|
|
pullPolicy: IfNotPresent
|
|
imagePullSecrets: []
|
|
|
|
# One control plane, always. Two against PostgreSQL would elect a leader and
|
|
# keep a standby, but a standby serves nothing and is never ready, so a
|
|
# rollout would wait on it for ever. Until the chart handles that, it runs
|
|
# one: Kubernetes replaces it within seconds when it goes, and the agents
|
|
# reconnect on their own.
|
|
replicas: 1
|
|
|
|
store:
|
|
# sqlite: a file on a PersistentVolumeClaim, for one replica.
|
|
# postgres: a PostgreSQL you run, its address in a Secret.
|
|
type: sqlite
|
|
sqlite:
|
|
size: 10Gi
|
|
storageClass: ""
|
|
postgres:
|
|
# A Secret holding the address, postgres://user:password@host:5432/llmgw?sslmode=verify-full
|
|
existingSecret: ""
|
|
secretKey: dsn
|
|
|
|
# The agent port's certificate. It comes from llmgw's own authority, not a
|
|
# public one — agents present certificates from the same authority — so it
|
|
# is made outside the cluster, where the authority's key stays:
|
|
#
|
|
# llmgw ctl cert init-ca --dir ./pki
|
|
# llmgw ctl cert issue-server llmgw-ctl --host <agent address> --dir ./pki --out ./pki
|
|
# kubectl create secret generic llmgw-agent-tls \
|
|
# --from-file=ca.pem=pki/ca.pem \
|
|
# --from-file=server.pem=pki/server.pem \
|
|
# --from-file=server-key.pem=pki/server-key.pem
|
|
#
|
|
# --host is every address agents dial: agentService's load balancer address
|
|
# or DNS name.
|
|
agentTLS:
|
|
existingSecret: ""
|
|
|
|
# The admin API's token (llmgw ctl, the console). Empty makes one, kept in a
|
|
# Secret across upgrades.
|
|
adminToken:
|
|
existingSecret: ""
|
|
secretKey: admin-token
|
|
|
|
# server.yaml beyond what the chart sets (listeners, TLS, store, admin
|
|
# token): any key of docs/reference.md, laid over the chart's. An offline
|
|
# cluster sets offline.mirror here, and offlineChecksums below.
|
|
config:
|
|
control:
|
|
enabled: true
|
|
auth:
|
|
enabled: true
|
|
signup_open: false
|
|
default_rpm: 60
|
|
default_tpm: 200000
|
|
default_price:
|
|
prompt_per_1k: 1000
|
|
completion_per_1k: 1000
|
|
max_key_ttl: 2160h
|
|
# mirror:
|
|
# api: https://models.internal:8082
|
|
# offline:
|
|
# mirror: http://models.internal:8081
|
|
# firewall_networks: ["10.42.0.0/16"] # the pods' network, so nodes' engines answer the router
|
|
|
|
# On an offline cluster, a ConfigMap or Secret holding the engine checksums
|
|
# 'llmgw bundle trust' writes (offline.sha256), mounted where offline.checksums
|
|
# points. Empty mounts none.
|
|
offlineChecksums:
|
|
configMap: ""
|
|
key: offline.sha256
|
|
|
|
# The OpenAI-compatible API and the metrics, inside the cluster.
|
|
service:
|
|
type: ClusterIP
|
|
port: 8080
|
|
metricsPort: 9091
|
|
annotations: {}
|
|
|
|
# Where agents connect, with mutual TLS. Nodes run outside the cluster, so
|
|
# this is a load balancer by default; the TLS ends in the pod, never at a
|
|
# proxy in between.
|
|
agentService:
|
|
type: LoadBalancer
|
|
port: 7443
|
|
annotations: {}
|
|
loadBalancerIP: ""
|
|
loadBalancerSourceRanges: []
|
|
# Keeps the agents' addresses as they are, which the node's identity does
|
|
# not need but the logs and the audit read better with.
|
|
externalTrafficPolicy: Local
|
|
|
|
ingress:
|
|
enabled: false
|
|
className: ""
|
|
annotations: {}
|
|
hosts:
|
|
- host: llm.example.com
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
tls: []
|
|
# - secretName: llm-example-com
|
|
# hosts: [llm.example.com]
|
|
|
|
# A ServiceMonitor for the Prometheus Operator, scraping the metrics port.
|
|
serviceMonitor:
|
|
enabled: false
|
|
interval: 30s
|
|
labels: {}
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 500m
|
|
memory: 512Mi
|
|
limits:
|
|
memory: 2Gi
|
|
|
|
nodeSelector: {}
|
|
tolerations: []
|
|
affinity: {}
|
|
podAnnotations: {}
|
|
podLabels: {}
|