An installation used to be this one: the deploy scripts and the Ansible defaults carried one site's domain, hypervisor and login, the Proxmox side and the database were set up by hand, and deploying meant copying scripts and a tree of roles beside the binary and keeping them in step with it. The binary now carries the roles, the Terraform module, the schema and the catalogue (embed.go), and installs itself: - hpcod setup: a full-screen installer for administrators. It asks where clusters are built, signs in to Proxmox before asking anything else (root's password once, or tokens), offers nodes and datastores from what Proxmox has, checks every field as it is typed, runs every step with its output live, checks the result and offers a test cluster. -yes runs the same steps from a site file; make deploy uses that. - hpcod verify: each fact checked by using it. - hpcod selftest: the smallest real cluster, by the portal's own path, a job run on it, then taken down. - Closed networks: hpcod serves HTTPS itself from a CA name-constrained to the site's domains, issuing and renewing its own certificate, and answers DNS for the portal and every cluster name. - A simulated Proxmox: Proxmox VE in QEMU on the control plane host, on a private bridge with NAT, for trying it without one. Every step reads back what it changed and is safe to run again. The site's values live in one file; group_vars/all/site.yml is generated from it, and the playbooks refuse to run without it. ci_user and -ssh-user no longer default to a personal account. scripts/B10, B20 and C20 are gone: their work is the setup steps. Proven end to end on a simulated Proxmox (Proxmox VE 9 in a VM on a real host): the install, the Proxmox step by the password path, the head template, verify READY, and a test cluster that ran srun on its compute node and was destroyed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
256 lines
7.0 KiB
Go
256 lines
7.0 KiB
Go
package setup
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// PVE is the small part of the Proxmox API setup needs. It is separate from
|
|
// the provider's client because that one only reads, with a token, and
|
|
// this one also writes - users, tokens, the SDN zone, storage - and can
|
|
// sign in with root's password, once, during the first install.
|
|
type PVE struct {
|
|
base string
|
|
http *http.Client
|
|
token string // "user@realm!id=secret"
|
|
ticket string
|
|
csrf string
|
|
}
|
|
|
|
// PVEError is a refusal from the API, with its HTTP status.
|
|
type PVEError struct {
|
|
Status int
|
|
Path string
|
|
Msg string
|
|
}
|
|
|
|
func (e *PVEError) Error() string {
|
|
return fmt.Sprintf("%s: HTTP %d: %s", e.Path, e.Status, e.Msg)
|
|
}
|
|
|
|
func newPVE(endpoint string, insecure bool) *PVE {
|
|
return &PVE{
|
|
base: strings.TrimRight(endpoint, "/") + "/api2/json",
|
|
http: &http.Client{
|
|
Timeout: 30 * time.Second,
|
|
Transport: &http.Transport{
|
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, //nolint:gosec // opt-in, per site
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// PVEWithToken talks to the API with an API token.
|
|
func PVEWithToken(endpoint string, insecure bool, token string) *PVE {
|
|
p := newPVE(endpoint, insecure)
|
|
p.token = token
|
|
return p
|
|
}
|
|
|
|
// PVEWithPassword signs in and keeps the ticket. The password is not kept.
|
|
func PVEWithPassword(ctx context.Context, endpoint string, insecure bool, user, password string) (*PVE, error) {
|
|
p := newPVE(endpoint, insecure)
|
|
var out struct {
|
|
Ticket string `json:"ticket"`
|
|
CSRF string `json:"CSRFPreventionToken"`
|
|
}
|
|
if err := p.do(ctx, http.MethodPost, "/access/ticket",
|
|
url.Values{"username": {user}, "password": {password}}, &out); err != nil {
|
|
return nil, err
|
|
}
|
|
if out.Ticket == "" {
|
|
return nil, fmt.Errorf("Proxmox returned no ticket for %s", user)
|
|
}
|
|
p.ticket, p.csrf = out.Ticket, out.CSRF
|
|
return p, nil
|
|
}
|
|
|
|
func (p *PVE) Get(ctx context.Context, path string, out any) error {
|
|
return p.do(ctx, http.MethodGet, path, nil, out)
|
|
}
|
|
|
|
func (p *PVE) Post(ctx context.Context, path string, form url.Values, out any) error {
|
|
return p.do(ctx, http.MethodPost, path, form, out)
|
|
}
|
|
|
|
func (p *PVE) Put(ctx context.Context, path string, form url.Values) error {
|
|
return p.do(ctx, http.MethodPut, path, form, nil)
|
|
}
|
|
|
|
func (p *PVE) Delete(ctx context.Context, path string) error {
|
|
return p.do(ctx, http.MethodDelete, path, nil, nil)
|
|
}
|
|
|
|
func (p *PVE) do(ctx context.Context, method, path string, form url.Values, out any) error {
|
|
var body io.Reader
|
|
if form != nil {
|
|
body = strings.NewReader(form.Encode())
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, method, p.base+path, body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if form != nil {
|
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
}
|
|
switch {
|
|
case p.token != "":
|
|
req.Header.Set("Authorization", "PVEAPIToken="+p.token)
|
|
case p.ticket != "":
|
|
req.AddCookie(&http.Cookie{Name: "PVEAuthCookie", Value: p.ticket})
|
|
if method != http.MethodGet {
|
|
req.Header.Set("CSRFPreventionToken", p.csrf)
|
|
}
|
|
}
|
|
resp, err := p.http.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
|
if resp.StatusCode/100 != 2 {
|
|
// Proxmox puts the reason in the status line, and the per-field
|
|
// detail in the body's "errors".
|
|
msg := resp.Status
|
|
var e struct {
|
|
Errors map[string]string `json:"errors"`
|
|
}
|
|
if json.Unmarshal(raw, &e) == nil && len(e.Errors) > 0 {
|
|
var parts []string
|
|
for k, v := range e.Errors {
|
|
parts = append(parts, k+": "+strings.TrimSpace(v))
|
|
}
|
|
msg += " (" + strings.Join(parts, "; ") + ")"
|
|
}
|
|
return &PVEError{Status: resp.StatusCode, Path: path, Msg: msg}
|
|
}
|
|
if out == nil {
|
|
return nil
|
|
}
|
|
var env struct {
|
|
Data json.RawMessage `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(raw, &env); err != nil {
|
|
return fmt.Errorf("%s: %w", path, err)
|
|
}
|
|
return json.Unmarshal(env.Data, out)
|
|
}
|
|
|
|
// WaitTask follows a task until it stops, and fails if it did not end OK.
|
|
func (p *PVE) WaitTask(ctx context.Context, node, upid string, limit time.Duration) error {
|
|
deadline := time.Now().Add(limit)
|
|
for {
|
|
var st struct {
|
|
Status string `json:"status"`
|
|
ExitStatus string `json:"exitstatus"`
|
|
}
|
|
if err := p.Get(ctx, "/nodes/"+node+"/tasks/"+url.PathEscape(upid)+"/status", &st); err != nil {
|
|
return err
|
|
}
|
|
if st.Status == "stopped" {
|
|
if st.ExitStatus != "OK" {
|
|
return fmt.Errorf("task %s ended: %s", upid, st.ExitStatus)
|
|
}
|
|
return nil
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return fmt.Errorf("task %s still running after %s", upid, limit)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(3 * time.Second):
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- what setup asks of it ----------------------------------------------------
|
|
|
|
// Version is the major version of pve-manager.
|
|
func (p *PVE) Version(ctx context.Context) (string, int, error) {
|
|
var v struct {
|
|
Version string `json:"version"`
|
|
}
|
|
if err := p.Get(ctx, "/version", &v); err != nil {
|
|
return "", 0, err
|
|
}
|
|
major := 0
|
|
fmt.Sscanf(v.Version, "%d", &major)
|
|
return v.Version, major, nil
|
|
}
|
|
|
|
// Nodes lists the cluster's node names.
|
|
func (p *PVE) Nodes(ctx context.Context) ([]string, error) {
|
|
var ns []struct {
|
|
Node string `json:"node"`
|
|
Status string `json:"status"`
|
|
}
|
|
if err := p.Get(ctx, "/nodes", &ns); err != nil {
|
|
return nil, err
|
|
}
|
|
var out []string
|
|
for _, n := range ns {
|
|
out = append(out, n.Node)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// StorageContent is the content types a datastore holds, as Proxmox lists
|
|
// them ("images,rootdir,snippets").
|
|
func (p *PVE) StorageContent(ctx context.Context, storage string) ([]string, error) {
|
|
var s struct {
|
|
Content string `json:"content"`
|
|
}
|
|
if err := p.Get(ctx, "/storage/"+storage, &s); err != nil {
|
|
return nil, err
|
|
}
|
|
var out []string
|
|
for _, c := range strings.Split(s.Content, ",") {
|
|
if c = strings.TrimSpace(c); c != "" {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Volumes lists a datastore's volume ids of one content type on a node.
|
|
func (p *PVE) Volumes(ctx context.Context, node, storage, content string) ([]string, error) {
|
|
var vs []struct {
|
|
VolID string `json:"volid"`
|
|
}
|
|
if err := p.Get(ctx, "/nodes/"+node+"/storage/"+storage+"/content?content="+content, &vs); err != nil {
|
|
return nil, err
|
|
}
|
|
var out []string
|
|
for _, v := range vs {
|
|
out = append(out, v.VolID)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// CreateToken makes a fresh token for user and returns "user!id=secret".
|
|
// One that exists is replaced: its secret was shown once, when it was made,
|
|
// and cannot be had again.
|
|
func (p *PVE) CreateToken(ctx context.Context, user, id, comment string) (string, error) {
|
|
path := "/access/users/" + url.PathEscape(user) + "/token/" + url.PathEscape(id)
|
|
_ = p.Delete(ctx, path) // absent is fine
|
|
var out struct {
|
|
FullID string `json:"full-tokenid"`
|
|
Value string `json:"value"`
|
|
}
|
|
if err := p.Post(ctx, path, url.Values{"privsep": {"0"}, "comment": {comment}}, &out); err != nil {
|
|
return "", err
|
|
}
|
|
if out.Value == "" {
|
|
return "", fmt.Errorf("Proxmox returned no secret for %s!%s", user, id)
|
|
}
|
|
return out.FullID + "=" + out.Value, nil
|
|
}
|