The catalogue assumed oras:// for any image it had no record of, which put an OCI image in it under an ORAS reference; every head that pulled it failed with "invalid layer mediatype". The scheme is now read from the registry's manifest - a SIF layer is oras://, an OCI layer docker:// - and an image whose shape cannot be read is left out and named, rather than guessed. build-push.sh checks before building that it has somewhere to write the catalogue, and after pushing that what it pushed is in it under the reference it was pushed as. osu-micro-benchmarks is built, pushed and pulled back, and offered. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
376 lines
16 KiB
Bash
Executable File
376 lines
16 KiB
Bash
Executable File
#!/bin/bash
|
|
# Build application SIFs and publish them to the registry the catalogue points at.
|
|
#
|
|
# ./build-push.sh # every application whose SIF is missing or stale
|
|
# ./build-push.sh gromacs lammps # named ones, always rebuilt
|
|
# ./build-push.sh --catalog # regenerate config/apps.yaml and stop
|
|
# ./build-push.sh --no-push # build and verify locally, publish nothing
|
|
#
|
|
# Run on the control plane host, as the account that owns the podman store.
|
|
# See image/build-push.sh for why rootless podman needs /etc/subuid and
|
|
# `loginctl enable-linger`; the same applies here and fails the same way.
|
|
#
|
|
# Three recipes, decided by what is in the application's directory:
|
|
#
|
|
# upstream: in app.yaml somebody else's image, converted to a SIF
|
|
# Containerfile an image we assemble
|
|
# spack.yaml compiled from source against our MPI
|
|
#
|
|
# Nothing here trusts an exit code on its own. Every image is opened and
|
|
# asked for its binaries before it is pushed, and every push is pulled back
|
|
# and compared by digest before the catalogue is told it exists.
|
|
set -euo pipefail
|
|
|
|
HERE=$(cd "$(dirname "$0")" && pwd)
|
|
REGISTRY=${REGISTRY:-gitea.mynodes.xyz}
|
|
OWNER=${OWNER:-hpcapps}
|
|
OUT=${OUT:-${HERE}/.build}
|
|
# Where the generated catalogue goes. Two levels up from here is the
|
|
# repository's config/, which is right in a checkout and wrong everywhere
|
|
# else: the control plane keeps a copy of this directory on its own, with no
|
|
# repository above it, and the default there resolved to /var/lib/config -
|
|
# a directory that does not exist. Nothing found out until after a build:
|
|
# the image was pushed and pulled back, the push was recorded, and then the
|
|
# regeneration died with FileNotFoundError. The catalogue kept saying what
|
|
# it had said the day before, about an image that had just changed shape,
|
|
# and the next cluster provisioned against it failed on the head node.
|
|
#
|
|
# So it is checked below, before anything is built, rather than discovered
|
|
# forty minutes in.
|
|
CATALOG=${CATALOG:-$(cd "$HERE/../.." && pwd)/config/apps.yaml}
|
|
|
|
# The build cache, in the same registry the images go to. Overridable, and
|
|
# emptied by setting it to "": an estate without one still builds, slowly.
|
|
SPACK_MIRROR=${SPACK_MIRROR-oci://${REGISTRY}/${OWNER}/spack-cache}
|
|
|
|
export REGISTRY_AUTH_FILE="${REGISTRY_AUTH_FILE:-$HOME/.config/containers/auth.json}"
|
|
export APPTAINER_CACHEDIR="${APPTAINER_CACHEDIR:-/var/cache/apptainer}"
|
|
|
|
PUSH=1
|
|
ONLY_CATALOG=0
|
|
APPS=()
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--catalog) ONLY_CATALOG=1 ;;
|
|
--no-push) PUSH=0 ;;
|
|
-*) echo "unknown option: $arg" >&2; exit 2 ;;
|
|
*) APPS+=("$arg") ;;
|
|
esac
|
|
done
|
|
|
|
die() { echo "ABORT: $*" >&2; exit 1; }
|
|
|
|
# Only the catalogue generator runs without a container stack, so that a
|
|
# checkout on a laptop can regenerate config/apps.yaml and commit it.
|
|
#
|
|
# Apptainer is optional, and this is not a concession - image/desktop has
|
|
# been published without it since it was written. With it, an image is
|
|
# converted here and published as a SIF over ORAS, so a head node pulls
|
|
# bytes and nothing else. Without it, the OCI image is pushed as it stands
|
|
# and the head converts it on arrival with `apptainer pull docker://`.
|
|
#
|
|
# What is lost in the second case is the SIF hash: the conversion happens on
|
|
# the head, so there is no file here whose digest could be recorded, and the
|
|
# head has nothing to compare what it produced against. The manifest digest
|
|
# is still recorded either way. The catalogue says which, the apps role says
|
|
# out loud when it cannot check, and neither pretends otherwise.
|
|
SIF=1
|
|
if [ "$ONLY_CATALOG" != 1 ]; then
|
|
command -v podman >/dev/null || die "podman is not installed"
|
|
if ! command -v apptainer >/dev/null; then
|
|
SIF=0
|
|
echo "note: apptainer is not installed, so images are published as OCI and"
|
|
echo " converted on the head. They will carry no SIF hash, which means"
|
|
echo " a head cannot check that what it pulled is what was published."
|
|
fi
|
|
fi
|
|
python3 -c 'import yaml' 2>/dev/null || die "python3 needs PyYAML (it is already a dependency of B40-publish-cluster.py)"
|
|
|
|
# Asserted before the first build, not after the last push. A run that
|
|
# cannot write the catalogue cannot publish anything, however well it
|
|
# builds - the images land in the registry and no cluster is ever told.
|
|
[ -d "$(dirname "$CATALOG")" ] || die "nowhere to write the catalogue: $(dirname "$CATALOG") does not exist.
|
|
Pass CATALOG=/path/to/apps.yaml - this directory is a copy without a repository above it."
|
|
|
|
|
|
# --- reading the recipes ---------------------------------------------------
|
|
# Normalising a recipe belongs in recipes.py, not here: a version may be
|
|
# stated at the top level or in a list, and the shell has no type that holds
|
|
# the result of working that out.
|
|
RECIPES="${HERE}/recipes.py"
|
|
[ -x "$RECIPES" ] || die "recipes.py is missing beside this script"
|
|
|
|
regen_catalog() {
|
|
python3 "$RECIPES" catalog "$HERE" "$CATALOG" "$REGISTRY" "$OWNER"
|
|
}
|
|
|
|
if [ "$ONLY_CATALOG" = 1 ]; then
|
|
regen_catalog
|
|
# Read it back through the same loader the portal uses, so a file that
|
|
# parses as YAML but not as a catalogue fails here rather than at startup.
|
|
( cd "$(dirname "$CATALOG")/.." && go run ./cmd/hpcod apps validate "$CATALOG" 2>/dev/null ) \
|
|
|| echo " note: 'hpcod apps validate' is not implemented yet; catalogue not checked"
|
|
exit 0
|
|
fi
|
|
|
|
# --- which applications ----------------------------------------------------
|
|
NAMED=1
|
|
if [ ${#APPS[@]} -eq 0 ]; then
|
|
NAMED=0
|
|
for d in "$HERE"/*/; do
|
|
[ -f "${d}app.yaml" ] && APPS+=("$(basename "$d")")
|
|
done
|
|
fi
|
|
[ ${#APPS[@]} -gt 0 ] || die "no applications found in $HERE"
|
|
|
|
mkdir -p "$OUT"
|
|
|
|
# --- build one version -----------------------------------------------------
|
|
# One call per version, not per application. An application with four
|
|
# versions published is four images, four tags and four catalogue entries,
|
|
# and the only thing they share is the recipe directory they came from.
|
|
build_version() {
|
|
local name=$1 version=$2 tag=$3 dir=$4 upstream=$5 binaries=$6
|
|
local build_args=$7 runtime_image=$8 spack_image=$9
|
|
local image="${REGISTRY}/${OWNER}/${name}:${tag}"
|
|
# The tag is in the filename: two versions of one application built in the
|
|
# same run would otherwise be one file, and the second would silently be
|
|
# measured and pushed as the first.
|
|
local sif="${OUT}/${name}-${tag}.sif"
|
|
|
|
echo
|
|
echo "==> ${name}:${tag}"
|
|
|
|
if [ -n "$upstream" ]; then
|
|
# Somebody else's image, republished under our own name so that a
|
|
# cluster does not depend on a public registry being up, or on that tag
|
|
# still meaning the same thing next month.
|
|
echo " upstream: ${upstream}"
|
|
podman pull "$upstream"
|
|
podman tag "$upstream" "$image"
|
|
[ "$SIF" = 1 ] && apptainer build --force "$sif" "docker://${upstream}"
|
|
|
|
else
|
|
local containerfile="$dir/Containerfile"
|
|
if [ -f "$dir/spack.yaml" ]; then
|
|
# Render the two-stage build from the shared template, into the build
|
|
# directory rather than beside the spack.yaml, so nobody edits the copy
|
|
# instead of the template.
|
|
containerfile="${OUT}/${name}-${tag}.Containerfile"
|
|
# spack/rockylinux9 is tagged by Spack release, and Spack went to
|
|
# 1.x: there has never been a v0.23.1 tag on that repository, so
|
|
# this default could not pull and no spack.yaml recipe had ever
|
|
# been built through it. Pinned rather than :latest, because the
|
|
# concretizer is what decides the whole catalogue's ABI and it
|
|
# should not change under us between two applications.
|
|
sed -e "s|@SPACK_IMAGE@|${spack_image:-docker.io/spack/rockylinux9:1.2.2}|g" \
|
|
-e "s|@RUNTIME_IMAGE@|${runtime_image:-docker.io/rockylinux:9}|g" \
|
|
"$HERE/spack.Containerfile.in" > "$containerfile"
|
|
grep -q '@SPACK_IMAGE@' "$containerfile" && die "template substitution did not land"
|
|
fi
|
|
[ -f "$containerfile" ] || die "$name:$tag has no upstream:, Containerfile or spack.yaml in $dir"
|
|
|
|
# --network=host because Docker is installed alongside podman on the
|
|
# control plane and sets FORWARD DROP, which leaves podman's own bridge
|
|
# unable to reach anything. See README.md, "Before the first build".
|
|
local args=()
|
|
for kv in $build_args; do args+=(--build-arg "$kv"); done
|
|
|
|
# The build cache. Reading it needs no credential - the registry is
|
|
# public - so the mirror is always passed and a cold estate simply
|
|
# finds nothing there. Writing needs one, and it goes in as a mounted
|
|
# secret rather than a build argument: arguments are recorded in the
|
|
# image's metadata, and these images are published.
|
|
#
|
|
# Nothing passed this before, so the template read from a cache that
|
|
# nothing had ever written to. Every application rebuilt the whole of
|
|
# OpenMPI from source, half an hour each, for a dependency tree they
|
|
# all share.
|
|
if [ -n "${SPACK_MIRROR:-}" ]; then
|
|
args+=(--build-arg "SPACK_MIRROR=${SPACK_MIRROR}")
|
|
local tokfile=""
|
|
if [ -n "${REGISTRY_TOKEN:-}" ]; then
|
|
tokfile=$(mktemp "${OUT}/.mirror-token.XXXXXX")
|
|
chmod 600 "$tokfile"
|
|
printf '%s' "${REGISTRY_TOKEN}" > "$tokfile"
|
|
args+=(--secret "id=mirror_token,src=${tokfile}")
|
|
else
|
|
echo " build cache: read-only (set REGISTRY_TOKEN to populate it)"
|
|
fi
|
|
fi
|
|
|
|
podman build --network=host --squash-all \
|
|
-f "$containerfile" "${args[@]}" -t "$image" "$dir"
|
|
[ -n "${tokfile:-}" ] && rm -f "$tokfile"
|
|
|
|
if [ "$SIF" = 1 ]; then
|
|
# Via an OCI archive rather than docker-daemon://: rootless podman's
|
|
# store is not readable by apptainer, and the error names the wrong
|
|
# thing.
|
|
local oci="${OUT}/${name}-${tag}.oci.tar"
|
|
podman save --format oci-archive -o "$oci" "$image"
|
|
apptainer build --force "$sif" "oci-archive://${oci}"
|
|
rm -f "$oci"
|
|
fi
|
|
fi
|
|
|
|
# --- verify ---
|
|
# Open the image and ask it for the binaries the catalogue promises. An
|
|
# image that built is not an image that runs, and the first sign of the
|
|
# difference would otherwise be a user's job failing after it queued and
|
|
# started. Asked of whichever artefact was actually produced, so the check
|
|
# is of the thing that will be published rather than of a proxy for it.
|
|
local missing=()
|
|
if [ "$SIF" = 1 ]; then
|
|
apptainer inspect "$sif" >/dev/null || die "${name}:${tag}: not a container"
|
|
for b in $binaries; do
|
|
apptainer exec "$sif" sh -c "command -v $b" >/dev/null 2>&1 || missing+=("$b")
|
|
done
|
|
else
|
|
for b in $binaries; do
|
|
podman run --rm --network=none "$image" sh -c "command -v $b" >/dev/null 2>&1 || missing+=("$b")
|
|
done
|
|
fi
|
|
[ ${#missing[@]} -eq 0 ] || die "${name}:${tag}: image is missing ${missing[*]}"
|
|
echo " binaries present: ${binaries:-none declared}"
|
|
if [ "$SIF" = 1 ]; then
|
|
echo " size: $(du -h "$sif" | cut -f1)"
|
|
else
|
|
podman image inspect "$image" --format ' size: {{.Size}} bytes'
|
|
fi
|
|
|
|
[ "$PUSH" = 1 ] || { echo " not pushing (--no-push)"; return 0; }
|
|
|
|
# A token, or a login that is already there. The control plane keeps one
|
|
# in the service account's REGISTRY_AUTH_FILE and has done since the node
|
|
# image was first published, so demanding the token again would mean
|
|
# fetching a secret that is already on the machine - and typing it into a
|
|
# shell history to do it.
|
|
local login=""
|
|
if [ -n "${REGISTRY_TOKEN:-}" ]; then
|
|
login=token
|
|
elif podman login --get-login "${REGISTRY}" >/dev/null 2>&1; then
|
|
login=stored
|
|
echo " using the stored login for ${REGISTRY} (${REGISTRY_AUTH_FILE})"
|
|
else
|
|
echo " not pushing: no REGISTRY_TOKEN and no stored login for ${REGISTRY}"
|
|
return 0
|
|
fi
|
|
|
|
# --- publish ---
|
|
local uri hash=""
|
|
if [ "$SIF" = 1 ]; then
|
|
if [ "$login" = token ]; then
|
|
echo "${REGISTRY_TOKEN}" | apptainer registry login --username "${REGISTRY_USER:-hpcod}" \
|
|
--password-stdin "oras://${REGISTRY}" >/dev/null
|
|
fi
|
|
apptainer push "$sif" "oras://${image}"
|
|
uri="oras://${image}"
|
|
|
|
# Read it back. A push that printed no error has not necessarily left
|
|
# anything a puller can fetch - and the head node is the one that finds
|
|
# out.
|
|
local back="${OUT}/.readback-${name}-${tag}.sif"
|
|
rm -f "$back"
|
|
apptainer pull --force --allow-unsigned "$back" "oras://${image}" >/dev/null
|
|
local b
|
|
hash=$(sha256sum "$sif" | cut -d' ' -f1)
|
|
b=$(sha256sum "$back" | cut -d' ' -f1)
|
|
rm -f "$back"
|
|
[ "$hash" = "$b" ] || die "${name}:${tag}: what came back from the registry is not what was pushed"
|
|
echo " published and pulled back: sha256:${hash:0:16}"
|
|
else
|
|
if [ "$login" = token ]; then
|
|
echo "${REGISTRY_TOKEN}" | podman login --username "${REGISTRY_USER:-hpcod}" \
|
|
--password-stdin "${REGISTRY}" >/dev/null
|
|
fi
|
|
podman push "$image"
|
|
uri="docker://${image}"
|
|
|
|
# The same read-back, by the only means available without a local file
|
|
# to compare: drop what we have and fetch it again. A push that left
|
|
# nothing pullable fails here rather than on a tenant's head node.
|
|
podman rmi "$image" >/dev/null 2>&1 || true
|
|
podman pull "$image" >/dev/null || die "${name}:${tag}: nothing came back from the registry"
|
|
echo " published and pulled back (OCI; the head converts it, so no SIF hash)"
|
|
fi
|
|
|
|
# Record what was actually produced, and where. The catalogue carries it to
|
|
# the cluster record; the hash is what lets a head notice a tag repushed
|
|
# underneath a rebuild, and is empty when the conversion happens there.
|
|
python3 "$RECIPES" record "$HERE" "$name" "$tag" "$hash" "" "$uri"
|
|
PUBLISHED+=("${name}:${tag}${SEP}${uri}")
|
|
}
|
|
|
|
# build_one walks the versions of one application. A version marked
|
|
# unpublished is still built when it is named explicitly - that is how a new
|
|
# version is tried before it is offered - but skipped in a whole-directory run.
|
|
build_one() {
|
|
local app=$1 named=$2 dir="$HERE/$1"
|
|
[ -f "$dir/app.yaml" ] || die "$app has no app.yaml"
|
|
local built=0
|
|
# \x1f, not tab: see the note in recipes.py. Tab is IFS whitespace, so
|
|
# an empty field would be swallowed and every column after it would shift.
|
|
while IFS=$'\x1f' read -r name version tag vdir upstream binaries args rimg simg published; do
|
|
[ -n "$name" ] || continue
|
|
if [ "" = "draft" ] && [ "" != "1" ]; then
|
|
echo " skipping ${name}:${tag} (draft)"
|
|
continue
|
|
fi
|
|
build_version "$name" "$version" "$tag" "$vdir" "$upstream" "$binaries" \
|
|
"$args" "$rimg" "$simg"
|
|
built=$((built + 1))
|
|
done < <(python3 "$RECIPES" versions "$dir")
|
|
[ "$built" -gt 0 ] || echo " ${app}: nothing to build"
|
|
}
|
|
|
|
# What this run pushed, and the reference it pushed it under. Checked against
|
|
# the catalogue at the end: a push that is recorded but not offered is the
|
|
# failure this whole script exists to prevent, and it is invisible from any
|
|
# exit code along the way.
|
|
PUBLISHED=()
|
|
# The same unit separator recipes.py uses, and for the same reason: a URI
|
|
# has colons in it and a tag can be empty.
|
|
SEP=$'\x1f'
|
|
|
|
FAILED=()
|
|
for app in "${APPS[@]}"; do
|
|
build_one "$app" "$NAMED" || FAILED+=("$app")
|
|
done
|
|
|
|
echo
|
|
if [ ${#FAILED[@]} -gt 0 ]; then
|
|
echo "FAILED: ${FAILED[*]}"
|
|
exit 1
|
|
fi
|
|
echo "==> all ${#APPS[@]} built"
|
|
echo " regenerating the catalogue"
|
|
regen_catalog
|
|
|
|
# Read the catalogue back and find what was just pushed in it, by the exact
|
|
# reference it was pushed under. The generator has every reason to leave an
|
|
# entry out - a tag the registry does not list, a shape it could not read -
|
|
# and it says so on its own output, which scrolls past at the end of a build
|
|
# nobody is watching. This turns that into an exit code.
|
|
# Guarded rather than written as a bare loop: under `set -u` an empty array
|
|
# expanded with "${a[@]}" is an unbound variable on older bash, and a run
|
|
# with nothing to push - every image already current - would die here
|
|
# instead of finishing.
|
|
missing=()
|
|
if [ ${#PUBLISHED[@]} -gt 0 ]; then
|
|
for line in "${PUBLISHED[@]}"; do
|
|
ref=${line%%"$SEP"*}
|
|
uri=${line#*"$SEP"}
|
|
grep -qF "image: ${uri}" "$CATALOG" || missing+=("${ref} -> ${uri}")
|
|
done
|
|
fi
|
|
if [ ${#missing[@]} -gt 0 ]; then
|
|
echo
|
|
echo "ABORT: pushed, but not in ${CATALOG}:" >&2
|
|
for m in "${missing[@]}"; do echo " $m" >&2; done
|
|
echo " A head node is told what to pull by this file alone, so these" >&2
|
|
echo " are published to nobody. The generator's output above says why." >&2
|
|
exit 1
|
|
fi
|
|
[ ${#PUBLISHED[@]} -eq 0 ] || echo " ${#PUBLISHED[@]} offered in ${CATALOG}"
|