Files
ilkerandClaude Opus 5.5 3a64678638 hpcod: publish osu-micro-benchmarks, and read an image's scheme from the registry
The catalogue assumed oras:// for any image it had no record of, which put
an OCI image in it under an ORAS reference; every head that pulled it
failed with "invalid layer mediatype". The scheme is now read from the
registry's manifest - a SIF layer is oras://, an OCI layer docker:// - and
an image whose shape cannot be read is left out and named, rather than
guessed.

build-push.sh checks before building that it has somewhere to write the
catalogue, and after pushing that what it pushed is in it under the
reference it was pushed as. osu-micro-benchmarks is built, pushed and
pulled back, and offered.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 18:19:51 +01:00

483 lines
20 KiB
Python
Executable File

#!/usr/bin/env python3
"""Normalise the application recipes, and generate the catalogue from them.
Split out of build-push.sh once a recipe could hold more than one version.
Reading one key at a time out of a YAML file with a shell function was
tolerable while an application was a single version; it is not once every
field can be stated per version or shared across them, and the shell has no
type that holds the result.
Two subcommands:
versions <dir> one TSV line per version of one application, for the
build loop to iterate over
catalog <dir> <out> <registry> <owner>
the whole catalogue, checked against the registry
A recipe is image/apps/<name>/app.yaml. It may state one version at the top
level, which is what every recipe did before this and still does:
name: stress-ng
version: "0.18.07"
published: true
or a list, where the top level holds what the versions share:
name: gromacs
module: gromacs
binaries: [gmx_mpi]
versions:
- version: "2024.4"
recipe: 2024.4 # a subdirectory with its own spack.yaml
published: false
- version: "2025.1"
recipe: 2025.1
published: true
Both are read into the same shape, so nothing downstream knows which was
written. That is the point: an application with one version and an
application with four differ in their recipe and nowhere else.
"""
import json
import os
import sys
import urllib.error
import urllib.request
import yaml
# Fields a version may state for itself. Anything else is shared by every
# version of the application - a description or a module name that differed
# between versions would be two applications wearing one name.
PER_VERSION = ('version', 'tag', 'image', 'upstream', 'recipe', 'published',
'size_gb', 'build_args', 'runtime_image', 'spack_image', 'status',
'status_note', 'sif_sha256')
SHARED = ('description', 'homepage', 'module', 'binaries', 'requires',
'mpi', 'license', 'signed')
# What was published, as opposed to what the recipes intend to publish.
#
# Kept apart from app.yaml deliberately. A recipe is hand-written and carries
# the reasoning for every choice in it; rewriting it from a script to record a
# hash would throw those comments away the first time an image was pushed.
# So: app.yaml is intent, published.json is measurement, and the generator
# joins them.
def published_path(here):
return os.path.join(here, 'published.json')
def published(here):
try:
with open(published_path(here)) as fh:
return json.load(fh)
except (OSError, ValueError):
return {}
def record(here, name, tag, sha256, dig='', image=''):
"""Note what a push actually produced, and where it put it.
The URI is recorded rather than derived, because it is not always the
same shape. With Apptainer on the build host an image is published as a
SIF over ORAS; without it the OCI image is pushed as it stands and the
head converts it. A catalogue that assumed one and got the other would
name a reference the head cannot resolve - and would find out at the
last task of a provision, after the machines were up.
"""
p = published_path(here)
db = published(here)
db[f"{name}:{tag}"] = {'sif_sha256': sha256, 'digest': dig, 'image': image}
tmp = p + '.new'
with open(tmp, 'w') as fh:
json.dump(db, fh, indent=2, sort_keys=True)
fh.write('\n')
os.replace(tmp, p)
print(f" recorded {name}:{tag} sha256:{sha256[:16]}")
def load(spec):
with open(spec) as fh:
return yaml.safe_load(fh) or {}
def versions(app_dir):
"""Every version of one application, as a list of flat dicts."""
a = load(os.path.join(app_dir, 'app.yaml'))
name = a.get('name') or os.path.basename(app_dir.rstrip('/'))
shared = {k: a[k] for k in SHARED if k in a}
raw = a.get('versions')
if not raw:
# The single-version form. Everything at the top level describes the
# one version there is.
raw = [{k: a[k] for k in PER_VERSION if k in a}]
out = []
for v in raw:
e = dict(shared)
# A shared default that a version may override: `published: false` at
# the top level parks a whole application without editing each entry.
for k in PER_VERSION:
if k in a and k not in ('version', 'tag', 'image'):
e[k] = a[k]
e.update(v)
e['name'] = name
e['version'] = str(e.get('version', ''))
e['tag'] = str(e.get('tag') or e['version'] or 'latest')
e['dir'] = os.path.join(app_dir, str(e['recipe'])) if e.get('recipe') else app_dir
e['status'] = normalise_status(e)
out.append(e)
return out
def normalise_status(e):
"""Where this version is in its life.
`published: true/false` was the old spelling and is still read, because
two ways to say the same thing in one file is worse than one awkward
one - but it only ever had two values, and the two that matter after
publication were the missing ones:
draft no image yet; left out of the catalogue entirely
available the normal state
deprecated a newer version exists; still offered, still supported
withdrawn do not start anything new on this; the image stays in the
registry, because clusters holding it may be rebuilt
"""
st = str(e.get('status') or '').strip().lower()
if st in ('draft', 'available', 'deprecated', 'withdrawn'):
return st
if st:
raise SystemExit(f"unknown status {st!r} for {e.get('name')}:{e.get('version')}")
return 'available' if e.get('published') else 'draft'
def all_versions(here):
for name in sorted(os.listdir(here)):
d = os.path.join(here, name)
if os.path.isfile(os.path.join(d, 'app.yaml')):
yield from versions(d)
# --- the registry ----------------------------------------------------------
def _token(host, repo, timeout=10):
return json.load(urllib.request.urlopen(
f"https://{host}/v2/token?scope=repository:{repo}:pull"
f"&service=container_registry", timeout=timeout)).get('token', '')
# What a SIF pushed over ORAS looks like from the outside. An OCI image
# carries tar+gzip layers instead, and Apptainer refuses to pull one of those
# through an oras:// reference: "invalid layer mediatype". That is the whole
# difference between the two schemes, and it is visible in the manifest -
# which is why this file reads it rather than assuming.
SIF_LAYER = 'application/vnd.sylabs.sif.layer.v1.sif'
def manifest(host, repo, tag, timeout=10):
"""What the tag points at, and which scheme can pull it.
Returns (digest, scheme). The scheme is 'oras' for a SIF pushed as one,
'docker' for an OCI image, and '' when the registry could not be asked or
answered with something that is neither.
A tag is mutable. `oras://…/gromacs:2024.4` names whatever was last
pushed under that name, so a cluster rebuilt after somebody repushed it
gets different bytes than the cluster that was allocated - and the record
that was supposed to freeze the cluster freezes nothing.
Recorded rather than pulled by: whether this Apptainer accepts an
`@sha256:` reference is not established, and a provision that fails at
its last task because of a syntax nobody tested would be worse than the
problem. What the head can do with no network and no version dependency
is check the file it ended up with, which is what apps_digest is for.
A GET rather than the HEAD this used to do. The digest comes back in the
same header either way; the body is what says which of the two shapes is
up there. One round trip, one more thing known.
"""
try:
req = urllib.request.Request(
f"https://{host}/v2/{repo}/manifests/{tag}",
headers={'Authorization': f'Bearer {_token(host, repo, timeout)}',
'Accept': 'application/vnd.oci.image.manifest.v1+json, '
'application/vnd.docker.distribution.manifest.v2+json'})
with urllib.request.urlopen(req, timeout=timeout) as r:
dig = r.headers.get('Docker-Content-Digest') or ''
body = json.load(r)
except Exception:
return '', ''
layers = [l.get('mediaType', '') for l in (body.get('layers') or [])]
if SIF_LAYER in layers:
return dig, 'oras'
if any(l.startswith('application/vnd.oci.image.layer.')
or l.startswith('application/vnd.docker.image.rootfs.') for l in layers):
return dig, 'docker'
return dig, ''
def tags(host, repo, timeout=10):
"""What the registry actually holds for this repository.
An entry in the catalogue is a promise that a head node can pull the
image. Until this asked, that promise was a hand-written `published:
true`, and the first generated catalogue pointed busybox at a tag that
had never existed - guessed from the upstream version rather than read
from the registry. The head would have found out at the last task of a
provision, after the machines were up and the lease had started.
None means the registry could not be asked. That is not an empty list
and the caller must not treat it as one.
"""
try:
tok = _token(host, repo, timeout)
req = urllib.request.Request(
f"https://{host}/v2/{repo}/tags/list",
headers={'Authorization': f'Bearer {tok}'})
return json.load(urllib.request.urlopen(req, timeout=timeout)).get('tags') or []
except urllib.error.HTTPError as e:
if e.code == 404:
return [] # the repository does not exist: a real answer
return None
except Exception:
return None
# Unit separator, not tab.
#
# bash's `read` treats tab as IFS whitespace, so two of them in a row are one
# delimiter - and an application with no upstream image, which is most of
# them, shifted every field after it one place left. The build then took the
# binary list as the upstream reference and the status as the binary list,
# and failed with "image is missing available". An awk -F'\t' check of the
# same output looked perfectly correct, because awk does not collapse.
SEP = '\x1f'
def cmd_versions(app_dir):
for e in versions(app_dir):
args = ' '.join(f"{k}={v}" for k, v in (e.get('build_args') or {}).items())
print(SEP.join(str(x) for x in (
e['name'], e['version'], e['tag'], e['dir'],
e.get('upstream') or '', ' '.join(e.get('binaries') or []),
args, e.get('runtime_image') or '', e.get('spack_image') or '',
e['status'])))
def previously(out):
"""The references the last generated catalogue carried, by name:version.
A third source of measurement, and the one that survives a machine. The
push record lives beside the build that made it - on the host that ran
it, in a directory that gets re-copied - so a checkout elsewhere, or the
same checkout after that copy, knows nothing about a push that happened.
The catalogue does: every reference in it was written by a run that had
asked the registry.
Only the reference is taken from it. The digest is not, because the whole
point of the digest is that it is what the tag points at *now*.
"""
try:
with open(out) as fh:
old = yaml.safe_load(fh) or {}
except (OSError, ValueError):
return {}
was = {}
for a in (old.get('apps') or []):
if a.get('name') and a.get('image'):
was[f"{a['name']}:{a.get('version', '')}"] = a['image']
return was
def cmd_catalog(here, out, registry, owner):
offline = os.environ.get('CATALOG_OFFLINE') == '1'
pub = published(here)
was = previously(out)
apps, skipped, unchecked, nodigest, withdrawn = [], [], [], [], []
corrected = []
seen = set()
for e in all_versions(here):
ref = f"{e['name']}:{e['version']}"
if e['status'] == 'draft':
# No image exists, so there is nothing to promise. Left out
# rather than marked, because an entry in this file is a promise
# a head node can pull it.
skipped.append(f"{ref} (draft: no image published)")
continue
if ref in seen:
# Two entries under one name and version would make the pinned
# reference ambiguous, and Lookup would pick whichever sorted
# last. Refusing is better than choosing.
print(f"ERROR: {ref} is declared twice", file=sys.stderr)
return 1
seen.add(ref)
entry = {
'name': e['name'],
'version': e['version'],
# Recorded for every entry, not only the unusual ones: a reader
# of this file should not have to know that a missing status
# means available.
'status': e['status'],
'description': e.get('description', ''),
# What was published, then what the recipe asked for, then the
# default. The measurement wins: it is the only one that was
# observed rather than intended.
# Filled in below, once the registry has said which scheme
# can actually pull it. Named here so it keeps its place in
# the file: a reader looks for it next to the description.
'image': '',
'signed': bool(e.get('signed', False)),
}
if e.get('requires'):
entry['requires'] = list(e['requires'])
sif = os.path.join(here, '.build', f"{e['name']}-{e['tag']}.sif")
legacy = os.path.join(here, '.build', f"{e['name']}.sif")
for p in (sif, legacy):
if os.path.exists(p):
entry['size_gb'] = round(os.path.getsize(p) / 1024**3, 2)
break
else:
if e.get('size_gb'):
entry['size_gb'] = float(e['size_gb'])
for k in ('module', 'binaries', 'mpi', 'license', 'homepage',
'status_note'):
if e.get(k):
entry[k] = e[k]
# The measurement wins over anything the recipe declared: this is the
# hash of the file that was actually pushed, and a recipe cannot know
# it in advance.
if pub.get(ref, {}).get('sif_sha256'):
entry['sif_sha256'] = pub[ref]['sif_sha256']
elif e.get('sif_sha256'):
entry['sif_sha256'] = e['sif_sha256']
if e['status'] == 'withdrawn':
# Kept in the catalogue on purpose. It is the record of what has
# been published, and dropping a withdrawn entry would make it
# indistinguishable from one that never existed - which is the
# difference an operator needs when a cluster is found still
# holding it. The portal is what stops it being offered.
withdrawn.append(ref)
# Where the image is, in two parts, because only one of them can be
# worked out from a name.
#
# The location - host, repository, tag - follows from the convention
# when nothing recorded it. The scheme does not. It depends on
# whether the build host had Apptainer on the afternoon this was
# pushed, and on nothing else, so assuming oras:// is what put an OCI
# image in this catalogue under an ORAS reference. Every head that
# tried to pull it failed with "invalid layer mediatype"; the
# first-boot playbook died at the apps role; and the portal that runs
# after it was never configured, which is how one unpublished image
# cost a whole cluster its front door.
#
# So the scheme is read from the registry below, and where it cannot
# be read it is not invented.
stated = (pub.get(ref, {}).get('image') or was.get(ref)
or e.get('image') or '')
scheme, _, location = stated.rpartition('://')
if not location:
location = f"{registry}/{owner}/{e['name']}:{e['tag']}"
loc, _, want = location.rpartition(':')
host, _, repo = loc.partition('/')
if offline:
unchecked.append(ref)
else:
have = tags(host, repo)
if have is None:
unchecked.append(ref)
elif want not in have:
found = ', '.join(sorted(have)) if have else 'nothing'
skipped.append(f"{ref} ({repo}:{want} is not in the registry; it holds {found})")
continue
else:
# What the tag points at today, and what shape it is in. The
# digest is carried to the cluster record so a rebuild can
# tell whether the tag has moved under it.
d, holds = manifest(host, repo, want)
if d:
entry['digest'] = d
else:
nodigest.append(ref)
if holds and holds != scheme:
if scheme:
corrected.append(
f"{ref} ({scheme}:// was recorded; the registry holds "
f"a {holds}:// artefact)")
scheme = holds
if not scheme:
# Nothing recorded it and the registry did not say. A reference
# picked here would be a coin toss settled on a tenant's head
# node, at the last task of a provision, with the machines
# already up - so the entry is left out and named instead.
skipped.append(f"{ref} (nothing says whether {location} is a SIF or an "
f"OCI image, so no reference can be stated; republish it)")
continue
entry['image'] = f"{scheme}://{location}"
apps.append(entry)
apps.sort(key=lambda a: (a['name'], a['version']))
header = (
"# Application catalogue. GENERATED by image/apps/build-push.sh --catalog\n"
"# from the app.yaml beside each image recipe, and checked against the\n"
"# registry. Do not edit: an entry here is a promise that a head node\n"
"# can pull the image, and this file is the only place that promise is\n"
"# checked rather than declared.\n"
"#\n"
"# The portal reads it via `hpcod serve -apps config/apps.yaml` and\n"
"# re-reads it when it changes, so publishing needs no restart.\n\n"
)
with open(out, 'w') as fh:
fh.write(header)
yaml.safe_dump({'apps': apps}, fh, sort_keys=False,
default_flow_style=False, width=100)
print(f"{len(apps)} entries -> {out}")
for line in sorted(skipped):
print(f" not offered: {line}")
if withdrawn:
print(" withdrawn, kept for clusters that already hold them: "
+ ", ".join(sorted(withdrawn)))
if corrected:
# Said out loud, every time. A recorded URI and the registry
# disagreeing means a push was made one way and noted another, and a
# catalogue that quietly papered over it is how the wrong one
# survived long enough to break a provision.
print(" scheme corrected from the registry:")
for line in sorted(corrected):
print(f" {line}")
if nodigest:
print(" no digest recorded for: " + ", ".join(sorted(nodigest)))
print(" a rebuild cannot tell whether the tag has moved.")
if unchecked:
# Loud, and not a skip. Dropping every entry because a laptop is on
# a train would turn an unreachable registry into an empty catalogue.
print(" WARNING: the registry was not asked about: " + ", ".join(sorted(unchecked)))
print(" these are in the catalogue on their declared word alone.")
return 0
if __name__ == '__main__':
if len(sys.argv) < 2:
print(__doc__, file=sys.stderr)
sys.exit(2)
if sys.argv[1] == 'versions':
cmd_versions(sys.argv[2])
elif sys.argv[1] == 'record':
record(*sys.argv[2:8])
elif sys.argv[1] == 'catalog':
sys.exit(cmd_catalog(*sys.argv[2:6]))
else:
print(f"unknown subcommand: {sys.argv[1]}", file=sys.stderr)
sys.exit(2)