The catalogue assumed oras:// for any image it had no record of, which put an OCI image in it under an ORAS reference; every head that pulled it failed with "invalid layer mediatype". The scheme is now read from the registry's manifest - a SIF layer is oras://, an OCI layer docker:// - and an image whose shape cannot be read is left out and named, rather than guessed. build-push.sh checks before building that it has somewhere to write the catalogue, and after pushing that what it pushed is in it under the reference it was pushed as. osu-micro-benchmarks is built, pushed and pulled back, and offered. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
483 lines
20 KiB
Python
Executable File
483 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Normalise the application recipes, and generate the catalogue from them.
|
|
|
|
Split out of build-push.sh once a recipe could hold more than one version.
|
|
Reading one key at a time out of a YAML file with a shell function was
|
|
tolerable while an application was a single version; it is not once every
|
|
field can be stated per version or shared across them, and the shell has no
|
|
type that holds the result.
|
|
|
|
Two subcommands:
|
|
|
|
versions <dir> one TSV line per version of one application, for the
|
|
build loop to iterate over
|
|
catalog <dir> <out> <registry> <owner>
|
|
the whole catalogue, checked against the registry
|
|
|
|
A recipe is image/apps/<name>/app.yaml. It may state one version at the top
|
|
level, which is what every recipe did before this and still does:
|
|
|
|
name: stress-ng
|
|
version: "0.18.07"
|
|
published: true
|
|
|
|
or a list, where the top level holds what the versions share:
|
|
|
|
name: gromacs
|
|
module: gromacs
|
|
binaries: [gmx_mpi]
|
|
versions:
|
|
- version: "2024.4"
|
|
recipe: 2024.4 # a subdirectory with its own spack.yaml
|
|
published: false
|
|
- version: "2025.1"
|
|
recipe: 2025.1
|
|
published: true
|
|
|
|
Both are read into the same shape, so nothing downstream knows which was
|
|
written. That is the point: an application with one version and an
|
|
application with four differ in their recipe and nowhere else.
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
import yaml
|
|
|
|
# Fields a version may state for itself. Anything else is shared by every
|
|
# version of the application - a description or a module name that differed
|
|
# between versions would be two applications wearing one name.
|
|
PER_VERSION = ('version', 'tag', 'image', 'upstream', 'recipe', 'published',
|
|
'size_gb', 'build_args', 'runtime_image', 'spack_image', 'status',
|
|
'status_note', 'sif_sha256')
|
|
|
|
SHARED = ('description', 'homepage', 'module', 'binaries', 'requires',
|
|
'mpi', 'license', 'signed')
|
|
|
|
|
|
# What was published, as opposed to what the recipes intend to publish.
|
|
#
|
|
# Kept apart from app.yaml deliberately. A recipe is hand-written and carries
|
|
# the reasoning for every choice in it; rewriting it from a script to record a
|
|
# hash would throw those comments away the first time an image was pushed.
|
|
# So: app.yaml is intent, published.json is measurement, and the generator
|
|
# joins them.
|
|
def published_path(here):
|
|
return os.path.join(here, 'published.json')
|
|
|
|
|
|
def published(here):
|
|
try:
|
|
with open(published_path(here)) as fh:
|
|
return json.load(fh)
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
|
|
def record(here, name, tag, sha256, dig='', image=''):
|
|
"""Note what a push actually produced, and where it put it.
|
|
|
|
The URI is recorded rather than derived, because it is not always the
|
|
same shape. With Apptainer on the build host an image is published as a
|
|
SIF over ORAS; without it the OCI image is pushed as it stands and the
|
|
head converts it. A catalogue that assumed one and got the other would
|
|
name a reference the head cannot resolve - and would find out at the
|
|
last task of a provision, after the machines were up.
|
|
"""
|
|
p = published_path(here)
|
|
db = published(here)
|
|
db[f"{name}:{tag}"] = {'sif_sha256': sha256, 'digest': dig, 'image': image}
|
|
tmp = p + '.new'
|
|
with open(tmp, 'w') as fh:
|
|
json.dump(db, fh, indent=2, sort_keys=True)
|
|
fh.write('\n')
|
|
os.replace(tmp, p)
|
|
print(f" recorded {name}:{tag} sha256:{sha256[:16]}")
|
|
|
|
|
|
def load(spec):
|
|
with open(spec) as fh:
|
|
return yaml.safe_load(fh) or {}
|
|
|
|
|
|
def versions(app_dir):
|
|
"""Every version of one application, as a list of flat dicts."""
|
|
a = load(os.path.join(app_dir, 'app.yaml'))
|
|
name = a.get('name') or os.path.basename(app_dir.rstrip('/'))
|
|
|
|
shared = {k: a[k] for k in SHARED if k in a}
|
|
raw = a.get('versions')
|
|
if not raw:
|
|
# The single-version form. Everything at the top level describes the
|
|
# one version there is.
|
|
raw = [{k: a[k] for k in PER_VERSION if k in a}]
|
|
|
|
out = []
|
|
for v in raw:
|
|
e = dict(shared)
|
|
# A shared default that a version may override: `published: false` at
|
|
# the top level parks a whole application without editing each entry.
|
|
for k in PER_VERSION:
|
|
if k in a and k not in ('version', 'tag', 'image'):
|
|
e[k] = a[k]
|
|
e.update(v)
|
|
e['name'] = name
|
|
e['version'] = str(e.get('version', ''))
|
|
e['tag'] = str(e.get('tag') or e['version'] or 'latest')
|
|
e['dir'] = os.path.join(app_dir, str(e['recipe'])) if e.get('recipe') else app_dir
|
|
e['status'] = normalise_status(e)
|
|
out.append(e)
|
|
return out
|
|
|
|
|
|
def normalise_status(e):
|
|
"""Where this version is in its life.
|
|
|
|
`published: true/false` was the old spelling and is still read, because
|
|
two ways to say the same thing in one file is worse than one awkward
|
|
one - but it only ever had two values, and the two that matter after
|
|
publication were the missing ones:
|
|
|
|
draft no image yet; left out of the catalogue entirely
|
|
available the normal state
|
|
deprecated a newer version exists; still offered, still supported
|
|
withdrawn do not start anything new on this; the image stays in the
|
|
registry, because clusters holding it may be rebuilt
|
|
"""
|
|
st = str(e.get('status') or '').strip().lower()
|
|
if st in ('draft', 'available', 'deprecated', 'withdrawn'):
|
|
return st
|
|
if st:
|
|
raise SystemExit(f"unknown status {st!r} for {e.get('name')}:{e.get('version')}")
|
|
return 'available' if e.get('published') else 'draft'
|
|
|
|
|
|
def all_versions(here):
|
|
for name in sorted(os.listdir(here)):
|
|
d = os.path.join(here, name)
|
|
if os.path.isfile(os.path.join(d, 'app.yaml')):
|
|
yield from versions(d)
|
|
|
|
|
|
# --- the registry ----------------------------------------------------------
|
|
|
|
def _token(host, repo, timeout=10):
|
|
return json.load(urllib.request.urlopen(
|
|
f"https://{host}/v2/token?scope=repository:{repo}:pull"
|
|
f"&service=container_registry", timeout=timeout)).get('token', '')
|
|
|
|
|
|
# What a SIF pushed over ORAS looks like from the outside. An OCI image
|
|
# carries tar+gzip layers instead, and Apptainer refuses to pull one of those
|
|
# through an oras:// reference: "invalid layer mediatype". That is the whole
|
|
# difference between the two schemes, and it is visible in the manifest -
|
|
# which is why this file reads it rather than assuming.
|
|
SIF_LAYER = 'application/vnd.sylabs.sif.layer.v1.sif'
|
|
|
|
|
|
def manifest(host, repo, tag, timeout=10):
|
|
"""What the tag points at, and which scheme can pull it.
|
|
|
|
Returns (digest, scheme). The scheme is 'oras' for a SIF pushed as one,
|
|
'docker' for an OCI image, and '' when the registry could not be asked or
|
|
answered with something that is neither.
|
|
|
|
A tag is mutable. `oras://…/gromacs:2024.4` names whatever was last
|
|
pushed under that name, so a cluster rebuilt after somebody repushed it
|
|
gets different bytes than the cluster that was allocated - and the record
|
|
that was supposed to freeze the cluster freezes nothing.
|
|
|
|
Recorded rather than pulled by: whether this Apptainer accepts an
|
|
`@sha256:` reference is not established, and a provision that fails at
|
|
its last task because of a syntax nobody tested would be worse than the
|
|
problem. What the head can do with no network and no version dependency
|
|
is check the file it ended up with, which is what apps_digest is for.
|
|
|
|
A GET rather than the HEAD this used to do. The digest comes back in the
|
|
same header either way; the body is what says which of the two shapes is
|
|
up there. One round trip, one more thing known.
|
|
"""
|
|
try:
|
|
req = urllib.request.Request(
|
|
f"https://{host}/v2/{repo}/manifests/{tag}",
|
|
headers={'Authorization': f'Bearer {_token(host, repo, timeout)}',
|
|
'Accept': 'application/vnd.oci.image.manifest.v1+json, '
|
|
'application/vnd.docker.distribution.manifest.v2+json'})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
dig = r.headers.get('Docker-Content-Digest') or ''
|
|
body = json.load(r)
|
|
except Exception:
|
|
return '', ''
|
|
layers = [l.get('mediaType', '') for l in (body.get('layers') or [])]
|
|
if SIF_LAYER in layers:
|
|
return dig, 'oras'
|
|
if any(l.startswith('application/vnd.oci.image.layer.')
|
|
or l.startswith('application/vnd.docker.image.rootfs.') for l in layers):
|
|
return dig, 'docker'
|
|
return dig, ''
|
|
|
|
|
|
def tags(host, repo, timeout=10):
|
|
"""What the registry actually holds for this repository.
|
|
|
|
An entry in the catalogue is a promise that a head node can pull the
|
|
image. Until this asked, that promise was a hand-written `published:
|
|
true`, and the first generated catalogue pointed busybox at a tag that
|
|
had never existed - guessed from the upstream version rather than read
|
|
from the registry. The head would have found out at the last task of a
|
|
provision, after the machines were up and the lease had started.
|
|
|
|
None means the registry could not be asked. That is not an empty list
|
|
and the caller must not treat it as one.
|
|
"""
|
|
try:
|
|
tok = _token(host, repo, timeout)
|
|
req = urllib.request.Request(
|
|
f"https://{host}/v2/{repo}/tags/list",
|
|
headers={'Authorization': f'Bearer {tok}'})
|
|
return json.load(urllib.request.urlopen(req, timeout=timeout)).get('tags') or []
|
|
except urllib.error.HTTPError as e:
|
|
if e.code == 404:
|
|
return [] # the repository does not exist: a real answer
|
|
return None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
# Unit separator, not tab.
|
|
#
|
|
# bash's `read` treats tab as IFS whitespace, so two of them in a row are one
|
|
# delimiter - and an application with no upstream image, which is most of
|
|
# them, shifted every field after it one place left. The build then took the
|
|
# binary list as the upstream reference and the status as the binary list,
|
|
# and failed with "image is missing available". An awk -F'\t' check of the
|
|
# same output looked perfectly correct, because awk does not collapse.
|
|
SEP = '\x1f'
|
|
|
|
|
|
def cmd_versions(app_dir):
|
|
for e in versions(app_dir):
|
|
args = ' '.join(f"{k}={v}" for k, v in (e.get('build_args') or {}).items())
|
|
print(SEP.join(str(x) for x in (
|
|
e['name'], e['version'], e['tag'], e['dir'],
|
|
e.get('upstream') or '', ' '.join(e.get('binaries') or []),
|
|
args, e.get('runtime_image') or '', e.get('spack_image') or '',
|
|
e['status'])))
|
|
|
|
|
|
def previously(out):
|
|
"""The references the last generated catalogue carried, by name:version.
|
|
|
|
A third source of measurement, and the one that survives a machine. The
|
|
push record lives beside the build that made it - on the host that ran
|
|
it, in a directory that gets re-copied - so a checkout elsewhere, or the
|
|
same checkout after that copy, knows nothing about a push that happened.
|
|
The catalogue does: every reference in it was written by a run that had
|
|
asked the registry.
|
|
|
|
Only the reference is taken from it. The digest is not, because the whole
|
|
point of the digest is that it is what the tag points at *now*.
|
|
"""
|
|
try:
|
|
with open(out) as fh:
|
|
old = yaml.safe_load(fh) or {}
|
|
except (OSError, ValueError):
|
|
return {}
|
|
was = {}
|
|
for a in (old.get('apps') or []):
|
|
if a.get('name') and a.get('image'):
|
|
was[f"{a['name']}:{a.get('version', '')}"] = a['image']
|
|
return was
|
|
|
|
|
|
def cmd_catalog(here, out, registry, owner):
|
|
offline = os.environ.get('CATALOG_OFFLINE') == '1'
|
|
pub = published(here)
|
|
was = previously(out)
|
|
apps, skipped, unchecked, nodigest, withdrawn = [], [], [], [], []
|
|
corrected = []
|
|
seen = set()
|
|
|
|
for e in all_versions(here):
|
|
ref = f"{e['name']}:{e['version']}"
|
|
if e['status'] == 'draft':
|
|
# No image exists, so there is nothing to promise. Left out
|
|
# rather than marked, because an entry in this file is a promise
|
|
# a head node can pull it.
|
|
skipped.append(f"{ref} (draft: no image published)")
|
|
continue
|
|
if ref in seen:
|
|
# Two entries under one name and version would make the pinned
|
|
# reference ambiguous, and Lookup would pick whichever sorted
|
|
# last. Refusing is better than choosing.
|
|
print(f"ERROR: {ref} is declared twice", file=sys.stderr)
|
|
return 1
|
|
seen.add(ref)
|
|
|
|
entry = {
|
|
'name': e['name'],
|
|
'version': e['version'],
|
|
# Recorded for every entry, not only the unusual ones: a reader
|
|
# of this file should not have to know that a missing status
|
|
# means available.
|
|
'status': e['status'],
|
|
'description': e.get('description', ''),
|
|
# What was published, then what the recipe asked for, then the
|
|
# default. The measurement wins: it is the only one that was
|
|
# observed rather than intended.
|
|
# Filled in below, once the registry has said which scheme
|
|
# can actually pull it. Named here so it keeps its place in
|
|
# the file: a reader looks for it next to the description.
|
|
'image': '',
|
|
'signed': bool(e.get('signed', False)),
|
|
}
|
|
if e.get('requires'):
|
|
entry['requires'] = list(e['requires'])
|
|
sif = os.path.join(here, '.build', f"{e['name']}-{e['tag']}.sif")
|
|
legacy = os.path.join(here, '.build', f"{e['name']}.sif")
|
|
for p in (sif, legacy):
|
|
if os.path.exists(p):
|
|
entry['size_gb'] = round(os.path.getsize(p) / 1024**3, 2)
|
|
break
|
|
else:
|
|
if e.get('size_gb'):
|
|
entry['size_gb'] = float(e['size_gb'])
|
|
for k in ('module', 'binaries', 'mpi', 'license', 'homepage',
|
|
'status_note'):
|
|
if e.get(k):
|
|
entry[k] = e[k]
|
|
|
|
# The measurement wins over anything the recipe declared: this is the
|
|
# hash of the file that was actually pushed, and a recipe cannot know
|
|
# it in advance.
|
|
if pub.get(ref, {}).get('sif_sha256'):
|
|
entry['sif_sha256'] = pub[ref]['sif_sha256']
|
|
elif e.get('sif_sha256'):
|
|
entry['sif_sha256'] = e['sif_sha256']
|
|
|
|
if e['status'] == 'withdrawn':
|
|
# Kept in the catalogue on purpose. It is the record of what has
|
|
# been published, and dropping a withdrawn entry would make it
|
|
# indistinguishable from one that never existed - which is the
|
|
# difference an operator needs when a cluster is found still
|
|
# holding it. The portal is what stops it being offered.
|
|
withdrawn.append(ref)
|
|
|
|
# Where the image is, in two parts, because only one of them can be
|
|
# worked out from a name.
|
|
#
|
|
# The location - host, repository, tag - follows from the convention
|
|
# when nothing recorded it. The scheme does not. It depends on
|
|
# whether the build host had Apptainer on the afternoon this was
|
|
# pushed, and on nothing else, so assuming oras:// is what put an OCI
|
|
# image in this catalogue under an ORAS reference. Every head that
|
|
# tried to pull it failed with "invalid layer mediatype"; the
|
|
# first-boot playbook died at the apps role; and the portal that runs
|
|
# after it was never configured, which is how one unpublished image
|
|
# cost a whole cluster its front door.
|
|
#
|
|
# So the scheme is read from the registry below, and where it cannot
|
|
# be read it is not invented.
|
|
stated = (pub.get(ref, {}).get('image') or was.get(ref)
|
|
or e.get('image') or '')
|
|
scheme, _, location = stated.rpartition('://')
|
|
if not location:
|
|
location = f"{registry}/{owner}/{e['name']}:{e['tag']}"
|
|
loc, _, want = location.rpartition(':')
|
|
host, _, repo = loc.partition('/')
|
|
|
|
if offline:
|
|
unchecked.append(ref)
|
|
else:
|
|
have = tags(host, repo)
|
|
if have is None:
|
|
unchecked.append(ref)
|
|
elif want not in have:
|
|
found = ', '.join(sorted(have)) if have else 'nothing'
|
|
skipped.append(f"{ref} ({repo}:{want} is not in the registry; it holds {found})")
|
|
continue
|
|
else:
|
|
# What the tag points at today, and what shape it is in. The
|
|
# digest is carried to the cluster record so a rebuild can
|
|
# tell whether the tag has moved under it.
|
|
d, holds = manifest(host, repo, want)
|
|
if d:
|
|
entry['digest'] = d
|
|
else:
|
|
nodigest.append(ref)
|
|
if holds and holds != scheme:
|
|
if scheme:
|
|
corrected.append(
|
|
f"{ref} ({scheme}:// was recorded; the registry holds "
|
|
f"a {holds}:// artefact)")
|
|
scheme = holds
|
|
|
|
if not scheme:
|
|
# Nothing recorded it and the registry did not say. A reference
|
|
# picked here would be a coin toss settled on a tenant's head
|
|
# node, at the last task of a provision, with the machines
|
|
# already up - so the entry is left out and named instead.
|
|
skipped.append(f"{ref} (nothing says whether {location} is a SIF or an "
|
|
f"OCI image, so no reference can be stated; republish it)")
|
|
continue
|
|
entry['image'] = f"{scheme}://{location}"
|
|
|
|
apps.append(entry)
|
|
|
|
apps.sort(key=lambda a: (a['name'], a['version']))
|
|
header = (
|
|
"# Application catalogue. GENERATED by image/apps/build-push.sh --catalog\n"
|
|
"# from the app.yaml beside each image recipe, and checked against the\n"
|
|
"# registry. Do not edit: an entry here is a promise that a head node\n"
|
|
"# can pull the image, and this file is the only place that promise is\n"
|
|
"# checked rather than declared.\n"
|
|
"#\n"
|
|
"# The portal reads it via `hpcod serve -apps config/apps.yaml` and\n"
|
|
"# re-reads it when it changes, so publishing needs no restart.\n\n"
|
|
)
|
|
with open(out, 'w') as fh:
|
|
fh.write(header)
|
|
yaml.safe_dump({'apps': apps}, fh, sort_keys=False,
|
|
default_flow_style=False, width=100)
|
|
|
|
print(f"{len(apps)} entries -> {out}")
|
|
for line in sorted(skipped):
|
|
print(f" not offered: {line}")
|
|
if withdrawn:
|
|
print(" withdrawn, kept for clusters that already hold them: "
|
|
+ ", ".join(sorted(withdrawn)))
|
|
if corrected:
|
|
# Said out loud, every time. A recorded URI and the registry
|
|
# disagreeing means a push was made one way and noted another, and a
|
|
# catalogue that quietly papered over it is how the wrong one
|
|
# survived long enough to break a provision.
|
|
print(" scheme corrected from the registry:")
|
|
for line in sorted(corrected):
|
|
print(f" {line}")
|
|
if nodigest:
|
|
print(" no digest recorded for: " + ", ".join(sorted(nodigest)))
|
|
print(" a rebuild cannot tell whether the tag has moved.")
|
|
if unchecked:
|
|
# Loud, and not a skip. Dropping every entry because a laptop is on
|
|
# a train would turn an unreachable registry into an empty catalogue.
|
|
print(" WARNING: the registry was not asked about: " + ", ".join(sorted(unchecked)))
|
|
print(" these are in the catalogue on their declared word alone.")
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if len(sys.argv) < 2:
|
|
print(__doc__, file=sys.stderr)
|
|
sys.exit(2)
|
|
if sys.argv[1] == 'versions':
|
|
cmd_versions(sys.argv[2])
|
|
elif sys.argv[1] == 'record':
|
|
record(*sys.argv[2:8])
|
|
elif sys.argv[1] == 'catalog':
|
|
sys.exit(cmd_catalog(*sys.argv[2:6]))
|
|
else:
|
|
print(f"unknown subcommand: {sys.argv[1]}", file=sys.stderr)
|
|
sys.exit(2)
|