novox/mesh-controller
3
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

A build records the bases it was handed, and the mesh reads its edges from builds

Browse Source 
Bases reach a recipe as build arguments, so the digest was never in the file the builder read
edges from: no build on the mesh recorded what it stood on, and 'build --on', the bases-first
order and the merge follow-up all walked a graph with no edges (novox/hq 04-ISSUES/131). The
builder now reports every base it resolved; the controller records them by artifact path and
reads the newest build's edges from the store, since a recorded manifest carries no build.on.
This commit is contained in:
jschoubben
2026-09-28 03:51:14 +02:00
parent aab6ded41b
commit 35252af665
6 changed files with 198 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/builder/builder.go
+34 -15
View File
@@ -169,6 +169,8 @@ func Build(ctx context.Context, run Runner, publish Publisher,
}
var built []catalogue.Built
// stoodOn is every base the build was handed, as resolved — the edges the catalogue derives.
var stoodOn []string
if manifest.Build != nil {
// What this module said it stands on, answered with what this mesh actually holds. Done
// before anything is built, so a missing base is refused in front of the person who can
@@ -186,11 +188,12 @@ func Build(ctx context.Context, run Runner, publish Publisher,
}
return from, nil
}
args, err := standingOn(ctx, manifest, held, mirror)
args, bases, err := standingOn(ctx, manifest, held, mirror)
if err != nil {
say("bases", "UNMET: %v", err)
return Result{}, err
}
stoodOn = bases
if len(args) > 0 {
say("bases", "%d resolved from what the mesh holds", len(args)/2)
}
@@ -217,7 +220,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
}
say("done", "%s at %s — %d artifact(s) pinned", manifest.Module, short(commit), len(built))
return Result{Manifest: resolved, Commit: commit, Built: built,
Against: against(within, manifest)}, nil
Against: against(within, manifest, stoodOn)}, nil
}
// Log is where a build says what it is doing, step by step. Nil is silent — the tests pass none,
@@ -339,14 +342,27 @@ func describe(path string) string {
// allowed to name — a tag is something somebody else can move under you.
var pinnedImage = regexp.MustCompile(`[A-Za-z0-9][A-Za-z0-9._/:-]*@sha256:[0-9a-f]{64}`)
// against reads what this module's image artifacts are built on top of, out of the files that
// build them. Nothing is guessed: a reference that is not written down is not reported.
func against(within string, manifest catalogue.Manifest) []string {
// against is what this module's image artifacts are built on top of: every base the mesh resolved
// and handed the recipe as a build argument (`build.on`), and any image a recipe pins by digest
// itself. Nothing is guessed: a reference that was neither resolved nor written down is not
// reported.
//
// **The resolved bases are the edges.** A recipe reads its base from an argument (`FROM
// ${RUNTIME_BASE}`), so the digest is never in the file, and a derivation that read files alone
// recorded no edge for any module on the mesh — which is why nothing knew what a changed base
// meant to rebuild (novox/hq 04-ISSUES/131).
func against(within string, manifest catalogue.Manifest, resolved []string) []string {
if manifest.Build == nil {
return nil
}
seen := map[string]bool{}
var out []string
for _, r := range resolved {
if r != "" && !seen[r] {
seen[r] = true
out = append(out, r)
}
}
for _, a := range manifest.Build.Artifacts {
if a.Kind != catalogue.ArtifactImage || a.From == "" {
continue
@@ -704,40 +720,42 @@ var _ io.Writer = (*stringWriter)(nil)
// built cannot be built here yet, and the useful sentence names which module is missing — not the
// one a container runtime produces when a recipe's first line refers to an image nobody has.
//
// The order is fixed so two builds of one commit invoke the same command.
// The order is fixed so two builds of one commit invoke the same command. Returned alongside the
// arguments is every reference they resolved to, which is what the build stood on.
func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[string]string,
mirror func(ctx context.Context, from, repository string) (string, error)) ([]string, error) {
mirror func(ctx context.Context, from, repository string) (string, error)) ([]string, []string, error) {
if manifest.Build == nil || len(manifest.Build.On) == 0 {
return nil, nil
return nil, nil, nil
}
on := append([]catalogue.BuildsOn{}, manifest.Build.On...)
sort.Slice(on, func(i, j int) bool { return on[i].Arg < on[j].Arg })
var args []string
var args, resolved []string
for _, base := range on {
if base.Image != "" {
// A vendor's image, declared (novox/hq 04-ISSUES/064, ADR 0097). Pinned, because a tag
// is what somebody else can move; copied into the mesh's registry, because a build
// that reaches a public registry on its own is a build that works sometimes.
if base.Arg == "" || base.Module != "" || base.Artifact != "" {
return nil, fmt.Errorf(
return nil, nil, fmt.Errorf(
"%s stands on the image %s, and a base is either a module's artifact or an "+
"image — never both — read from one build argument", manifest.Module, base.Image)
}
if !strings.Contains(base.Image, "@sha256:") {
return nil, fmt.Errorf(
return nil, nil, fmt.Errorf(
"%s stands on the image %q, which is not pinned by digest. A tag is what "+
"somebody else can move; name it as <image>@sha256:…", manifest.Module, base.Image)
}
reference, err := mirror(ctx, base.Image, manifest.Module+"/on-"+strings.ToLower(base.Arg))
if err != nil {
return nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
return nil, nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
}
args = append(args, "--build-arg", base.Arg+"="+reference)
resolved = append(resolved, reference)
continue
}
if base.Arg == "" || base.Module == "" || base.Artifact == "" {
return nil, fmt.Errorf(
return nil, nil, fmt.Errorf(
"%s says its build stands on something, and does not say all of what: a base "+
"needs the module, the artifact, and the build argument the recipe reads it "+
"from", manifest.Module)
@@ -745,14 +763,15 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin
key := base.Module + "/" + base.Artifact
reference, has := held[key]
if !has {
return nil, fmt.Errorf(
return nil, nil, fmt.Errorf(
"%s builds on %s, and this mesh has not built it. Build %s first — every module "+
"in this toolchain stands on it, so it is the thing to have before anything "+
"else", manifest.Module, key, base.Module)
}
args = append(args, "--build-arg", base.Arg+"="+reference)
resolved = append(resolved, reference)
}
return args, nil
return args, resolved, nil
}
// compile runs a module's own code through its toolchain, and says where the result is.
internal/builder/standing_on_test.go
+34 -6
View File
@@ -22,7 +22,7 @@ func TestABaseTheMeshHasNotBuiltIsRefused(t *testing.T) {
On: []catalogue.BuildsOn{{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"}},
},
}
_, err := standingOn(context.Background(), manifest, map[string]string{}, noMirror)
_, _, err := standingOn(context.Background(), manifest, map[string]string{}, noMirror)
if err == nil {
t.Fatal("a base nothing has built was accepted; the build would have failed on its first line")
}
@@ -42,7 +42,7 @@ func TestABaseTheMeshHoldsBecomesABuildArgument(t *testing.T) {
},
}
held := map[string]string{"mesh-tools/runtime": "127.0.0.1:5000/mesh-tools/runtime@sha256:" + strings.Repeat("a", 64)}
args, err := standingOn(context.Background(), manifest, held, noMirror)
args, _, err := standingOn(context.Background(), manifest, held, noMirror)
if err != nil {
t.Fatalf("a base this mesh holds was refused: %v", err)
}
@@ -54,7 +54,7 @@ func TestABaseTheMeshHoldsBecomesABuildArgument(t *testing.T) {
// A module naming no base asks for nothing, which is most modules.
func TestAModuleNamingNoBaseAddsNoArguments(t *testing.T) {
args, err := standingOn(context.Background(), catalogue.Manifest{Module: "hello-web", Build: &catalogue.Build{}}, nil, noMirror)
args, _, err := standingOn(context.Background(), catalogue.Manifest{Module: "hello-web", Build: &catalogue.Build{}}, nil, noMirror)
if err != nil || args != nil {
t.Fatalf("a module naming no base produced %v, %v", args, err)
}
@@ -66,7 +66,7 @@ func TestAnIncompleteBaseIsRefused(t *testing.T) {
Module: "postgres",
Build: &catalogue.Build{On: []catalogue.BuildsOn{{Module: "mesh-tools", Artifact: "runtime"}}},
}
if _, err := standingOn(context.Background(), manifest, map[string]string{"mesh-tools/runtime": "x"}, noMirror); err == nil {
if _, _, err := standingOn(context.Background(), manifest, map[string]string{"mesh-tools/runtime": "x"}, noMirror); err == nil {
t.Fatal("a base with no build argument was accepted; nothing would have read it")
}
}
@@ -86,7 +86,7 @@ func TestADeclaredVendorImageIsCopiedInAndHandedToTheRecipe(t *testing.T) {
},
}
var asked []string
args, err := standingOn(context.Background(), manifest, nil, func(_ context.Context, from, repository string) (string, error) {
args, _, err := standingOn(context.Background(), manifest, nil, func(_ context.Context, from, repository string) (string, error) {
asked = append(asked, from+" -> "+repository)
return "127.0.0.1:5000/" + repository + "@sha256:" + strings.Repeat("d", 64), nil
})
@@ -101,7 +101,7 @@ func TestADeclaredVendorImageIsCopiedInAndHandedToTheRecipe(t *testing.T) {
}
// Unpinned, it is refused: a tag is what somebody else can move.
manifest.Build.On[0].Image = "quay.io/minio/mc:latest"
if _, err := standingOn(context.Background(), manifest, nil, noMirror); err == nil || !strings.Contains(err.Error(), "not pinned") {
if _, _, err := standingOn(context.Background(), manifest, nil, noMirror); err == nil || !strings.Contains(err.Error(), "not pinned") {
t.Fatalf("an unpinned vendor image was accepted: %v", err)
}
}
@@ -151,3 +151,31 @@ func TestARecipeIsReadAsInstructions(t *testing.T) {
t.Fatalf("a heredoc line or a continued stage was read as a base: %v", bases)
}
}
// What a build was handed as its bases is what it stood on — recorded, so a changed base knows what
// to rebuild (novox/hq 04-ISSUES/131). A recipe reads the base from an argument, so nothing else
// could know.
func TestTheBasesABuildWasHandedAreWhatItStoodOn(t *testing.T) {
manifest := catalogue.Manifest{
Module: "gitea",
Build: &catalogue.Build{
On: []catalogue.BuildsOn{
{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"},
{Arg: "BUILD_BASE", Module: "mesh-tools", Artifact: "build"},
},
Artifacts: []catalogue.Artifact{{Name: "runtime", Kind: catalogue.ArtifactImage, From: "Dockerfile"}},
},
}
held := map[string]string{
"mesh-tools/runtime": "127.0.0.1:5000/mesh-tools/runtime@sha256:" + strings.Repeat("a", 64),
"mesh-tools/build": "127.0.0.1:5000/mesh-tools/build@sha256:" + strings.Repeat("b", 64),
}
_, resolved, err := standingOn(context.Background(), manifest, held, noMirror)
if err != nil {
t.Fatal(err)
}
got := against(t.TempDir(), manifest, resolved)
if len(got) != 2 || got[0] != held["mesh-tools/build"] || got[1] != held["mesh-tools/runtime"] {
t.Fatalf("the bases the build was handed were not what it stood on: %v", got)
}
}
internal/inventory/builds.go
+35
View File
@@ -164,6 +164,41 @@ func (i *Inventory) Held(ctx context.Context) (map[string]string, error) {
return held, rows.Err()
}
// BuiltAgainst is what each module's newest successful build stood on, as recorded — the build
// edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a
// module standing on nothing looks like: an edge the mesh has not derived is not an edge.
func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) {
rows, err := i.store.Pool().Query(ctx,
`select distinct on (module) module, built_against
from build
where module is not null and module <> '' and failed = ''
order by module, at desc`)
if err != nil {
return nil, err
}
defer rows.Close()
against := map[string][]string{}
for rows.Next() {
var module string
var raw []byte
if err := rows.Scan(&module, &raw); err != nil {
return nil, err
}
if len(raw) == 0 {
continue
}
var refs []string
if err := json.Unmarshal(raw, &refs); err != nil {
continue
}
if len(refs) > 0 {
against[module] = refs
}
}
return against, rows.Err()
}
// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept".
//
// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one