H1 refused-at-failed-build: a placed module whose newest build failed is published as a refusal

The node-engine writes a refused key, with the build's own error, at the indexer's
fingerprint in composed-<node> for a placed module whose newest build the builder
holds failed; a build queued or running stays unpublished (U1). A failed build of a
placed module's newest fingerprint also moves the pass (takeBuildDone), so the key is
written when it lands (U8).

Builds: design 47 fix-unpublished-placed-module U1, U8, ADR 0593 §6

internal/selfcompose/composedstate.go

@@ -28,7 +28,9 @@ import (
 //     build of it, so the plan exists before the want moves (build done, the node composes and writes, the provisioner
 //     reads, the providers provide, the assigner moves the want); a module it runs that no want names, at what runs;
 //   - each module's plan (catalogue.Plan) exactly when the provisioner's plans name it, from the same pass's builds;
-//   - a module refused alone at a fingerprint as that key with its refusal (Refused), and no resources.
+//   - a module refused alone at a fingerprint as that key with its refusal (Refused), and no resources; so is a module
+//     placed here whose newest build the builder holds failed (a build queued or running stays unpublished), at the
+//     indexer's fingerprint with the build's own error, so the provisioner holds it with those words and does not wait.
 //
 // A key is written only when its value differs from what the state holds (read once per run), and a module's key is
 // deleted once its fingerprint is not among the five it wrote last (engcomp.Keep) and no want names it. A pass
@@ -84,18 +86,25 @@ func (c *Composer) publishComposed(ctx context.Context) {
 
 // newestFPs is, of the modules placed here, the fingerprint each is composed at for the state beyond the wanted ones:
 // the indexer's, where the builder holds a done build of it; else the want's; else what runs of it. A module with none
-// is left out.
-func (c *Composer) newestFPs(ctx context.Context, at *published) map[string]string {
-	out := map[string]string{}
+// is left out. A module whose newest build the builder holds failed is also returned in failed, by module, with the
+// indexer's fingerprint and the build's error: it is published as a refusal, not composed.
+func (c *Composer) newestFPs(ctx context.Context, at *published) (out map[string]string, failed map[string][2]string) {
+	out = map[string]string{}
+	failed = map[string][2]string{}
 	r := reader{c: c}
 	for _, module := range at.placed {
 		c.mu.Lock()
 		fp2 := c.modules[module].Fp2
 		c.mu.Unlock()
 		if fp2 != "" {
-			if b, err := r.Build(ctx, module, fp2); err == nil && b != nil && b.State == builds.BuildDone {
-				out[module] = fp2
-				continue
+			if b, err := r.Build(ctx, module, fp2); err == nil && b != nil {
+				switch b.State {
+				case builds.BuildDone:
+					out[module] = fp2
+					continue
+				case builds.BuildFailed:
+					failed[module] = [2]string{fp2, b.Error}
+				}
 			}
 		}
 		if fp, ok := at.fps[module]; ok {
@@ -104,7 +113,7 @@ func (c *Composer) newestFPs(ctx context.Context, at *published) map[string]stri
 			out[module] = fp
 		}
 	}
-	return out
+	return out, failed
 }
 
 // writeComposed works out the state's keys from at and writes those that differ, then deletes what is past keeping.
@@ -177,7 +186,7 @@ func (c *Composer) composedValues(ctx context.Context, at *published) (map[strin
 		held  map[string]string
 	}{{at.scope, at.fps, at.comps, at.held}}
 	// The newest builds, composed only where they differ from the wanted ones.
-	newest := c.newestFPs(ctx, at)
+	newest, failed := c.newestFPs(ctx, at)
 	if !maps.Equal(newest, at.fps) {
 		scope := at.scope
 		scope.KeptAt = map[string]string{}
@@ -240,6 +249,19 @@ func (c *Composer) composedValues(ctx context.Context, at *published) (map[strin
 			}
 		}
 	}
+	// A placed module whose newest build failed is refused alone at that build, with the build's words, unless a
+	// composition of it at that fingerprint is already written above.
+	for module, f := range failed {
+		comp := engcomp.Composed{Node: c.env.Node, Module: module, FP: f[0], Refused: f[1]}
+		if key, err := composedKeyOf(comp); err != nil {
+			return nil, err
+		} else if _, written := out[key]; written {
+			continue
+		}
+		if err := put(comp, nil); err != nil {
+			return nil, err
+		}
+	}
 	return out, nil
 }
 

internal/selfcompose/composedstate_test.go

@@ -76,8 +76,11 @@ func TestTheNodePublishesItsCompositions(t *testing.T) {
 	digest := meshcatalogclient.ManifestDigest([]byte(lemurs))
 	oldie := `{"module":"oldie","version":"1","capabilities":["systemd"],"resources":[{"id":"service","type":"service","unit":"oldie.service","state":"running","boot":"enabled"}]}`
 	odigest := meshcatalogclient.ManifestDigest([]byte(oldie))
-	for d, json := range map[string]string{digest: lemurs, odigest: oldie} {
-		man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: map[string]string{digest: "lemurs", odigest: "oldie"}[d], Manifest: json})
+	brokenJSON := strings.ReplaceAll(oldie, "oldie", "broken")
+	waitingJSON := strings.ReplaceAll(oldie, "oldie", "waiting")
+	bdigest, wdigest := meshcatalogclient.ManifestDigest([]byte(brokenJSON)), meshcatalogclient.ManifestDigest([]byte(waitingJSON))
+	for d, json := range map[string]string{digest: lemurs, odigest: oldie, bdigest: brokenJSON, wdigest: waitingJSON} {
+		man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: map[string]string{digest: "lemurs", odigest: "oldie", bdigest: "broken", wdigest: "waiting"}[d], Manifest: json})
 		key, _ := meshcatalogclient.ManifestsKey(d)
 		b.kv[buckets.Manifests+"/"+key] = man
 	}
@@ -92,7 +95,7 @@ func TestTheNodePublishesItsCompositions(t *testing.T) {
 		t.Fatal(err)
 	}
 	place := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"},
-		Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}}})
+		Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "broken"}, {Module: "waiting"}}})
 	have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
 	mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"})
 	omod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: ofp, Manifest: odigest, Repository: "novox/o", Commit: "c"})
@@ -102,6 +105,12 @@ func TestTheNodePublishesItsCompositions(t *testing.T) {
 	b.put(t, buckets.Have, "n1.node", have)
 	b.put(t, buckets.Modules, "lemurs", mod)
 	b.put(t, buckets.Modules, "oldie", omod)
+	bfp := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64)
+	wfp := meshcatalogclient.SourcePrefix2 + strings.Repeat("e", 64)
+	for m, f := range map[string]string{"broken": bfp, "waiting": wfp} {
+		raw, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: map[string]string{"broken": bdigest, "waiting": wdigest}[m], Repository: "novox/o", Commit: "c"})
+		b.put(t, buckets.Modules, m, raw)
+	}
 	b.put(t, buckets.Wants, "want.n1.lemurs", want)
 	b.put(t, buckets.Wants, "want.n1.node", nodeWant)
 	for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} {
@@ -162,6 +171,43 @@ func TestTheNodePublishesItsCompositions(t *testing.T) {
 		t.Errorf("the same build landing again wrote %v", got[3:])
 	}
 
+	// A placed module whose newest build failed is published as a refusal at that build with the build's words (the
+	// provisioner holds it and does not wait); one whose build is queued or running stays unpublished.
+	for m, st := range map[string]builds.BuilderBuild{
+		"broken":  {State: builds.BuildFailed, Error: `refused kind: package of language "go"`},
+		"waiting": {State: builds.BuildQueued},
+	} {
+		f := map[string]string{"broken": bfp, "waiting": wfp}[m]
+		raw, _ := builds.EncodeBuilderBuild(st)
+		k, _ := builds.BuilderBuildKey(m, f)
+		b.mu.Lock()
+		b.kv[builds.BuilderBuildsBucket+"/"+k] = raw
+		b.mu.Unlock()
+		b.put(t, builds.BuilderBuildsBucket, k, raw)
+	}
+	brokenKey, _ := engcomp.Key("n1", "broken", bfp)
+	got = waitFor("broken's failed build", func(got []string) bool { return len(got) >= 4 })
+	// The pass that wrote it says so ("1 key(s) written" was said once before, for oldie); then only brokenKey is new.
+	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
+		n := 0
+		b.mu.Lock()
+		for _, l := range b.said {
+			if strings.Contains(l, "this node's composition state: 1 key(s) written") {
+				n++
+			}
+		}
+		b.mu.Unlock()
+		if n >= 2 {
+			break
+		}
+	}
+	if got := state.written(); len(got) != 4 || got[3] != brokenKey {
+		t.Errorf("a failed build and a queued one wrote %v, want only %s", got[3:], brokenKey)
+	}
+	if comp, err := engcomp.Decode(state.kv[brokenKey].Value); err != nil || comp.Refused != `refused kind: package of language "go"` || len(comp.Resources) != 0 {
+		t.Errorf("the failed build's key: %+v, %v", comp, err)
+	}
+
 	// What is kept: the five keys of a module written last, and one a want names; the rest is deleted.
 	keep := New(b.env())
 	keep.pubKept = map[string]ComposedEntry{}

internal/selfcompose/selfcompose.go

@@ -433,7 +433,8 @@ func (c *Composer) inputs() []input {
 }
 
 // takeBuildDone says whether a build that landed is one this node composes from: done, of a module it wants at that
-// build, or wanted or placed here whose trunk's build it is. Every other build, and a build not done, moves nothing.
+// build, or wanted or placed here whose trunk's build it is; or failed, of a module placed here whose trunk's build it
+// is, which the node publishes as a refusal (ADR 0593 §6). Every other build moves nothing.
 func (c *Composer) takeBuildDone(u agent.Update) bool {
 	// Any write or delete of a build's key forgets the build kept as done: the builder claims a done key again as a new
 	// revision (a gone step, issue 1937), which lands done with other artifacts.
@@ -457,11 +458,14 @@ func (c *Composer) takeBuildDone(u agent.Update) bool {
 	c.mu.Unlock()
 	// A module placed here is composed at its trunk's build once that is done, for the composition state, before its
 	// want moves (ADR 0593 §6).
-	if !(wanted && w.Fp == fp) && !((wanted || placedHere) && m.Fp2 == fp) {
+	b, err := builds.DecodeBuilderBuild(u.Value)
+	if err != nil {
 		return false
 	}
-	b, err := builds.DecodeBuilderBuild(u.Value)
-	return err == nil && b.State == builds.BuildDone
+	if b.State == builds.BuildFailed {
+		return placedHere && m.Fp2 == fp
+	}
+	return b.State == builds.BuildDone && (wanted && w.Fp == fp || (wanted || placedHere) && m.Fp2 == fp)
 }
 
 // Watch starts the composer and answers the compositions it makes: each as a key of `composed-<node>` would be, then a