Merge pull request 'The planner puts the new node-engine and the new controller in one tier for release 6's switch, and orders every other plan as today (issue 1942)' (#455) from fix/1942-switch-one-tier into main

Reviewed-on: #453

cmd/mesh-controller/batches.go

@@ -1210,13 +1210,14 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
 		}
 	}
 	sort.Strings(also)
-	ordered, err := changeOrderEdges(ctx, merges, members, entries, moved, edges)
+	ordered, together, err := changeOrderEdges(ctx, merges, members, entries, moved, edges)
 	if err != nil {
 		return err
 	}
 	var withheld []string
 	plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...))
 	placedBy(&plan, ordered)
+	placedTogether(&plan, together)
 	// **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the
 	// merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it
 	// would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change,
@@ -1341,6 +1342,19 @@ func placedBy(plan *inventory.Plan, ordered []inventory.Edge) {
 	}
 }
 
+// placedTogether writes the switch's line on the controller's and the node-engine's tier (ADR 0539 §11), so the plan
+// says why they share one: the record and each edge overruled.
+func placedTogether(plan *inventory.Plan, together string) {
+	if together == "" {
+		return
+	}
+	for _, n := range []string{theController, theEngine} {
+		if m, ok := plan.Modules[n]; ok {
+			m.Placed = strings.Trim(m.Placed+"; "+together, "; ")
+		}
+	}
+}
+
 // unchangedSaid names the modules a walk says matched what runs, for its note.
 func unchangedSaid(unchanged map[string]string) string {
 	names := make([]string, 0, len(unchanged))
@@ -1536,7 +1550,7 @@ type batchMember struct {
 // A cycle is said, never lost: the plan puts the modules left in its last tier, and the composed check fails the
 // group before it merges.
 func changeOrderEdges(ctx context.Context, merges []inventory.BatchedMerge, members []batchMember,
-	entries []inventory.Entry, moved []string, edges []inventory.Edge) ([]inventory.Edge, error) {
+	entries []inventory.Entry, moved []string, edges []inventory.Edge) ([]inventory.Edge, string, error) {
 	// What each repository's branch held before the batch: the first parent of its earliest merge in it.
 	earliest := map[string]inventory.BatchedMerge{}
 	for _, k := range merges {
@@ -1559,17 +1573,27 @@ func changeOrderEdges(ctx context.Context, merges []inventory.BatchedMerge, memb
 	}
 	changes, err := changesOf(ctx, sources)
 	if err != nil {
-		return nil, fmt.Errorf("what the merges say of their manifests cannot be read, so the order of their modules "+
+		return nil, "", fmt.Errorf("what the merges say of their manifests cannot be read, so the order of their modules "+
 			"cannot be drawn: %w", err)
 	}
-	order := orderEdges(moved, changes, edges)
+	named := false
+	for _, k := range merges {
+		var e link.SourceMoved
+		if json.Unmarshal(k.Event, &e) == nil && switchNamed(e.Body) {
+			named = true
+		}
+	}
+	order := orderEdges(moved, changes, edges, named)
 	for _, e := range order.Edges {
 		fmt.Printf("  order: %s\n", edgeSaid(e))
 	}
+	if order.Together != "" {
+		fmt.Printf("  order: %s\n", order.Together)
+	}
 	if said := order.CycleSaid(); said != "" {
 		fmt.Printf("  %s\n", said)
 	}
-	return order.Edges, nil
+	return order.Edges, order.Together, nil
 }
 
 // searchAfterFailures starts the search a failed walk leaves (ADR 0276 decision 3): every merge it answered

cmd/mesh-controller/delivery.go

@@ -711,7 +711,7 @@ func orderOf(members []orderMember, reach map[string]orderReach, edges []invento
 			}
 		}
 	}
-	order := orderEdges(all, changes, edges)
+	order := orderEdges(all, changes, edges, false)
 	out.Edges = order.Edges
 	out.CycleSaid = order.CycleSaid()
 	for _, e := range order.Edges {

cmd/mesh-controller/delivery_edges.go

@@ -20,6 +20,11 @@ const (
 	ruleEngineFirst      = "engine before controller"
 )
 
+// switchField is the line of a pull request's description that names release 6's switch (novox/hq ADR 0539 §11):
+// "Release-switch: 6". A plan whose merges include a pull request with it puts the node-engine and the controller in
+// one tier. Read by switchNamed, from the description the forge announced with the merge.
+const switchField = "Release-switch:"
+
 // The two modules the rollout rules name.
 const (
 	theController = "mesh-controller"
@@ -39,6 +44,20 @@ type planOrder struct {
 	Edges []inventory.Edge
 	// Cycle is every edge of a cycle among the order's edges and the graph's, the module graph's included.
 	Cycle []inventory.Edge
+	// Together is, when the switch is named and the plan moves the controller, the line its tier carries (ADR 0539
+	// §11): the record and each edge it overruled (ADR 0419 clause 5). Empty otherwise.
+	Together string
+}
+
+// switchNamed says whether a pull request's description names release 6's switch: a line that is exactly the field
+// with the value 6.
+func switchNamed(body string) bool {
+	for _, line := range strings.Split(body, "\n") {
+		if rest, ok := strings.CutPrefix(strings.TrimSpace(line), switchField); ok && strings.TrimSpace(rest) == "6" {
+			return true
+		}
+	}
+	return false
 }
 
 // CycleSaid names every module and edge of the cycle, for a check to fail with (ADR 0419 clause 6).
@@ -72,10 +91,14 @@ func orders(e inventory.Edge) bool {
 // it, goes before every module whose manifest the plan changes, the node-engine excepted; and the node-engine goes
 // before the controller by default, when the plan moves both and no edge orders them either way, transitively. An
 // edge is "From after To". graph is the module graph, whose edges order too and are never drawn again.
-func orderEdges(moved []string, changes map[string]moduleChange, graph []inventory.Edge) planOrder {
+// switchNamed is release 6's switch (ADR 0539 §11): the version-skew edge and the engine-first default are not drawn
+// and the plan says it overruled them (Together); the verb edges and the graph still order.
+func orderEdges(moved []string, changes map[string]moduleChange, graph []inventory.Edge, switchNamed bool) planOrder {
 	var out planOrder
 	sorted := append([]string(nil), moved...)
 	sort.Strings(sorted)
+	// The switch acts only on a plan that moves both the engine and the controller.
+	switchNamed = switchNamed && slices.Contains(sorted, theController) && slices.Contains(sorted, theEngine)
 	for _, server := range sorted {
 		for _, caller := range sorted {
 			if server == caller {
@@ -89,15 +112,25 @@ func orderEdges(moved []string, changes map[string]moduleChange, graph []invento
 			}
 		}
 	}
+	var overruled []string
 	if slices.Contains(sorted, theController) {
 		for _, m := range sorted {
 			if m != theController && m != theEngine && changes[m].Manifest {
+				if switchNamed {
+					overruled = append(overruled, ruleVersionSkew+" ("+m+" after "+theController+")")
+					continue
+				}
 				out.Edges = append(out.Edges, inventory.Edge{From: m, To: theController, Kind: edgeChangeOrder,
 					Why: ruleVersionSkew})
 			}
 		}
 	}
-	if slices.Contains(sorted, theController) && slices.Contains(sorted, theEngine) {
+	if switchNamed {
+		overruled = append(overruled, ruleEngineFirst)
+	}
+	if switchNamed {
+		out.Together = switchSaid(overruled)
+	} else if slices.Contains(sorted, theController) && slices.Contains(sorted, theEngine) {
 		all := orderingEdges(graph, out.Edges)
 		switch {
 		case pathBetween(all, theController, theEngine) != nil:
@@ -127,6 +160,15 @@ func orderEdges(moved []string, changes map[string]moduleChange, graph []invento
 	return out
 }
 
+// switchSaid is the line of a tier the switch shapes: the record that decided it and each edge it overruled.
+func switchSaid(overruled []string) string {
+	said := "release 6's switch (ADR 0539 §11)"
+	if len(overruled) > 0 {
+		said += " overrules " + strings.Join(overruled, ", ") + " (ADR 0419 clause 5)"
+	}
+	return said
+}
+
 // orderingEdges is the graph's edges that order and the edges drawn, together.
 func orderingEdges(graph, drawn []inventory.Edge) []inventory.Edge {
 	var out []inventory.Edge

cmd/mesh-controller/delivery_edges_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"slices"
 	"strings"
 	"testing"
 
@@ -28,7 +29,7 @@ func TestAVerbIsServedBeforeItsCallerIsSent(t *testing.T) {
 		"server": {Serves: []string{"mesh-delivery.groups"}, Manifest: true},
 		"caller": {Invokes: []string{"mesh-delivery.groups"}, Manifest: true},
 	}
-	got := orderEdges([]string{"caller", "server"}, changes, nil)
+	got := orderEdges([]string{"caller", "server"}, changes, nil, false)
 	e, ok := hasEdge(got.Edges, "caller", "server")
 	if !ok || !strings.Contains(e.Why, "verb before caller") || !strings.Contains(e.Why, "mesh-delivery.groups") {
 		t.Fatalf("the caller does not follow the server by the verb: %+v", got.Edges)
@@ -47,7 +48,7 @@ func TestEdgesBothWaysAreACycleNamingEveryModuleAndEdge(t *testing.T) {
 		"x": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}},
 		"y": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}},
 	}
-	got := orderEdges([]string{"x", "y"}, changes, nil)
+	got := orderEdges([]string{"x", "y"}, changes, nil, false)
 	if len(got.Cycle) < 2 {
 		t.Fatalf("no cycle found: %+v", got)
 	}
@@ -62,7 +63,7 @@ func TestEdgesBothWaysAreACycleNamingEveryModuleAndEdge(t *testing.T) {
 // The node-engine goes before the controller by default, and says so; an edge of the manifests the other way
 // overrules it, and the edge that did is said.
 func TestTheEngineGoesBeforeTheControllerUnlessAnEdgeSaysOtherwise(t *testing.T) {
-	got := orderEdges([]string{"mesh-controller", "mesh-host"}, nil, nil)
+	got := orderEdges([]string{"mesh-controller", "mesh-host"}, nil, nil, false)
 	e, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host")
 	if !ok || e.Why != "engine before controller" {
 		t.Fatalf("the engine does not go first: %+v", got.Edges)
@@ -71,7 +72,7 @@ func TestTheEngineGoesBeforeTheControllerUnlessAnEdgeSaysOtherwise(t *testing.T)
 		"mesh-host":       {Serves: []string{"s.v"}},
 		"mesh-controller": {Invokes: []string{"s.v"}},
 	}
-	got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil)
+	got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil, false)
 	if _, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host"); !ok || len(got.Edges) != 1 ||
 		!strings.Contains(got.Edges[0].Why, "verb before caller") {
 		t.Fatalf("the verb edge does not place the controller after the engine: %+v", got.Edges)
@@ -81,7 +82,7 @@ func TestTheEngineGoesBeforeTheControllerUnlessAnEdgeSaysOtherwise(t *testing.T)
 		"mesh-controller": {Serves: []string{"s.v"}},
 		"mesh-host":       {Invokes: []string{"s.v"}},
 	}
-	got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil)
+	got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil, false)
 	e, ok = hasEdge(got.Edges, "mesh-host", "mesh-controller")
 	if !ok || len(got.Edges) != 1 || !strings.Contains(e.Why, "overrules engine before controller") {
 		t.Fatalf("the verb edge does not say it overruled the default: %+v", got.Edges)
@@ -91,14 +92,14 @@ func TestTheEngineGoesBeforeTheControllerUnlessAnEdgeSaysOtherwise(t *testing.T)
 // The controller goes before every module whose manifest the plan changes (version skew), the node-engine excepted.
 func TestTheControllerGoesBeforeEveryChangedManifest(t *testing.T) {
 	changes := map[string]moduleChange{"app": {Manifest: true}, "other": {}}
-	got := orderEdges([]string{"mesh-controller", "app", "other"}, changes, nil)
+	got := orderEdges([]string{"mesh-controller", "app", "other"}, changes, nil, false)
 	if e, ok := hasEdge(got.Edges, "app", "mesh-controller"); !ok || e.Why != "version skew" {
 		t.Fatalf("the changed manifest does not follow the controller: %+v", got.Edges)
 	}
 	if _, ok := hasEdge(got.Edges, "other", "mesh-controller"); ok {
 		t.Fatalf("a module with an unchanged manifest was ordered: %+v", got.Edges)
 	}
-	if got := orderEdges([]string{"app", "other"}, changes, nil); len(got.Edges) != 0 {
+	if got := orderEdges([]string{"app", "other"}, changes, nil, false); len(got.Edges) != 0 {
 		t.Fatalf("version skew ordered a plan that does not move the controller: %+v", got.Edges)
 	}
 }
@@ -109,7 +110,7 @@ func TestTheOrderIsPerModuleNotPerRepository(t *testing.T) {
 		"a": {Serves: []string{"s.v"}},
 		"c": {Invokes: []string{"s.v"}},
 	}
-	got := orderEdges([]string{"a", "b", "c"}, changes, nil)
+	got := orderEdges([]string{"a", "b", "c"}, changes, nil, false)
 	tiers := tiersOf([]string{"a", "b", "c"}, got.Edges)
 	if len(tiers) != 2 || strings.Join(tiers[0], ",") != "a,b" || strings.Join(tiers[1], ",") != "c" {
 		t.Fatalf("tiers %v, want [a b] then [c]", tiers)
@@ -181,3 +182,63 @@ func TestATruncatedChangeIsReadAsChangingTheManifestsOfWhatItMoves(t *testing.T)
 		t.Fatalf("the member's change is read as %+v", src)
 	}
 }
+
+// 1942/AC1: release 6's switch puts the node-engine and the controller in one tier; without it the controller's
+// tier comes first as today (ADR 0539 §11).
+func TestTheSwitchPutsTheEngineAndTheControllerInOneTier(t *testing.T) {
+	moved := []string{"mesh-controller", "mesh-host", "app"}
+	changes := map[string]moduleChange{"mesh-host": {Manifest: true}, "app": {Manifest: true}}
+	with := orderEdges(moved, changes, nil, true)
+	if len(with.Edges) != 0 || len(with.Cycle) != 0 {
+		t.Fatalf("the switch still draws edges: %+v", with)
+	}
+	if tiers := tiersOf(moved, with.Edges); len(tiers) != 1 || len(tiers[0]) != 3 {
+		t.Fatalf("tiers %v, want one tier", tiers)
+	}
+	without := orderEdges(moved, changes, nil, false)
+	tiers := tiersOf(moved, without.Edges)
+	if len(tiers) < 2 || slices.Contains(tiers[0], "mesh-controller") && slices.Contains(tiers[0], "mesh-host") {
+		t.Fatalf("without the switch the tiers are %v, want the engine's tier before the controller's", tiers)
+	}
+	if without.Together != "" {
+		t.Fatalf("a plan without the switch carries its line: %q", without.Together)
+	}
+	// A verb edge between them still orders them.
+	verb := map[string]moduleChange{"mesh-host": {Serves: []string{"s.v"}}, "mesh-controller": {Invokes: []string{"s.v"}}}
+	if got := orderEdges([]string{"mesh-controller", "mesh-host"}, verb, nil, true); len(got.Edges) != 1 {
+		t.Fatalf("the switch dropped a verb edge: %+v", got.Edges)
+	}
+}
+
+// 1942/AC2: the shared tier's line names the record and each edge overruled.
+func TestTheSwitchSaysTheEdgesItOverruled(t *testing.T) {
+	changes := map[string]moduleChange{"app": {Manifest: true}}
+	got := orderEdges([]string{"mesh-controller", "mesh-host", "app"}, changes, nil, true)
+	for _, want := range []string{"ADR 0539 §11", "engine before controller", "version skew (app after mesh-controller)", "ADR 0419"} {
+		if !strings.Contains(got.Together, want) {
+			t.Fatalf("the line %q lacks %q", got.Together, want)
+		}
+	}
+	plan := inventory.Plan{Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "mesh-host": {}}}
+	placedTogether(&plan, got.Together)
+	if plan.Modules["mesh-controller"].Placed != got.Together || plan.Modules["mesh-host"].Placed != got.Together {
+		t.Fatalf("the tier does not carry the line: %+v %+v", plan.Modules["mesh-controller"], plan.Modules["mesh-host"])
+	}
+}
+
+// The switch is named by a line of a pull request's description.
+func TestTheSwitchIsNamedByAFieldOfTheDescription(t *testing.T) {
+	if !switchNamed("Fixes 1\n\nRelease-switch: 6\n") || switchNamed("Release-switch:\n") || switchNamed("Release-switch: 7") || switchNamed("about the Release-switch: 6") {
+		t.Fatal("the field is not read as a line with a value")
+	}
+}
+
+// 1942/AC3: the switch acts only on a plan that moves the engine and the controller: without mesh-host the
+// version-skew edge stays, the line present or not.
+func TestTheSwitchWithoutTheEngineKeepsVersionSkew(t *testing.T) {
+	changes := map[string]moduleChange{"app": {Manifest: true}}
+	got := orderEdges([]string{"mesh-controller", "app"}, changes, nil, true)
+	if e, ok := hasEdge(got.Edges, "app", "mesh-controller"); !ok || e.Why != "version skew" || got.Together != "" {
+		t.Fatalf("version skew lost without the engine: %+v", got)
+	}
+}

cmd/mesh-controller/held.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"github.com/novox/mesh-controller/internal/artifacts"
+	"github.com/novox/mesh-controller/internal/builderkeys"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The verb `held` and the event `artifacts-gone` (novox/hq issue 1937).
+//
+// **A done key can name a blob the store no longer holds.** A build module pushes a bundle or an archive as a bare blob
+// (ADR 0498 §1); the store's nightly collector deletes every blob no manifest names; and only a take-in or a sweep holds
+// one by its manifest. A key the controller has not taken in by the next collection names a blob that is gone. On
+// 2026-10-24 that was node-tools', mesh-host's and the forge bridge's. Since issue 1933 the take-in refuses such a key;
+// these two let the builder build it again: `held` before it reuses a done key, `artifacts-gone` when a take-in found it.
+
+// heldMost is the most references one ask of held judges: a builder asks for one key's artifacts at a time.
+const heldMost = 64
+
+// heldAsk asks the store whether it holds one reference: nil, artifacts.Gone or another error. A variable so a test
+// needs no store.
+var heldAsk = func(ctx context.Context, address, reference string) error {
+	return artifacts.Store{Address: address}.Holds(ctx, reference)
+}
+
+// heldAddress is the artifact store's address the serving controller asks. A variable so a test needs no records.
+var heldAddress = func(ctx context.Context) (string, error) {
+	d := doctorFrom
+	if d == nil || d.open == nil || d.open.inventory == nil {
+		return "", errors.New("this controller is not serving, so it asks the artifact store nothing: ask again, and " +
+			"the serving controller answers")
+	}
+	shelf, err := d.open.inventory.Catalogue(ctx)
+	if err != nil {
+		return "", err
+	}
+	address, err := artifactStoreAddress(ctx, d.open.inventory, shelf, "")
+	if err != nil {
+		return "", err
+	}
+	if address == "" {
+		return "", errors.New("the artifact store is on no network the controller reaches")
+	}
+	return address, nil
+}
+
+// heldAnswer is the verb `held`: each reference given, held, gone (the store answered 404) or unknown with why. It
+// holds and writes nothing: a reference the store holds and no record names is left for the take-in to hold.
+func heldAnswer(ctx context.Context, given string) (any, error) {
+	var refs []string
+	for _, r := range strings.Split(given, ",") {
+		if r = strings.TrimSpace(r); r != "" && !slices.Contains(refs, r) {
+			refs = append(refs, r)
+		}
+	}
+	if len(refs) == 0 {
+		return nil, errors.New("held judges the references given, and none was")
+	}
+	if len(refs) > heldMost {
+		return nil, fmt.Errorf("held judges at most %d references at once, and %d were given", heldMost, len(refs))
+	}
+	for _, r := range refs {
+		if _, ours := catalogue.InArtifactStore(r); !ours {
+			return nil, fmt.Errorf("%s is not a reference into the mesh's artifact store (artifact-store://...): nothing was asked", r)
+		}
+	}
+	address, err := heldAddress(ctx)
+	if err != nil {
+		return nil, err
+	}
+	asking, stop := context.WithTimeout(ctx, 20*time.Second)
+	defer stop()
+	a := sdkdelivery.HeldAnswer{Store: address, Held: []string{}, Gone: []string{}}
+	for _, r := range refs {
+		switch err := heldAsk(asking, address, r); {
+		case err == nil:
+			a.Held = append(a.Held, r)
+		case errors.Is(err, artifacts.Gone):
+			a.Gone = append(a.Gone, r)
+		default:
+			if a.Unknown == nil {
+				a.Unknown = map[string]string{}
+			}
+			a.Unknown[r] = err.Error()
+		}
+	}
+	out := fmt.Sprintf("the artifact store at %s holds %d of the %d reference(s) asked", address, len(a.Held), len(refs))
+	if len(a.Gone) > 0 {
+		out += "; gone: " + strings.Join(a.Gone, ", ")
+	}
+	if len(a.Unknown) > 0 {
+		out += fmt.Sprintf("; %d could not be asked", len(a.Unknown))
+	}
+	return verbAnswer{Output: out + "\n", OK: true, Answer: a}, nil
+}
+
+// sayArtifactsGone says `artifacts-gone` for a builder's key the take-in refused because the store does not hold what
+// it names (issue 1937): the builder hears it and builds the key again. Said from the serving controller only, and never
+// in the take-in's way: a bus that cannot carry it is said in the journal.
+var sayArtifactsGone = func(ctx context.Context, k builderkeys.Key, module string, gone []string) {
+	bus := checkEvents
+	if bus == nil || len(gone) == 0 {
+		return
+	}
+	key, _ := sdkdelivery.BuilderBuildKey(module, k.Fingerprint)
+	raw, err := sdkdelivery.EncodeArtifactsGone(sdkdelivery.ArtifactsGone{Module: module, Key: key,
+		Fingerprint: k.Fingerprint, Gone: gone, At: time.Now().UTC()})
+	if err != nil {
+		fmt.Printf("%s: its key names archives the store does not hold, and that could not be said: %v\n", module, err)
+		return
+	}
+	saying, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+	defer stop()
+	if err := bus.PublishSeatEvent(saying, link.MeshControllerSeat, link.KeyArtifactsGone, raw); err != nil {
+		fmt.Printf("%s: its key names archives the store does not hold, and %s could not be said: %v\n",
+			module, link.KeyArtifactsGone, err)
+	}
+}

cmd/mesh-controller/held_test.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"github.com/novox/mesh-controller/internal/artifacts"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// 1937/AC1: held answers each reference held, gone (the store's 404) or unknown, asks only HEADs, and its answer is what
+// the builder reads (mesh-sdk delivery.DecodeHeldAnswer).
+func TestHeldAnswersEachReferenceHeldOrGone(t *testing.T) {
+	present := "artifact-store://node-tools/runtime/blobs/" + shaOf("held")
+	missing := "artifact-store://node-tools/runtime/blobs/" + shaOf("gone")
+	broken := "artifact-store://mesh-host/host-arch/blobs/" + shaOf("broken")
+	address, ask := heldAddress, heldAsk
+	t.Cleanup(func() { heldAddress, heldAsk = address, ask })
+	heldAddress = func(context.Context) (string, error) { return "store.test:5100", nil }
+	var asked []string
+	var mu sync.Mutex
+	heldAsk = func(_ context.Context, at, reference string) error {
+		mu.Lock()
+		asked = append(asked, at+" "+reference)
+		mu.Unlock()
+		switch reference {
+		case missing:
+			return artifacts.Gone
+		case broken:
+			return errors.New("the artifact store answered 500")
+		}
+		return nil
+	}
+	got, err := heldAnswer(t.Context(), present+", "+missing+","+broken+","+present)
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := json.Marshal(got)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, err := sdkdelivery.DecodeHeldAnswer(raw, []string{present, missing, broken})
+	if err != nil {
+		t.Fatalf("the builder cannot read held's answer: %v\n%s", err, raw)
+	}
+	if !slices.Equal(a.Held, []string{present}) || !slices.Equal(a.Gone, []string{missing}) || a.Unknown[broken] == "" {
+		t.Errorf("held answered %+v", a)
+	}
+	if len(asked) != 3 {
+		t.Errorf("held asked the store %d times for three distinct references: %v", len(asked), asked)
+	}
+	if !strings.Contains(got.(verbAnswer).Output, missing) {
+		t.Errorf("held's words do not name the gone reference: %s", got.(verbAnswer).Output)
+	}
+
+	for _, refused := range []string{"", "registry.example/x@sha256:00"} {
+		if _, err := heldAnswer(t.Context(), refused); err == nil {
+			t.Errorf("held took %q", refused)
+		}
+	}
+	// A verb answered in the serving process, never by a command line.
+	if !inProcess["held"] {
+		t.Error("held is not answered by the serving controller")
+	}
+	if _, err := argvFor("held", map[string]any{"references": present}); err == nil {
+		t.Error("held composed a command line")
+	}
+}
+
+// saidGone keeps the artifacts-gone events said.
+type saidGone struct {
+	link.Bus // only PublishSeatEvent is said on
+	mu       sync.Mutex
+	said     []sdkdelivery.ArtifactsGone
+}
+
+func (s *saidGone) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
+	if seat != link.MeshControllerSeat || event != link.KeyArtifactsGone {
+		return errors.New("said under the wrong seat or name: " + seat + " " + event)
+	}
+	e, err := sdkdelivery.DecodeArtifactsGone(body)
+	if err != nil {
+		return err
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.said = append(s.said, e)
+	return nil
+}
+
+// 1937/AC2: a take-in refused for a gone archive names every gone one and says artifacts-gone with the module, the key,
+// its fingerprint and the gone references, so the builder builds the key again; a hand registration says nothing.
+func TestARefusedTakeInSaysArtifactsGone(t *testing.T) {
+	program := "artifact-store://x/tools/blobs/" + shaOf("t")
+	image := "artifact-store://x/img@" + shaOf("i")
+	keys, trunk := aKeyOf(tsBundle, 3,
+		sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"), Reference: program},
+		sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i"), Reference: image})
+	bus := &saidGone{}
+	was := checkEvents
+	checkEvents = bus
+	t.Cleanup(func() { checkEvents = was })
+	gone := func(context.Context, string) error { return artifacts.Gone }
+
+	_, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{hold: gone})
+	if err == nil || !strings.Contains(err.Error(), program) || !strings.Contains(err.Error(), image) ||
+		!strings.Contains(err.Error(), "not in the artifact store") {
+		t.Fatalf("a take-in of a key whose archives are gone: %v", err)
+	}
+	if len(bus.said) != 1 {
+		t.Fatalf("said %d artifacts-gone events; want one", len(bus.said))
+	}
+	e := bus.said[0]
+	wantKey, _ := sdkdelivery.BuilderBuildKey("x", keyFP)
+	if e.Module != "x" || e.Fingerprint != keyFP || e.Key != wantKey || !slices.Equal(e.Gone, []string{image, program}) {
+		t.Errorf("said %+v", e)
+	}
+
+	bus.said = nil
+	_, _, _ = buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk, Hand: map[string]string{"tools": program}},
+		keyContext{hold: gone})
+	if len(bus.said) != 0 {
+		t.Errorf("a registration by hand said %v: no builder's key was refused", bus.said)
+	}
+}

cmd/mesh-controller/key_takein.go

@@ -351,6 +351,9 @@ func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
 	// archive the store does not hold is sent, and every machine assigned the module fails its apply on the 404. An
 	// archive is held by its manifest, so the store's own collector keeps it too.
 	if kc.hold != nil {
+		// **Every gone archive named, and said** (issue 1937): the builder hears `artifacts-gone` and builds the key
+		// again, so the refusal is the last word on this key, not a plan that fails at every retry.
+		var gone, goneNames []string
 		for _, a := range k.Build.Artifacts {
 			reference := catalogue.Recorded(a.Reference)
 			if _, ours := catalogue.InArtifactStore(reference); !ours {
@@ -360,13 +363,21 @@ func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
 			switch {
 			case err == nil, errors.Is(err, artifacts.ErrNotOurs):
 			case errors.Is(err, artifacts.Gone):
-				return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder's key of %s names %s for its "+
-					"artifact %q, which is not in the artifact store: nothing was recorded or registered, and the key "+
-					"must be built again", t.Module, reference, a.Name)
+				gone = append(gone, reference)
+				goneNames = append(goneNames, fmt.Sprintf("%s for its artifact %q", reference, a.Name))
 			default:
 				return waits("the artifact store could not hold %s's artifact %q (%s): %v", t.Module, a.Name, reference, err)
 			}
 		}
+		if len(gone) > 0 {
+			if t.Hand == nil {
+				sayArtifactsGone(ctx, k, t.Module, gone)
+			}
+			return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder's key of %s names %s, not in "+
+				"the artifact store: nothing was recorded or registered, and the key must be built "+
+				"again (said as %s, which the builder builds it again on)", t.Module, strings.Join(goneNames, " and "),
+				link.KeyArtifactsGone)
+		}
 	}
 	built, err := builtOfKey(ctx, kc.archive, manifest, k)
 	if err != nil {

cmd/mesh-controller/seatverbs.go

@@ -318,6 +318,8 @@ func (a *verbArguments) commandLine() ([]string, error) {
 		return nil, errors.New("dead-letters is answered by the serving controller, on its own connection, not by a command")
 	case "root-free":
 		return nil, errors.New("root-free is judged by the serving controller, on its own connection, not by a command")
+	case "held":
+		return nil, errors.New("held is asked by the serving controller, of the artifact store, not by a command")
 	case "status":
 		return []string{"status", "--json"}, nil
 	case "nodes":
@@ -1253,6 +1255,9 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
 				if verb == "dead-letters" {
 					return deadLettersAnswer(ctx, a)
 				}
+				if verb == "held" {
+					return heldAnswer(ctx, a.given["references"])
+				}
 				if verb == "root-free" {
 					return rootFreeAnswer(ctx, a.given["machines"], rootClock())
 				}
@@ -1376,7 +1381,10 @@ var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true,
 	"dead-letters": true,
 	// Whether a machine is root-free, judged live on the serving controller's store and connection (novox/hq ADR
 	// 0259 §8): the router asks it before an approval.
-	"root-free": true}
+	"root-free": true,
+	// Whether the artifact store holds what a builder's key names, asked by the builder before it reuses the key (novox/hq
+	// issue 1937).
+	"held": true}
 
 // pushFileVar names the file the controller's push settings are written to (module.json, `push-wait`).
 const pushFileVar = "MESH_PUSH_FILE"

go.mod

@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0

go.sum

@@ -2,6 +2,8 @@ git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYg
 git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/broker/nats.go

@@ -406,6 +406,12 @@ var DegradableGrants = []DegradableGrant{
 		Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
 			"first push, and keeps the packages each node declares only after that, holding the lease (ADR 0507 §5, issue 1840)",
 	},
+	{
+		Subject: "mesh.seat." + ControllerSeat + ".event.artifacts-gone",
+		Reason: "refused: the take-in is refused all the same and says why; only the builder's rebuild waits for the " +
+			"controller's first push, and the builder also asks held when a module's value or its own start comes (novox/hq " +
+			"issue 1937)",
+	},
 	{
 		Subject: "mesh.seat." + ControllerSeat + ".event.sent",
 		Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +

internal/broker/streams.go

@@ -379,7 +379,9 @@ var ControllerStates = []string{"applied", "refused", "built-before",
 	"sent",
 	// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
 	// each registered module's declared config (design 57 F3).
-	"assignment-changed", "declared-config-changed", "predefined-changed"}
+	"assignment-changed", "declared-config-changed", "predefined-changed",
+	// And a builder's key not taken in because the artifact store does not hold what it names (novox/hq issue 1937).
+	"artifacts-gone"}
 
 // BusAdvisories are what the bus server says about the mesh's own account that the controller
 // reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it

internal/broker/testdata/composed.conf

@@ -29,7 +29,7 @@ accounts {
     ]
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.artifacts-gone", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }

internal/link/events.go

@@ -89,6 +89,9 @@ const (
 	KeyDeclaredConfigChanged = "declared-config-changed"
 	// KeyPredefinedChanged: the predefined-keys mirror's one key was written (design 57 K3): a newer list kept.
 	KeyPredefinedChanged = "predefined-changed"
+	// KeyArtifactsGone: a builder's key was not taken in because the artifact store does not hold an artifact it names
+	// (novox/hq issue 1937): the module, the key, its fingerprint and every gone reference. The builder builds the key again.
+	KeyArtifactsGone = "artifacts-gone"
 )
 
 // Applied is what a machine now runs, as the mesh states it.

module.json

@@ -102,6 +102,7 @@
     "data",
     "build",
     "artifacts",
+    "held",
     "collect",
     "images",
     "mirrors",

testdata/beside/CAPTURED

@@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself.
 
 mesh-catalog  2246ccc492e59579a140752e6f574b2121d296af  modules/*/module.json, modules/*/files, modules/*/templates, predefined.json
 mesh-host     9d9e3a46c4f5943cfffb413d962ce48b75e2fd39  examples/foundation-first-node.lock
-mesh-sdk      a2c813a6b468899bb5a95666b8f6f305be52f200  conformance/events/module-event.json
+mesh-sdk      34f210e5eba1d16abe06b526de1026c282692b83  conformance/events/module-event.json
 
 To move them, from this repository's root, with the two repositories checked out beside it: