Merge pull request 'The provisioner keeps a provision while a module's newest composition is unread, and withdraws an ask once a key is needed again (issue 1966, recovery)' (#638) from fix/1966-provisioner-newest-held into main

modules/mesh-provisioner/cmd/mesh-provisioner/composed_test.go

@@ -18,6 +18,7 @@ import (
 	catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
 	catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
 	"os"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -191,6 +192,55 @@ func TestARefusedOrMissingCompositionHoldsItsModule(t *testing.T) {
 	if e, _ := s.Get(PlansState, "n1.restic"); e == nil || string(e.Value) != string(plan.Value) {
 		t.Errorf("the plan was not kept while unpublished: %s", valueOf(e))
 	}
+	// Its newest build's composition not published (issue 1966): held at its newest, and what only that build requires
+	// is kept, never judged. Then refused there: held with the engine's words, the want's plan kept.
+	s.write(t, WantsRead, "want.n1.restic", `{"v":1,"fp":"`+fpsOf["restic"]+`","self":true}`)
+	published(t, s, c)
+	if _, err := h.ComposeNode(context.Background(), c.Node); err != nil {
+		t.Fatal(err)
+	}
+	for _, row := range []struct {
+		name, composed, held string
+	}{
+		{"not published", "", "is not published"},
+		{"refused", `{"v":1,"node":"n1","module":"restic","fp":"` + fpsOf["restic"] + `","resources":[],"refused":"no postgres here"}`, "no postgres here"},
+	} {
+		ahead := "src2:" + strings.Repeat("7", 64)
+		e, _ := s.Get(ModulesRead, "restic")
+		s.write(t, ModulesRead, "restic", strings.ReplaceAll(string(e.Value), fpsOf["restic"], ahead))
+		if row.composed != "" {
+			s.write(t, ComposedRead, mustKey(composedKey("n1", "restic", ahead)), row.composed)
+		}
+		comp := FromCompositions(s)(context.Background(), c.Node, map[string]string{"restic": fpsOf["restic"]})
+		if !strings.Contains(comp.Newest.Held["restic"], row.held) || slices.Contains(comp.UnpublishedNewest, "restic") != (row.composed == "") {
+			t.Errorf("%s: newest held %v, unpublished at the newest %v", row.name, comp.Newest.Held, comp.UnpublishedNewest)
+		}
+		// A pass writes the newest hold into `refused` beside the wanted build's, and deletes it once it is read.
+		h.Compose = func(context.Context, string, map[string]string) Composition { return comp }
+		if _, err := h.ComposeNode(context.Background(), c.Node); err != nil {
+			t.Fatal(err)
+		}
+		if e, _ := s.Get(RefusedState, "n1.restic=newest"); e == nil || !strings.Contains(string(e.Value), row.held) {
+			t.Errorf("%s: refused holds %s", row.name, valueOf(e))
+		}
+		h.Compose = FromCompositions(s)
+		if _, held := comp.AtWants.Held["restic"]; held {
+			t.Errorf("%s: its wanted build is composed, but the module is held there: %v", row.name, comp.AtWants.Held)
+		}
+	}
+	s.write(t, ModulesRead, "restic", strings.ReplaceAll(valueOf(mustEntry(t, s)), "src2:"+strings.Repeat("7", 64), fpsOf["restic"]))
+	if _, err := h.ComposeNode(context.Background(), c.Node); err != nil {
+		t.Fatal(err)
+	}
+	if e, _ := s.Get(RefusedState, "n1.restic=newest"); e != nil {
+		t.Errorf("the newest hold stays once the composition is read: %s", valueOf(e))
+	}
+}
+
+func mustEntry(t *testing.T, s *memStore) *Entry {
+	t.Helper()
+	e, _ := s.Get(ModulesRead, "restic")
+	return e
 }
 
 func readJSON(t *testing.T, path string, v any) {

modules/mesh-provisioner/cmd/mesh-provisioner/holder.go

@@ -64,6 +64,9 @@ type Refusal struct {
 	Module string    `json:"module"`
 	Why    string    `json:"why"`
 	Since  time.Time `json:"since"`
+	// Scope is "newest" for a module held at its newest build (its composition there unread: key `<node>.<module>=newest`),
+	// "" for one held at its wanted build.
+	Scope string `json:"scope,omitempty"`
 }
 
 // The reads it makes the plans from. ComposedRead is every node's published compositions, `composed-<node>` (ADR 0593
@@ -110,7 +113,10 @@ type Composition struct {
 	// Unpublished is each module whose composition at the build it is read at is not published yet: the node's picture
 	// is not whole, so no retirement is judged on it (issue 940).
 	Unpublished []string
-	Took        []Phase
+	// UnpublishedNewest is each module whose composition at its newest build (which is not its wanted build) is not
+	// published yet: the newest build's needs are unread, so no key only that build requires is judged (issue 1966).
+	UnpublishedNewest []string
+	Took              []Phase
 }
 
 // HeldPlans is one scope's plans by module, the modules held with why, and a failure that is no single module's.
@@ -253,12 +259,22 @@ func (h *Holder) ComposeNodeWith(ctx context.Context, node string, retireMany bo
 			out.Problems = append(out.Problems, fmt.Sprintf("%s at its newest build is not composed: %s", m, heldNewest[m]))
 		}
 	}
-	if err := h.writeRefused(node, heldAtWants); err != nil {
+	if err := h.writeRefused(node, heldAtWants, heldNewest); err != nil {
 		out.Problems = append(out.Problems, err.Error())
 	}
 	// A module whose composition is not published at its build is held (heldAtWants): its own plan and keys are kept,
 	// and nothing else on the node waits for it (review 1 of issue 940). A hold that lasts is said on the health.
-	h.countUnpublished(node, c.Unpublished, heldAtWants)
+	// One held at its newest build too is counted under its own name, so a lasting gap is said on the health (issue 1966).
+	unreadFor := map[string]string{}
+	for m, why := range heldAtWants {
+		unreadFor[m] = why
+	}
+	unread := append([]string(nil), c.Unpublished...)
+	for _, m := range c.UnpublishedNewest {
+		unreadFor[m+newestSuffix] = heldNewest[m]
+		unread = append(unread, m+newestSuffix)
+	}
+	h.countUnpublished(node, unread, unreadFor)
 	if err := h.writePlans(node, atWants, heldAtWants, &out); err != nil {
 		out.Problems = append(out.Problems, err.Error())
 	}
@@ -331,6 +347,13 @@ func (h *Holder) ComposeNodeWith(ctx context.Context, node string, retireMany bo
 			untouched[m] = "it has a want or a have on " + node + " and no plan of it was read"
 		}
 	}
+	// A module whose composition at its newest build is unread is left as it is too: not read is not unneeded, and a key
+	// only that build requires is not judged (ADR 0570 clause 5, issue 1966).
+	for m, why := range heldNewest {
+		if _, u := untouched[m]; !u {
+			untouched[m] = newestKept + why
+		}
+	}
 	h.judgeElsewhere(elsewhere, &out)
 	h.judgeNode(node, needed, ahead, untouched, retireMany, &out)
 	lap("provisions judged")
@@ -339,16 +362,35 @@ func (h *Holder) ComposeNodeWith(ctx context.Context, node string, retireMany bo
 
 // writeRefused keeps, for each module held on node, why, and deletes the refusal of one that composes again: a change
 // of the state is the event a watcher hears.
-func (h *Holder) writeRefused(node string, held map[string]string) error {
+func (h *Holder) writeRefused(node string, held, heldNewest map[string]string) error {
 	var problems []error
-	for _, module := range sortedKeys(held) {
-		key, err := meshprovisionerclient.PlanKey(node, module)
-		if err != nil {
-			problems = append(problems, err)
-			continue
+	// Each hold has a key: `<node>.<module>` at the wanted build, `<node>.<module>=newest` at the newest.
+	type hold struct {
+		key string
+		r   Refusal
+	}
+	var holds []hold
+	for _, scope := range []struct {
+		name string
+		by   map[string]string
+	}{{"", held}, {"newest", heldNewest}} {
+		for _, module := range sortedKeys(scope.by) {
+			key, err := meshprovisionerclient.PlanKey(node, module)
+			if err != nil {
+				problems = append(problems, err)
+				continue
+			}
+			if scope.name != "" {
+				key += "=" + scope.name
+			}
+			holds = append(holds, hold{key, Refusal{Node: node, Module: module, Why: scope.by[module], Since: h.Now().UTC(), Scope: scope.name}})
 		}
-		r := Refusal{Node: node, Module: module, Why: held[module], Since: h.Now().UTC()}
-		if e, err := h.Store.Get(RefusedState, key); err == nil && e != nil {
+	}
+	still := map[string]bool{}
+	for _, d := range holds {
+		still[d.key] = true
+		r := d.r
+		if e, err := h.Store.Get(RefusedState, d.key); err == nil && e != nil {
 			var was Refusal
 			if json.Unmarshal(e.Value, &was) == nil && was.Why == r.Why {
 				continue
@@ -359,11 +401,11 @@ func (h *Holder) writeRefused(node string, held map[string]string) error {
 			problems = append(problems, err)
 			continue
 		}
-		if err := h.Store.Put(RefusedState, key, json.RawMessage(raw)); err != nil {
-			problems = append(problems, fmt.Errorf("the refusal %s: %w", key, err))
+		if err := h.Store.Put(RefusedState, d.key, json.RawMessage(raw)); err != nil {
+			problems = append(problems, fmt.Errorf("the refusal %s: %w", d.key, err))
 			continue
 		}
-		h.Logf("[%s] %s on %s has no plan composed: %s", Module, module, node, r.Why)
+		h.Logf("[%s] %s on %s has no plan composed%s: %s", Module, r.Module, node, map[bool]string{true: " at its newest build"}[r.Scope != ""], r.Why)
 	}
 	keys, err := h.Store.Keys(RefusedState)
 	if err != nil {
@@ -371,10 +413,7 @@ func (h *Holder) writeRefused(node string, held map[string]string) error {
 	}
 	for _, k := range keys {
 		n, module, ok := strings.Cut(k, ".")
-		if !ok || n != node {
-			continue
-		}
-		if _, still := held[module]; still {
+		if !ok || n != node || still[k] {
 			continue
 		}
 		if err := h.Store.Delete(RefusedState, k); err != nil {
@@ -452,12 +491,17 @@ func (h *Holder) judgeElsewhere(keys map[string]meshprovisionerclient.Provision,
 		next := keys[key]
 		var have meshprovisionerclient.Provision
 		e, err := h.Store.Get(ProvisionsState, key)
+		decoded := false
 		if err == nil && e != nil {
 			if have, err = meshprovisionerclient.Decode(e.Value); err == nil {
 				next.Reference = have.Reference
+				decoded = true
 			}
 		}
 		h.outcome(key, &next)
+		if decoded {
+			h.withdrawAsk(key, have, &next, out)
+		}
 		if e != nil && same(have, next) {
 			continue
 		}
@@ -572,8 +616,11 @@ func (h *Holder) judgeNode(node string, needed map[string]meshprovisionerclient.
 	for key, have := range held {
 		consumers[have.Consumer.Module] = true
 		if _, isNeeded := needed[key]; !isNeeded && have.Consumer.Node == node {
-			if _, left := untouched[have.Consumer.Module]; left {
+			if why, left := untouched[have.Consumer.Module]; left {
 				delete(held, key)
+				if strings.HasPrefix(why, newestKept) && have.State != meshprovisionerclient.Retiring {
+					out.Changed = append(out.Changed, key+" kept: "+why)
+				}
 			}
 		}
 	}
@@ -628,6 +675,11 @@ func (h *Holder) judgeNode(node string, needed map[string]meshprovisionerclient.
 				next.Reference = have.Reference
 			}
 			h.outcome(key, &next)
+			// A key needed again is not asked about any more: its open ask is withdrawn on the operator's channel, and
+			// kept on the key until the router took the cancel, so the next pass tries again (issue 1966).
+			if isHeld {
+				h.withdrawAsk(key, have, &next, out)
+			}
 			if module, isAhead := ahead[key]; isAhead && next.State == meshprovisionerclient.Asked {
 				// The reason the provider's helper says too: no plan names it yet, and it binds it from this key's entry.
 				why := "no plan names it yet: the newest build of " + module + " requires it and its wanted build does " +
@@ -1093,6 +1145,26 @@ func (h *Holder) ask(p *meshprovisionerclient.Provision, key string) error {
 	return nil
 }
 
+// withdrawAsk withdraws the open retire ask of a key that is needed again (next), keeping the ask on it when the router
+// does not take the cancel, so the next pass tries again.
+func (h *Holder) withdrawAsk(key string, have meshprovisionerclient.Provision, next *meshprovisionerclient.Provision, out *Composed) {
+	if have.Ask == "" || have.Warrant != nil {
+		return
+	}
+	if err := h.cancelAsk(have.Ask); err != nil {
+		out.Problems = append(out.Problems, fmt.Sprintf("the ask %s of %s could not be withdrawn: %v", have.Ask, key, err))
+		next.Ask = have.Ask
+	}
+}
+
+// cancelAsk withdraws a retiring ask on the operator's channel, under mesh-provisioner's own name (accept.cancel).
+func (h *Holder) cancelAsk(id string) error {
+	if h.Asks == nil {
+		return errors.New("the operator's channel is not reachable from here")
+	}
+	return h.Asks.Publish(oc.AcceptSubjectCancel(Module), oc.CancelAccept{ID: id}, "cancel."+id)
+}
+
 // openAsks is how many retiring keys hold an ask with no answer yet.
 func (h *Holder) openAsks() int {
 	n := 0
@@ -1282,6 +1354,13 @@ func (h *Holder) countUnpublished(node string, modules []string, held map[string
 	h.unpublished[node] = now
 }
 
+// newestSuffix names a module's hold at its newest build beside its hold at its wanted build; newestKept is what an
+// untouched module's why starts with when its newest build's composition is the one unread.
+const (
+	newestSuffix = " at its newest build"
+	newestKept   = "its newest build's composition is unread: "
+)
+
 // Faults is, by node, each retirement a pass held back as a fault of the mesh.
 func (h *Holder) Faults() map[string]string {
 	h.mu.Lock()
@@ -1466,7 +1545,7 @@ func orSaid(words ...string) string {
 // nothing, and takes each module's plan from the key of `composed-<node>` at the build it is read at. At the wanted
 // builds a module is read at the fingerprint keptAt names, else at the indexer's newest (FP2); at the newest, each placed
 // module at FP2. A key whose engine refused the module holds it with the engine's words; a key not published yet holds
-// it at its wanted build (its last plan kept, as for a refusal) and leaves it out of the newest, whose build the engine
+// it at its wanted build (its last plan kept, as for a refusal) and at its newest too (issue 1966), whose build the engine
 // writes once it is done (ADR 0593 §6), and says it unpublished, so the pass judges no retirement on the node (issue
 // 940). A placed module no build of which is known is held too. A plan absent from a key is a module that hands the
 // provisioner nothing.
@@ -1492,7 +1571,7 @@ func FromCompositions(s Store) Composer {
 		for m, fp := range keptAt {
 			atWants[m] = fp
 		}
-		var unpublished []string
+		var unpublished, unpublishedNewest []string
 		for _, m := range sortedKeys(atWants) {
 			fp := atWants[m]
 			if fp == "" {
@@ -1519,13 +1598,19 @@ func FromCompositions(s Store) Composer {
 			c, ok := published[composedAt(node, m, fp2[m])]
 			switch {
 			case !ok:
+				// Not read is not unneeded: the module is held at its newest build, so what only that build requires is kept
+				// (issue 1966). Held at its wanted build too, it is said there.
+				newest.Held[m] = fmt.Sprintf("the composition of %s at %s is not published by the node-engine of %s yet", m, fp2[m], node)
+				if fp2[m] != atWants[m] {
+					unpublishedNewest = append(unpublishedNewest, m)
+				}
 			case c.Refused != "":
 				newest.Held[m] = c.Refused
 			case c.Plan != nil:
 				newest.Plans[m] = *c.Plan
 			}
 		}
-		return Composition{AtWants: at, Newest: newest, Unpublished: unpublished,
+		return Composition{AtWants: at, Newest: newest, Unpublished: unpublished, UnpublishedNewest: unpublishedNewest,
 			Took: []Phase{{Name: "compositions read", Took: time.Since(start)}}}
 	}
 }

modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go

@@ -31,9 +31,24 @@ type router struct {
 	asked   map[string]oc.Ask
 	records map[string][]byte
 	drop    bool
+	// cancelled is the ids of the asks withdrawn on accept.cancel, in order.
+	cancelled []string
+	// failCancel is a router that refuses a cancel.
+	failCancel bool
 }
 
-func (r *router) Publish(_ string, body any, id string) error {
+func (r *router) Publish(subject string, body any, id string) error {
+	if c, ok := body.(oc.CancelAccept); ok {
+		if subject != oc.AcceptSubjectCancel(Module) {
+			return fmt.Errorf("a cancel sent on %s", subject)
+		}
+		if r.failCancel {
+			return fmt.Errorf("the router is away")
+		}
+		r.cancelled = append(r.cancelled, c.ID)
+		delete(r.records, c.ID)
+		return nil
+	}
 	r.asked[id] = body.(oc.Ask)
 	if !r.drop {
 		r.records[id], _ = json.Marshal(map[string]any{"state": "open", "ask": body})
@@ -71,20 +86,26 @@ type mesh struct {
 	heldAt   map[string]map[string]string // by node, the modules its composition at the wanted builds holds
 	// unpublished is, by node, the modules whose composition is not published yet.
 	unpublished map[string][]string
-	composed    []string
+	// heldNewest is, by node, the modules its composition at the newest builds holds, and unpublishedNewest those of them
+	// not published yet.
+	heldNewest        map[string]map[string]string
+	unpublishedNewest map[string][]string
+	composed          []string
 }
 
 func newMesh() *mesh {
 	m := &mesh{s: newStore(), r: &router{asked: map[string]oc.Ask{}, records: map[string][]byte{}},
 		atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{},
-		heldAt: map[string]map[string]string{}, unpublished: map[string][]string{}}
+		heldAt: map[string]map[string]string{}, unpublished: map[string][]string{},
+		heldNewest: map[string]map[string]string{}, unpublishedNewest: map[string][]string{}}
 	now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
 	m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {},
 		Now: func() time.Time { now = now.Add(time.Second); return now },
 		Compose: func(_ context.Context, node string, keptAt map[string]string) Composition {
 			m.composed = append(m.composed, node)
 			return Composition{AtWants: HeldPlans{Plans: m.atWant[node], Held: m.heldAt[node]},
-				Newest: HeldPlans{Plans: m.atNewest[node]}, Unpublished: m.unpublished[node]}
+				Newest:      HeldPlans{Plans: m.atNewest[node], Held: m.heldNewest[node]},
+				Unpublished: m.unpublished[node], UnpublishedNewest: m.unpublishedNewest[node]}
 		}}
 	// What each provider's manifest declares: postgres, keycloak and the vault keep entries, the others report by their
 	// have.
@@ -295,19 +316,21 @@ func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) {
 		asked   []string
 		states  map[string]string
 		removed bool
+		// cancelled is how many asks have been withdrawn on the operator's channel once the step is done (issue 1966).
+		cancelled int
 	}{
 		{"a key composed ahead of its want is asked, not retiring", func() {}, nil,
-			map[string]string{pg: meshprovisionerclient.Provided, kc: meshprovisionerclient.Asked}, false},
+			map[string]string{pg: meshprovisionerclient.Provided, kc: meshprovisionerclient.Asked}, false, 0},
 		{"one requirement removed and another added: only the removed retires", func() {
 			m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
 			m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
-		}, []string{pg}, map[string]string{pg: meshprovisionerclient.Retiring, kc: meshprovisionerclient.Asked}, false},
-		{"composed again: no second ask", func() {}, nil, map[string]string{pg: meshprovisionerclient.Retiring}, false},
-		{"needed again before the warrant: back to its outcome", func() {
+		}, []string{pg}, map[string]string{pg: meshprovisionerclient.Retiring, kc: meshprovisionerclient.Asked}, false, 0},
+		{"composed again: no second ask", func() {}, nil, map[string]string{pg: meshprovisionerclient.Retiring}, false, 0},
+		{"needed again before the warrant: back to its outcome, its ask withdrawn", func() {
 			m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)}
-		}, nil, map[string]string{pg: meshprovisionerclient.Provided}, false},
+		}, nil, map[string]string{pg: meshprovisionerclient.Provided}, false, 1},
 		{"gone again: asked again", func() { m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} },
-			[]string{pg}, map[string]string{pg: meshprovisionerclient.Retiring}, false},
+			[]string{pg}, map[string]string{pg: meshprovisionerclient.Retiring}, false, 1},
 	}
 	for _, s := range steps {
 		s.change()
@@ -320,7 +343,40 @@ func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) {
 				t.Errorf("%s: %s is %s, want %s (%s)", s.name, k, p.State, st, p.Why)
 			}
 		}
+		if len(m.r.cancelled) != s.cancelled {
+			t.Errorf("%s: %d asks withdrawn, want %d", s.name, len(m.r.cancelled), s.cancelled)
+		}
+		if s.cancelled == 1 && (len(m.r.cancelled) == 0 || m.state(t, pg).Ask == m.r.cancelled[0]) {
+			t.Errorf("%s: the withdrawn ask %v stays on the key", s.name, m.r.cancelled)
+		}
 	}
+	// A withdrawal the router does not take keeps the ask on the key and says so; the next pass withdraws it.
+	m.r.failCancel = true
+	was := m.state(t, pg).Ask
+	m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)}
+	if c := m.compose(t, "g14"); !strings.Contains(strings.Join(c.Problems, " "), "could not be withdrawn") || m.state(t, pg).Ask != was {
+		t.Errorf("a refused cancel: %v, ask %q (was %q)", c.Problems, m.state(t, pg).Ask, was)
+	}
+	m.r.failCancel = false
+	m.compose(t, "g14")
+	if m.state(t, pg).Ask != "" || !slices.Equal(m.r.cancelled[len(m.r.cancelled)-1:], []string{was}) {
+		t.Errorf("the next pass did not withdraw %s: ask %q, withdrawn %v", was, m.state(t, pg).Ask, m.r.cancelled)
+	}
+	m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
+	m.compose(t, "g14")
+	// A key of another node (a provider's shared credential) needed again withdraws its ask too, by the same helper.
+	retiring := m.state(t, pg)
+	if retiring.Ask == "" {
+		t.Fatal("the key holds no ask")
+	}
+	needed := retiring
+	needed.State, needed.Ask, needed.Warrant = "", "", nil
+	before := len(m.r.cancelled)
+	m.h.judgeElsewhere(map[string]meshprovisionerclient.Provision{pg: needed}, &Composed{})
+	if len(m.r.cancelled) != before+1 || m.r.cancelled[before] != retiring.Ask || m.state(t, pg).Ask != "" {
+		t.Errorf("a credential key needed again: withdrawn %v, ask %q", m.r.cancelled, m.state(t, pg).Ask)
+	}
+	m.compose(t, "g14") // gone again: asked again
 	p := m.state(t, pg)
 	a := m.r.asked[p.Ask]
 	if !strings.Contains(a.Explanation, "postgres") || !strings.Contains(a.Explanation, "ace") ||
@@ -665,6 +721,32 @@ func TestARetirementIsJudgedOnlyFromAWholePicture(t *testing.T) {
 					t.Errorf("the hold is said after it ended: %+v", a)
 				}
 			}},
+		{name: "the newest composition of the consumer is unread: its key is kept, said, and a lasting gap is unhealthy", consumer: 2,
+			gone: 1, change: func(m *mesh) {
+				m.unpublishedNewest["novox"] = []string{"de-spiegel"}
+				m.heldNewest["novox"] = map[string]string{"de-spiegel": "the composition of de-spiegel at " + fp + " is not published yet"}
+			}, after: func(t *testing.T, m *mesh) {
+				c := m.compose(t, "novox")
+				if !strings.Contains(strings.Join(c.Changed, " "), "novox.de-spiegel.route.1 kept: its newest build's composition is unread") {
+					t.Errorf("the pass does not say the key is kept: %v", c.Changed)
+				}
+				// The hold is on the bus, in `refused` beside the wanted build's, so a restart or another instance sees it.
+				if e, _ := m.s.Get(RefusedState, "novox.de-spiegel=newest"); e == nil || !strings.Contains(string(e.Value), `"scope":"newest"`) ||
+					!strings.Contains(string(e.Value), "not published") {
+					t.Errorf("refused holds %s", valueOf(e))
+				}
+				now := m.h.Now()
+				m.h.Now = func() time.Time { return now.Add(unpublishedFor + time.Minute) }
+				m.compose(t, "novox")
+				if a := m.h.health(); a.Healthy || !strings.Contains(a.Why, "novox.de-spiegel at its newest build") {
+					t.Errorf("a lasting gap at the newest build: %+v", a)
+				}
+				m.heldNewest["novox"], m.unpublishedNewest["novox"] = nil, nil
+				m.compose(t, "novox")
+				if m.state(t, "novox.de-spiegel.route.1").State != meshprovisionerclient.Retiring {
+					t.Errorf("read again and not needed, the key does not retire")
+				}
+			}},
 		{name: "a consumer that still has a want: not retired", consumer: 2, gone: 1,
 			change: func(m *mesh) {
 				m.s.write(t, WantsRead, "want.novox.de-spiegel", `{"v":1,"fp":"`+fp+`","self":true}`)

modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go

@@ -42,7 +42,8 @@ func holderTools(h *Holder) []stdio.Tool {
 			Description: "Every module's provisioning plan the seat keeps, <node>.<module>, as the node-engine's published " +
 				"composition carries it (composed-<node>), narrowed by node, with how many keys and bytes the state holds, " +
 				"and each placed module held with its last plan kept and why (refused: the node-engine refused its " +
-				"composition, or has not published it at the wanted build yet); or one plan whole.",
+				"composition, or has not published it at the wanted build yet), and newest-held: each module whose composition at its newest " +
+				"build is unread (not published or refused), whose keys only that build requires are kept; or one plan whole.",
 			Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, <node>.<module>"),
 				"node": str("a node")}},
 			Run: func(a map[string]any) (any, error) {
@@ -55,6 +56,14 @@ func holderTools(h *Holder) []stdio.Tool {
 					return nil, err
 				}
 				out.(map[string]any)[RefusedState] = refused.(map[string]any)[RefusedState]
+				// The modules held at their newest build are the refused rows of that scope, kept on the bus beside the others.
+				newest := []map[string]any{}
+				for _, row := range refused.(map[string]any)[RefusedState].([]map[string]any) {
+					if row["scope"] == "newest" {
+						newest = append(newest, row)
+					}
+				}
+				out.(map[string]any)["newest-held"] = newest
 				return out, nil
 			}},
 		{Name: ComposeTool,
@@ -81,7 +90,7 @@ func holderTools(h *Holder) []stdio.Tool {
 			Description: "The provisioner's runtime health, as the node-engine asks it (novox/hq ADR 0366): unhealthy while " +
 				"a pass held back a retirement of most of a node's consumers at once as a fault of the mesh (issue 940), " +
 				"naming the node, the consumers and the likely cause, or while a module's composition has stayed " +
-				"unpublished at one build for longer than 45 minutes (longer than a build takes), naming the module, " +
+				"unpublished at one build, wanted or newest, for longer than 45 minutes (longer than a build takes), naming the module, " +
 				"the build and since when.",
 			Run: func(map[string]any) (any, error) { return h.health(), nil }},
 	}
@@ -148,7 +157,7 @@ func (h *Holder) read(state, key, node, want string) (any, error) {
 			if err := json.Unmarshal(e.Value, &r); err != nil {
 				row["unreadable"] = err.Error()
 			} else {
-				row["module"], row["why"], row["since"] = r.Module, r.Why, r.Since
+				row["module"], row["why"], row["since"], row["scope"] = r.Module, r.Why, r.Since, r.Scope
 			}
 		}
 		rows = append(rows, row)

modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go

@@ -2,7 +2,9 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
+	"regexp"
 	"slices"
 	"sort"
 	"strings"
@@ -32,7 +34,14 @@ func (m *memStore) Get(state, key string) (*Entry, error) {
 	return &Entry{Key: key, Value: append([]byte(nil), v...)}, nil
 }
 
+// busKey is the bus's key-value key rule (nats.go's validKeyRe): the store refuses what the bus refuses, so a key
+// the live bus cannot hold fails here too.
+var busKey = regexp.MustCompile(`^[-/_=\.a-zA-Z0-9]+$`)
+
 func (m *memStore) Put(state, key string, value any) error {
+	if !busKey.MatchString(key) {
+		return fmt.Errorf("nats: invalid key %q", key)
+	}
 	raw, err := json.Marshal(value)
 	if err != nil {
 		return err