One bad module must not drop its node: the provisioner composes each module's plan on its own (issue 774)

novox since 15:36 (nats's built manifest) and ace since 15:37 (baserow's config.json) had no plans at all, so the
vault refused every secret their clients asked. The holder now composes with frombus.PlansHeld: a module whose build
or plan refuses is held, its why kept in the new state refused and read by provisioner_plans, its last plan and its
provisions left as they were, and every other module on the node composed and written. Pins mesh-sdk 59311407.

modules/mesh-provisioner/cmd/mesh-provisioner/holder.go

@@ -37,12 +37,22 @@ import (
 	"git.novox.be/novox/mesh-sdk/go/provisions"
 )
 
-// The states it keeps, beside Registry, Needs, Applied and Collect.
+// The states it keeps, beside Registry, Needs, Applied and Collect. RefusedState holds, by `<node>.<module>`, why a
+// placed module has no plan composed (issue 774): written while it is held, deleted once it composes again.
 const (
 	ProvisionsState = provisions.ProvisionsState
 	PlansState      = provisions.PlansState
+	RefusedState    = "refused"
 )
 
+// Refusal is one module held on a node, as RefusedState keeps it.
+type Refusal struct {
+	Node   string    `json:"node"`
+	Module string    `json:"module"`
+	Why    string    `json:"why"`
+	Since  time.Time `json:"since"`
+}
+
 // The reads it composes from.
 const (
 	PlacementRead = delivery.PlacementRead
@@ -63,8 +73,11 @@ var BoundReads = []string{"gitea", "keycloak", "postgres", "route-adapter", "mes
 func boundRead(module string) string { return module + ".bound" }
 
 // Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names
-// (module → fingerprint), the newest build of every other placed module.
-type Composer func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error)
+// (module → fingerprint), the newest build of every other placed module. A module whose build or plan refuses is held
+// (frombus.PlansHeld; issue 774, one bad module must not drop its node): in held with why, none of its plan in plans.
+// err is a refusal that is no single module's.
+type Composer func(ctx context.Context, node string, keptAt map[string]string) (plans map[string]catalogue.Plan,
+	held map[string]string, err error)
 
 // Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's
 // record of it.
@@ -86,12 +99,15 @@ type Holder struct {
 
 // Composed is what one node's composition did.
 type Composed struct {
-	Node     string   `json:"node"`
-	Plans    []string `json:"plans-written,omitempty"`
-	Gone     []string `json:"plans-removed,omitempty"`
-	Changed  []string `json:"provisions-changed,omitempty"`
-	Asked    []string `json:"retiring-asked,omitempty"`
-	Problems []string `json:"problems,omitempty"`
+	Node    string   `json:"node"`
+	Plans   []string `json:"plans-written,omitempty"`
+	Gone    []string `json:"plans-removed,omitempty"`
+	Changed []string `json:"provisions-changed,omitempty"`
+	Asked   []string `json:"retiring-asked,omitempty"`
+	// Held is each module placed on the node whose plan is not composed, with why: its last plan and its provisions
+	// stay as they were, and every other module is composed (issue 774).
+	Held     map[string]string `json:"modules-held,omitempty"`
+	Problems []string          `json:"problems,omitempty"`
 }
 
 // ComposeNode composes one node: its plans and its provisions.
@@ -110,15 +126,28 @@ func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error)
 			wanted[module] = fp
 		}
 	}
-	atWants, err := h.Compose(ctx, node, wanted)
+	atWants, heldAtWants, err := h.Compose(ctx, node, wanted)
 	if err != nil {
 		return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err)
 	}
-	newest, err := h.Compose(ctx, node, nil)
+	newest, heldNewest, err := h.Compose(ctx, node, nil)
 	if err != nil {
 		return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err)
 	}
-	if err := h.writePlans(node, atWants, &out); err != nil {
+	// A module held at its wanted build has no plan composed: its last plan and its provisions are kept, said in
+	// `refused`. One held at its newest build alone keeps its plan at its want; what its newest build needs ahead waits.
+	if len(heldAtWants) > 0 {
+		out.Held = heldAtWants
+	}
+	for _, m := range sortedKeys(heldNewest) {
+		if _, ok := heldAtWants[m]; !ok {
+			out.Problems = append(out.Problems, fmt.Sprintf("%s at its newest build is not composed: %s", m, heldNewest[m]))
+		}
+	}
+	if err := h.writeRefused(node, heldAtWants); err != nil {
+		out.Problems = append(out.Problems, err.Error())
+	}
+	if err := h.writePlans(node, atWants, heldAtWants, &out); err != nil {
 		out.Problems = append(out.Problems, err.Error())
 	}
 	needed := map[string]provisions.Provision{}
@@ -166,10 +195,59 @@ func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error)
 		}
 	}
 	h.judgeElsewhere(elsewhere, &out)
-	h.judgeNode(node, needed, &out)
+	h.judgeNode(node, needed, heldAtWants, &out)
 	return out, nil
 }
 
+// writeRefused keeps, for each module held on node, why, and deletes the refusal of one that composes again: a change
+// of the state is the event a watcher hears.
+func (h *Holder) writeRefused(node string, held map[string]string) error {
+	var problems []error
+	for _, module := range sortedKeys(held) {
+		key, err := provisions.PlanKey(node, module)
+		if err != nil {
+			problems = append(problems, err)
+			continue
+		}
+		r := Refusal{Node: node, Module: module, Why: held[module], Since: h.Now().UTC()}
+		if e, err := h.Store.Get(RefusedState, key); err == nil && e != nil {
+			var was Refusal
+			if json.Unmarshal(e.Value, &was) == nil && was.Why == r.Why {
+				continue
+			}
+		}
+		raw, err := json.Marshal(r)
+		if err != nil {
+			problems = append(problems, err)
+			continue
+		}
+		if err := h.Store.Put(RefusedState, key, json.RawMessage(raw)); err != nil {
+			problems = append(problems, fmt.Errorf("the refusal %s: %w", key, err))
+			continue
+		}
+		h.Logf("[%s] %s on %s has no plan composed: %s", Module, module, node, r.Why)
+	}
+	keys, err := h.Store.Keys(RefusedState)
+	if err != nil {
+		return errors.Join(append(problems, err)...)
+	}
+	for _, k := range keys {
+		n, module, ok := strings.Cut(k, ".")
+		if !ok || n != node {
+			continue
+		}
+		if _, still := held[module]; still {
+			continue
+		}
+		if err := h.Store.Delete(RefusedState, k); err != nil {
+			problems = append(problems, err)
+			continue
+		}
+		h.Logf("[%s] %s on %s composes again", Module, module, node)
+	}
+	return errors.Join(problems...)
+}
+
 // providerPlan is the plan of a provision's provider on its node: from this composition when it is on this node, else
 // as kept in `plans`; nil when the provider is not handed.
 func (h *Holder) providerPlan(p catalogue.PlanProvision, node string, here map[string]catalogue.Plan) *catalogue.Plan {
@@ -297,8 +375,9 @@ func (h *Holder) runningOn(node string) map[string]string {
 	return out
 }
 
-// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there.
-func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error {
+// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there; a held
+// module's plan is kept as it was (issue 774).
+func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, held map[string]string, out *Composed) error {
 	var problems []error
 	for _, module := range sortedKeys(plans) {
 		key, err := provisions.PlanKey(node, module)
@@ -332,6 +411,9 @@ func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *C
 		if _, still := plans[module]; still {
 			continue
 		}
+		if _, kept := held[module]; kept {
+			continue
+		}
 		if err := h.Store.Delete(PlansState, k); err != nil {
 			problems = append(problems, err)
 			continue
@@ -341,9 +423,17 @@ func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *C
 	return errors.Join(problems...)
 }
 
-// judgeNode brings the node's provision keys to what is needed and what each provider said.
-func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, out *Composed) {
+// judgeNode brings the node's provision keys to what is needed and what each provider said. A key of a module held on
+// the node (heldModules) that its composition no longer names is left as it is: not composed is not unneeded.
+func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, heldModules map[string]string, out *Composed) {
 	held := h.held(node)
+	for key, have := range held {
+		if _, isNeeded := needed[key]; !isNeeded && have.Consumer.Node == node {
+			if _, notComposed := heldModules[have.Consumer.Module]; notComposed {
+				delete(held, key)
+			}
+		}
+	}
 	// A key of this node another node's provision depends on is needed (a provider's one shared credential).
 	for _, dep := range h.dependedOn(node) {
 		if have, ok := held[dep]; ok {
@@ -902,7 +992,7 @@ func orSaid(words ...string) string {
 // credential is read (compose.Planning).
 func FromBus() Composer {
 	r := fromruntime.Reader()
-	return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) {
-		return frombus.Plans(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt})
+	return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, map[string]string, error) {
+		return frombus.PlansHeld(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt})
 	}
 }

modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go

@@ -168,8 +168,8 @@ func TestTheHolderComposesProvidesAndAsksOnTheBus(t *testing.T) {
 	}
 	a := &asked{}
 	h := &Holder{Store: s, Asks: a, Now: time.Now, Logf: t.Logf,
-		Compose: func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) {
-			return frombus.Plans(ctx, mesh, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt})
+		Compose: func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, map[string]string, error) {
+			return frombus.PlansHeld(ctx, mesh, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt})
 		}}
 	state := func(key string) provisions.Provision {
 		e, _ := s.Get(ProvisionsState, key)

modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go

@@ -56,21 +56,23 @@ type mesh struct {
 	r        *router
 	atWant   map[string]map[string]catalogue.Plan
 	atNewest map[string]map[string]catalogue.Plan
+	heldAt   map[string]map[string]string // by node, the modules its composition at the wanted builds holds
 	composed []string
 }
 
 func newMesh() *mesh {
 	m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}},
-		atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}}
+		atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{},
+		heldAt: map[string]map[string]string{}}
 	now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
 	m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {},
 		Now: func() time.Time { now = now.Add(time.Second); return now },
-		Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) {
+		Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, map[string]string, error) {
 			m.composed = append(m.composed, node)
 			if keptAt != nil {
-				return m.atWant[node], nil
+				return m.atWant[node], m.heldAt[node], nil
 			}
-			return m.atNewest[node], nil
+			return m.atNewest[node], nil, nil
 		}}
 	// What each provider's manifest declares: postgres, keycloak and the vault keep entries, the others report by their
 	// have.
@@ -314,6 +316,56 @@ func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) {
 	}
 }
 
+// Issue 774, one bad module must not drop its node: a module whose manifest or plan refuses is held, named in `refused`
+// with why, its last plan and its provisions kept; every other module on the node is composed and written; once it
+// composes again its refusal goes.
+func TestOneBadModuleDoesNotDropItsNode(t *testing.T) {
+	m := newMesh()
+	why := "the built manifest of nats at src2:6258f3 cannot be used"
+	steps := []struct {
+		name    string
+		plans   map[string]catalogue.Plan
+		held    map[string]string
+		written []string // plans the step writes
+		refused bool     // g14.nats in refused
+	}{
+		{"a node new to the seat with one bad module: the others are composed",
+			map[string]catalogue.Plan{"gitea": planOf(pgDB)}, map[string]string{"nats": why}, []string{"g14.gitea"}, true},
+		{"the bad one composes: its plan is written and its refusal goes",
+			map[string]catalogue.Plan{"gitea": planOf(pgDB), "nats": planOf(oidc)}, nil, []string{"g14.nats"}, false},
+		{"it refuses again: its last plan and its provision are kept, the others composed",
+			map[string]catalogue.Plan{"gitea": planOf(pgDB, split)}, map[string]string{"nats": why}, []string{"g14.gitea"}, true},
+	}
+	for _, s := range steps {
+		m.atWant["g14"], m.atNewest["g14"], m.heldAt["g14"] = s.plans, s.plans, s.held
+		c := m.compose(t, "g14")
+		if !slices.Equal(c.Plans, s.written) || len(c.Gone) != 0 || len(c.Asked) != 0 {
+			t.Errorf("%s: %+v", s.name, c)
+		}
+		e, _ := m.s.Get(RefusedState, "g14.nats")
+		var r Refusal
+		if e != nil {
+			_ = json.Unmarshal(e.Value, &r)
+		}
+		if (e != nil) != s.refused || (s.refused && (r.Module != "nats" || r.Why != why || c.Held["nats"] != why)) {
+			t.Errorf("%s: refused %+v, held %v", s.name, r, c.Held)
+		}
+		if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil {
+			t.Errorf("%s: gitea has no plan", s.name)
+		}
+	}
+	if e, _ := m.s.Get(PlansState, "g14.nats"); e == nil {
+		t.Error("the held module's last plan was removed")
+	}
+	if p := m.state(t, "g14.nats.oidc-client"); p.State != provisions.Asked {
+		t.Errorf("the held module's provision is %s (%s), not kept", p.State, p.Why)
+	}
+	got, err := m.h.read(RefusedState, "", "g14", "")
+	if rows := got.(map[string]any)[RefusedState].([]map[string]any); err != nil || len(rows) != 1 || rows[0]["why"] != why {
+		t.Errorf("the refusal reads %v, %v", got, err)
+	}
+}
+
 // Review of 2026-10-30: a module held on a node with no want (the bus on the control-node) is composed at the build
 // its have says it runs, so the holder writes it a plan and its client keeps filling it.
 func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) {
@@ -322,12 +374,12 @@ func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) {
 	m.s.write(t, HaveRead, "novox.nats", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`)
 	m.s.write(t, HaveRead, "g14.dunst", `{"v":1,"fp":"src2:`+strings.Repeat("ee", 32)+`","phase":"running","health":"ok"}`)
 	var keptAt map[string]string
-	m.h.Compose = func(_ context.Context, node string, at map[string]string) (map[string]catalogue.Plan, error) {
+	m.h.Compose = func(_ context.Context, node string, at map[string]string) (map[string]catalogue.Plan, map[string]string, error) {
 		if at != nil {
 			keptAt = at
-			return map[string]catalogue.Plan{"nats": {}}, nil
+			return map[string]catalogue.Plan{"nats": {}}, nil, nil
 		}
-		return nil, nil
+		return nil, nil, nil
 	}
 	c := m.compose(t, "novox")
 	if keptAt["nats"] != fp || len(keptAt) != 1 {

modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go

@@ -37,10 +37,22 @@ func holderTools(h *Holder) []stdio.Tool {
 			}},
 		{Name: PlansTool,
 			Description: "Every module's provisioning plan the seat composed, <node>.<module>, narrowed by node, with how many " +
-				"keys and bytes the state holds; or one plan whole.",
+				"keys and bytes the state holds, and each placed module held with no plan composed and why (refused: its " +
+				"manifest or its plan refuses, the node's other modules composed); or one plan whole.",
 			Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, <node>.<module>"),
 				"node": str("a node")}},
-			Run: func(a map[string]any) (any, error) { return h.read(PlansState, arg(a, "key"), arg(a, "node"), "") }},
+			Run: func(a map[string]any) (any, error) {
+				out, err := h.read(PlansState, arg(a, "key"), arg(a, "node"), "")
+				if err != nil || arg(a, "key") != "" {
+					return out, err
+				}
+				refused, err := h.read(RefusedState, "", arg(a, "node"), "")
+				if err != nil {
+					return nil, err
+				}
+				out.(map[string]any)[RefusedState] = refused.(map[string]any)[RefusedState]
+				return out, nil
+			}},
 		{Name: ComposeTool,
 			Description: "Compose one node's plans and provisions now from the bus, as a change event would, and say what " +
 				"was written, removed, changed and asked.",
@@ -98,6 +110,14 @@ func (h *Holder) read(state, key, node, want string) (any, error) {
 				row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why
 			}
 		}
+		if state == RefusedState {
+			var r Refusal
+			if err := json.Unmarshal(e.Value, &r); err != nil {
+				row["unreadable"] = err.Error()
+			} else {
+				row["module"], row["why"], row["since"] = r.Module, r.Why, r.Since
+			}
+		}
 		rows = append(rows, row)
 	}
 	return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil

modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go

@@ -305,7 +305,7 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) {
 		t.Errorf("the claim: %+v", m.Claims)
 	}
 	// needs stays declared this release so its first start can empty it (ADR 0570 clause 1).
-	if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState}; !slices.Equal(m.State, want) {
+	if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState, RefusedState}; !slices.Equal(m.State, want) {
 		t.Errorf("the state is %v, want %v", m.State, want)
 	}
 	// Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools

modules/mesh-provisioner/go.mod

@@ -3,7 +3,7 @@ module mesh-provisioner
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030165056-59311407e240
 	github.com/nats-io/nats.go v1.54.0
 )
 

modules/mesh-provisioner/go.sum

@@ -1,5 +1,5 @@
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030165056-59311407e240 h1:oZUgNXIhEV6YN//k3k7dku357TlR3suApfJFdLHVUog=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030165056-59311407e240/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

modules/mesh-provisioner/module.json

@@ -40,7 +40,8 @@
     "applied",
     "collect",
     "provisions",
-    "plans"
+    "plans",
+    "refused"
   ],
   "own-secrets": {
     "broker": "${dir:state}/broker"