diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go index c8a8b357..96e4bc13 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -37,12 +37,22 @@ import ( "git.novox.be/novox/mesh-sdk/go/provisions" ) -// The states it keeps, beside Registry, Needs, Applied and Collect. +// The states it keeps, beside Registry, Needs, Applied and Collect. RefusedState holds, by `.`, why a +// placed module has no plan composed (issue 774): written while it is held, deleted once it composes again. const ( ProvisionsState = provisions.ProvisionsState PlansState = provisions.PlansState + RefusedState = "refused" ) +// Refusal is one module held on a node, as RefusedState keeps it. +type Refusal struct { + Node string `json:"node"` + Module string `json:"module"` + Why string `json:"why"` + Since time.Time `json:"since"` +} + // The reads it composes from. const ( PlacementRead = delivery.PlacementRead @@ -63,8 +73,11 @@ var BoundReads = []string{"gitea", "keycloak", "postgres", "route-adapter", "mes func boundRead(module string) string { return module + ".bound" } // Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names -// (module → fingerprint), the newest build of every other placed module. -type Composer func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) +// (module → fingerprint), the newest build of every other placed module. A module whose build or plan refuses is held +// (frombus.PlansHeld; issue 774, one bad module must not drop its node): in held with why, none of its plan in plans. +// err is a refusal that is no single module's. +type Composer func(ctx context.Context, node string, keptAt map[string]string) (plans map[string]catalogue.Plan, + held map[string]string, err error) // Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's // record of it. @@ -86,12 +99,15 @@ type Holder struct { // Composed is what one node's composition did. type Composed struct { - Node string `json:"node"` - Plans []string `json:"plans-written,omitempty"` - Gone []string `json:"plans-removed,omitempty"` - Changed []string `json:"provisions-changed,omitempty"` - Asked []string `json:"retiring-asked,omitempty"` - Problems []string `json:"problems,omitempty"` + Node string `json:"node"` + Plans []string `json:"plans-written,omitempty"` + Gone []string `json:"plans-removed,omitempty"` + Changed []string `json:"provisions-changed,omitempty"` + Asked []string `json:"retiring-asked,omitempty"` + // Held is each module placed on the node whose plan is not composed, with why: its last plan and its provisions + // stay as they were, and every other module is composed (issue 774). + Held map[string]string `json:"modules-held,omitempty"` + Problems []string `json:"problems,omitempty"` } // ComposeNode composes one node: its plans and its provisions. @@ -110,15 +126,28 @@ func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) wanted[module] = fp } } - atWants, err := h.Compose(ctx, node, wanted) + atWants, heldAtWants, err := h.Compose(ctx, node, wanted) if err != nil { return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err) } - newest, err := h.Compose(ctx, node, nil) + newest, heldNewest, err := h.Compose(ctx, node, nil) if err != nil { return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err) } - if err := h.writePlans(node, atWants, &out); err != nil { + // A module held at its wanted build has no plan composed: its last plan and its provisions are kept, said in + // `refused`. One held at its newest build alone keeps its plan at its want; what its newest build needs ahead waits. + if len(heldAtWants) > 0 { + out.Held = heldAtWants + } + for _, m := range sortedKeys(heldNewest) { + if _, ok := heldAtWants[m]; !ok { + out.Problems = append(out.Problems, fmt.Sprintf("%s at its newest build is not composed: %s", m, heldNewest[m])) + } + } + if err := h.writeRefused(node, heldAtWants); err != nil { + out.Problems = append(out.Problems, err.Error()) + } + if err := h.writePlans(node, atWants, heldAtWants, &out); err != nil { out.Problems = append(out.Problems, err.Error()) } needed := map[string]provisions.Provision{} @@ -166,10 +195,59 @@ func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) } } h.judgeElsewhere(elsewhere, &out) - h.judgeNode(node, needed, &out) + h.judgeNode(node, needed, heldAtWants, &out) return out, nil } +// writeRefused keeps, for each module held on node, why, and deletes the refusal of one that composes again: a change +// of the state is the event a watcher hears. +func (h *Holder) writeRefused(node string, held map[string]string) error { + var problems []error + for _, module := range sortedKeys(held) { + key, err := provisions.PlanKey(node, module) + if err != nil { + problems = append(problems, err) + continue + } + r := Refusal{Node: node, Module: module, Why: held[module], Since: h.Now().UTC()} + if e, err := h.Store.Get(RefusedState, key); err == nil && e != nil { + var was Refusal + if json.Unmarshal(e.Value, &was) == nil && was.Why == r.Why { + continue + } + } + raw, err := json.Marshal(r) + if err != nil { + problems = append(problems, err) + continue + } + if err := h.Store.Put(RefusedState, key, json.RawMessage(raw)); err != nil { + problems = append(problems, fmt.Errorf("the refusal %s: %w", key, err)) + continue + } + h.Logf("[%s] %s on %s has no plan composed: %s", Module, module, node, r.Why) + } + keys, err := h.Store.Keys(RefusedState) + if err != nil { + return errors.Join(append(problems, err)...) + } + for _, k := range keys { + n, module, ok := strings.Cut(k, ".") + if !ok || n != node { + continue + } + if _, still := held[module]; still { + continue + } + if err := h.Store.Delete(RefusedState, k); err != nil { + problems = append(problems, err) + continue + } + h.Logf("[%s] %s on %s composes again", Module, module, node) + } + return errors.Join(problems...) +} + // providerPlan is the plan of a provision's provider on its node: from this composition when it is on this node, else // as kept in `plans`; nil when the provider is not handed. func (h *Holder) providerPlan(p catalogue.PlanProvision, node string, here map[string]catalogue.Plan) *catalogue.Plan { @@ -297,8 +375,9 @@ func (h *Holder) runningOn(node string) map[string]string { return out } -// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there. -func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error { +// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there; a held +// module's plan is kept as it was (issue 774). +func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, held map[string]string, out *Composed) error { var problems []error for _, module := range sortedKeys(plans) { key, err := provisions.PlanKey(node, module) @@ -332,6 +411,9 @@ func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *C if _, still := plans[module]; still { continue } + if _, kept := held[module]; kept { + continue + } if err := h.Store.Delete(PlansState, k); err != nil { problems = append(problems, err) continue @@ -341,9 +423,17 @@ func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *C return errors.Join(problems...) } -// judgeNode brings the node's provision keys to what is needed and what each provider said. -func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, out *Composed) { +// judgeNode brings the node's provision keys to what is needed and what each provider said. A key of a module held on +// the node (heldModules) that its composition no longer names is left as it is: not composed is not unneeded. +func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, heldModules map[string]string, out *Composed) { held := h.held(node) + for key, have := range held { + if _, isNeeded := needed[key]; !isNeeded && have.Consumer.Node == node { + if _, notComposed := heldModules[have.Consumer.Module]; notComposed { + delete(held, key) + } + } + } // A key of this node another node's provision depends on is needed (a provider's one shared credential). for _, dep := range h.dependedOn(node) { if have, ok := held[dep]; ok { @@ -902,7 +992,7 @@ func orSaid(words ...string) string { // credential is read (compose.Planning). func FromBus() Composer { r := fromruntime.Reader() - return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { - return frombus.Plans(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) + return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, map[string]string, error) { + return frombus.PlansHeld(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go index 9aa766c3..4b650470 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go @@ -168,8 +168,8 @@ func TestTheHolderComposesProvidesAndAsksOnTheBus(t *testing.T) { } a := &asked{} h := &Holder{Store: s, Asks: a, Now: time.Now, Logf: t.Logf, - Compose: func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { - return frombus.Plans(ctx, mesh, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) + Compose: func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, map[string]string, error) { + return frombus.PlansHeld(ctx, mesh, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) }} state := func(key string) provisions.Provision { e, _ := s.Get(ProvisionsState, key) diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go index 238b589c..c26fd22e 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go @@ -56,21 +56,23 @@ type mesh struct { r *router atWant map[string]map[string]catalogue.Plan atNewest map[string]map[string]catalogue.Plan + heldAt map[string]map[string]string // by node, the modules its composition at the wanted builds holds composed []string } func newMesh() *mesh { m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}}, - atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}} + atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}, + heldAt: map[string]map[string]string{}} now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {}, Now: func() time.Time { now = now.Add(time.Second); return now }, - Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, map[string]string, error) { m.composed = append(m.composed, node) if keptAt != nil { - return m.atWant[node], nil + return m.atWant[node], m.heldAt[node], nil } - return m.atNewest[node], nil + return m.atNewest[node], nil, nil }} // What each provider's manifest declares: postgres, keycloak and the vault keep entries, the others report by their // have. @@ -314,6 +316,56 @@ func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) { } } +// Issue 774, one bad module must not drop its node: a module whose manifest or plan refuses is held, named in `refused` +// with why, its last plan and its provisions kept; every other module on the node is composed and written; once it +// composes again its refusal goes. +func TestOneBadModuleDoesNotDropItsNode(t *testing.T) { + m := newMesh() + why := "the built manifest of nats at src2:6258f3 cannot be used" + steps := []struct { + name string + plans map[string]catalogue.Plan + held map[string]string + written []string // plans the step writes + refused bool // g14.nats in refused + }{ + {"a node new to the seat with one bad module: the others are composed", + map[string]catalogue.Plan{"gitea": planOf(pgDB)}, map[string]string{"nats": why}, []string{"g14.gitea"}, true}, + {"the bad one composes: its plan is written and its refusal goes", + map[string]catalogue.Plan{"gitea": planOf(pgDB), "nats": planOf(oidc)}, nil, []string{"g14.nats"}, false}, + {"it refuses again: its last plan and its provision are kept, the others composed", + map[string]catalogue.Plan{"gitea": planOf(pgDB, split)}, map[string]string{"nats": why}, []string{"g14.gitea"}, true}, + } + for _, s := range steps { + m.atWant["g14"], m.atNewest["g14"], m.heldAt["g14"] = s.plans, s.plans, s.held + c := m.compose(t, "g14") + if !slices.Equal(c.Plans, s.written) || len(c.Gone) != 0 || len(c.Asked) != 0 { + t.Errorf("%s: %+v", s.name, c) + } + e, _ := m.s.Get(RefusedState, "g14.nats") + var r Refusal + if e != nil { + _ = json.Unmarshal(e.Value, &r) + } + if (e != nil) != s.refused || (s.refused && (r.Module != "nats" || r.Why != why || c.Held["nats"] != why)) { + t.Errorf("%s: refused %+v, held %v", s.name, r, c.Held) + } + if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil { + t.Errorf("%s: gitea has no plan", s.name) + } + } + if e, _ := m.s.Get(PlansState, "g14.nats"); e == nil { + t.Error("the held module's last plan was removed") + } + if p := m.state(t, "g14.nats.oidc-client"); p.State != provisions.Asked { + t.Errorf("the held module's provision is %s (%s), not kept", p.State, p.Why) + } + got, err := m.h.read(RefusedState, "", "g14", "") + if rows := got.(map[string]any)[RefusedState].([]map[string]any); err != nil || len(rows) != 1 || rows[0]["why"] != why { + t.Errorf("the refusal reads %v, %v", got, err) + } +} + // Review of 2026-10-30: a module held on a node with no want (the bus on the control-node) is composed at the build // its have says it runs, so the holder writes it a plan and its client keeps filling it. func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) { @@ -322,12 +374,12 @@ func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) { m.s.write(t, HaveRead, "novox.nats", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`) m.s.write(t, HaveRead, "g14.dunst", `{"v":1,"fp":"src2:`+strings.Repeat("ee", 32)+`","phase":"running","health":"ok"}`) var keptAt map[string]string - m.h.Compose = func(_ context.Context, node string, at map[string]string) (map[string]catalogue.Plan, error) { + m.h.Compose = func(_ context.Context, node string, at map[string]string) (map[string]catalogue.Plan, map[string]string, error) { if at != nil { keptAt = at - return map[string]catalogue.Plan{"nats": {}}, nil + return map[string]catalogue.Plan{"nats": {}}, nil, nil } - return nil, nil + return nil, nil, nil } c := m.compose(t, "novox") if keptAt["nats"] != fp || len(keptAt) != 1 { diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go index 30a16446..8690cd73 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go @@ -37,10 +37,22 @@ func holderTools(h *Holder) []stdio.Tool { }}, {Name: PlansTool, Description: "Every module's provisioning plan the seat composed, ., narrowed by node, with how many " + - "keys and bytes the state holds; or one plan whole.", + "keys and bytes the state holds, and each placed module held with no plan composed and why (refused: its " + + "manifest or its plan refuses, the node's other modules composed); or one plan whole.", Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, ."), "node": str("a node")}}, - Run: func(a map[string]any) (any, error) { return h.read(PlansState, arg(a, "key"), arg(a, "node"), "") }}, + Run: func(a map[string]any) (any, error) { + out, err := h.read(PlansState, arg(a, "key"), arg(a, "node"), "") + if err != nil || arg(a, "key") != "" { + return out, err + } + refused, err := h.read(RefusedState, "", arg(a, "node"), "") + if err != nil { + return nil, err + } + out.(map[string]any)[RefusedState] = refused.(map[string]any)[RefusedState] + return out, nil + }}, {Name: ComposeTool, Description: "Compose one node's plans and provisions now from the bus, as a change event would, and say what " + "was written, removed, changed and asked.", @@ -98,6 +110,14 @@ func (h *Holder) read(state, key, node, want string) (any, error) { row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why } } + if state == RefusedState { + var r Refusal + if err := json.Unmarshal(e.Value, &r); err != nil { + row["unreadable"] = err.Error() + } else { + row["module"], row["why"], row["since"] = r.Module, r.Why, r.Since + } + } rows = append(rows, row) } return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go index dd0ccde3..f717ea06 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go @@ -305,7 +305,7 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { t.Errorf("the claim: %+v", m.Claims) } // needs stays declared this release so its first start can empty it (ADR 0570 clause 1). - if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState}; !slices.Equal(m.State, want) { + if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState, RefusedState}; !slices.Equal(m.State, want) { t.Errorf("the state is %v, want %v", m.State, want) } // Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools diff --git a/modules/mesh-provisioner/go.mod b/modules/mesh-provisioner/go.mod index db34240f..e2cfbde4 100644 --- a/modules/mesh-provisioner/go.mod +++ b/modules/mesh-provisioner/go.mod @@ -3,7 +3,7 @@ module mesh-provisioner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030165056-59311407e240 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-provisioner/go.sum b/modules/mesh-provisioner/go.sum index a2b8e018..fdb63f14 100644 --- a/modules/mesh-provisioner/go.sum +++ b/modules/mesh-provisioner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030165056-59311407e240 h1:oZUgNXIhEV6YN//k3k7dku357TlR3suApfJFdLHVUog= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030165056-59311407e240/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-provisioner/module.json b/modules/mesh-provisioner/module.json index 1aba09af..b417b373 100644 --- a/modules/mesh-provisioner/module.json +++ b/modules/mesh-provisioner/module.json @@ -40,7 +40,8 @@ "applied", "collect", "provisions", - "plans" + "plans", + "refused" ], "own-secrets": { "broker": "${dir:state}/broker"