Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955)
mesh/delivery-group group feat/774-l3-reland rejected: composed together: the checker's verdict at the members' heads (b48c90fea31f): integration: mesh-self-ch…
mesh/delivery closed: a newer head of the same pull request
mesh/check failed: integration: mesh-self-check/cmd/mesh-self-check.TestTheSelfCheckWholeRunsOnTheBus: the run answered {Run:doctor-1793351737-1 At:20…

One commit with exactly the tree of the reviewed head 35b4509c0a, on main 12c2574c9d, so two compiled programs added and removed on the branch leave no trace in history (issue 1577).
This commit is contained in:
2026-10-30 02:21:51 +01:00
parent 12c2574c9d
commit 35ff67bda0
331 changed files with 24352 additions and 1792 deletions
@@ -0,0 +1,908 @@
package main
// The seat's holder (novox/hq ADR 0570 clauses 1 to 5, 7; issue 1954 H1-H3): mesh-provisioner composes every placed
// module's provisioning plan and every provision from bus state, on change events and never on a timer, and keeps them
// in its states `plans` (`<node>.<module>`) and `provisions` (`<node>.<consumer>.<provision>[.<local>]`), which it alone
// writes.
//
// - **Composed per node.** A change of a node's placement or of one of its wants composes that node; a change of the
// indexer's module composes the nodes the module is placed on. Each node is composed twice: at the builds its wants
// name (its plans, what its client applies) and at the newest build of each placed module, so a build that adds a
// requirement has its key before its want (clause 2). A provision is needed while either needs it (clause 5).
// - **A key's state comes from its provider's outcome alone** (clause 3, 7): the provider's own entry for that
// consumer, `<provider>_bound` under the same key: present is provided (the vault's generation carried into the
// reference, clause 6), refused is failed with why, waits is waits with what. The holder never sets provided by
// itself: a key of a provider that writes no outcome stays asked.
// - **Retiring is asked** (clause 5): a key no longer needed is `retiring`, one approve ask under mesh-provisioner's
// name says what goes from which provider on which node; the warrant goes into the key and the provider acts on it;
// a decline is kept and not asked again while nothing changes; a key needed again returns to its outcome's state.
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/compose/fromruntime"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// The states it keeps, beside Registry, Needs, Applied and Collect.
const (
ProvisionsState = provisions.ProvisionsState
PlansState = provisions.PlansState
)
// The reads it composes from.
const (
PlacementRead = delivery.PlacementRead
WantsRead = delivery.AssignerModule + ".wants"
ModulesRead = delivery.IndexerModule + ".modules"
ManifestsRead = delivery.IndexerModule + ".manifests"
BuildsRead = delivery.BuilderModule + ".builds"
HaveRead = "node-engine.have"
ResolvedRead = "mesh-config.resolved"
)
// BoundReads are the providers' entries this module's manifest reads, `<module>.bound`, whose changes it hears. Which
// path judges a provider is its manifest's word (keepsEntries), not this list: a provider that declares `bound` and is
// not read here leaves its keys asked, saying its entries are not read.
var BoundReads = []string{"gitea", "keycloak", "postgres", "route-adapter", "mesh-vault"}
// boundRead is a provider's entries as this module reads them.
func boundRead(module string) string { return module + ".bound" }
// Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names
// (module → fingerprint), the newest build of every other placed module.
type Composer func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error)
// Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's
// record of it.
type Asker interface {
Publish(subject string, body any, id string) error
Record(id string) ([]byte, bool, error)
}
// Holder is the seat's holder.
type Holder struct {
Store Store
Compose Composer
Asks Asker
Now func() time.Time
Logf func(format string, args ...any)
mu sync.Mutex
}
// Composed is what one node's composition did.
type Composed struct {
Node string `json:"node"`
Plans []string `json:"plans-written,omitempty"`
Gone []string `json:"plans-removed,omitempty"`
Changed []string `json:"provisions-changed,omitempty"`
Asked []string `json:"retiring-asked,omitempty"`
Problems []string `json:"problems,omitempty"`
}
// ComposeNode composes one node: its plans and its provisions.
func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) {
h.mu.Lock()
defer h.mu.Unlock()
out := Composed{Node: node}
wanted, err := h.wantedOn(node)
if err != nil {
return out, err
}
// A module held on the node with no want (the bus on the control-node) is composed at the build its have says it
// runs, so it has a plan too and its client keeps filling it (review of 2026-10-30).
for module, fp := range h.runningOn(node) {
if _, ok := wanted[module]; !ok {
wanted[module] = fp
}
}
atWants, err := h.Compose(ctx, node, wanted)
if err != nil {
return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err)
}
newest, err := h.Compose(ctx, node, nil)
if err != nil {
return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err)
}
if err := h.writePlans(node, atWants, &out); err != nil {
out.Problems = append(out.Problems, err.Error())
}
needed := map[string]provisions.Provision{}
elsewhere := map[string]provisions.Provision{}
vaultNode := h.vaultNode()
for _, plans := range []map[string]catalogue.Plan{atWants, newest} {
for module, plan := range plans {
for _, p := range plan.Provisions {
consumer := provisions.End{Module: module, Node: node}
provider := provisions.End{Module: p.ProviderModule, Node: p.Provider}
pv := provisions.Provision{Provision: p.Provision, Local: p.Local, Consumer: consumer, Provider: provider,
DependsOn: provisions.DependsOn(consumer, p, h.providerPlan(p, node, atWants))}
key, err := pv.Key()
if err != nil {
out.Problems = append(out.Problems, err.Error())
continue
}
if _, seen := needed[key]; !seen {
needed[key] = pv
}
// Its credential is a provision of its own, the vault's (clause 6): on the consumer's node for a value of the
// consumer's own, on the provider's for the provider's one shared value.
ck, ok := provisions.CredentialKey(p.CredentialForm, consumer, provider, p.Provision, p.Local)
if !ok {
continue
}
cn, cm, cp, cl, err := provisions.ParseKey(ck)
if err != nil {
continue
}
if vaultNode == "" {
out.Problems = append(out.Problems, "no node runs mesh-vault, so the credential "+ck+" has no provider")
continue
}
cred := provisions.Provision{Provision: cp, Local: cl, Consumer: provisions.End{Module: cm, Node: cn},
Provider: provisions.End{Module: provisions.VaultModule, Node: vaultNode}}
if cn == node {
if _, seen := needed[ck]; !seen {
needed[ck] = cred
}
} else {
elsewhere[ck] = cred
}
}
}
}
h.judgeElsewhere(elsewhere, &out)
h.judgeNode(node, needed, &out)
return out, nil
}
// providerPlan is the plan of a provision's provider on its node: from this composition when it is on this node, else
// as kept in `plans`; nil when the provider is not handed.
func (h *Holder) providerPlan(p catalogue.PlanProvision, node string, here map[string]catalogue.Plan) *catalogue.Plan {
if p.Provider == node {
if plan, ok := here[p.ProviderModule]; ok {
return &plan
}
return nil
}
key, err := provisions.PlanKey(p.Provider, p.ProviderModule)
if err != nil {
return nil
}
e, err := h.Store.Get(PlansState, key)
if err != nil || e == nil {
return nil
}
plan, err := provisions.DecodePlan(e.Value)
if err != nil {
return nil
}
return &plan.Plan
}
// vaultNode is the node mesh-vault is placed on: the provider of every credential.
func (h *Holder) vaultNode() string {
nodes := h.placedOn(provisions.VaultModule)
if len(nodes) == 0 {
return ""
}
return nodes[0]
}
// dependedOn is every key of a node that a provision of another node depends on.
func (h *Holder) dependedOn(node string) []string {
var out []string
keys, err := h.Store.Keys(ProvisionsState)
if err != nil {
return nil
}
for _, k := range keys {
if strings.HasPrefix(k, node+".") {
continue
}
e, err := h.Store.Get(ProvisionsState, k)
if err != nil || e == nil {
continue
}
if p, err := provisions.Decode(e.Value); err == nil {
for _, d := range p.DependsOn {
if strings.HasPrefix(d, node+".") {
out = append(out, d)
}
}
}
}
return out
}
// judgeElsewhere writes each needed key of another node (a provider's shared credential), its state from its outcome;
// that node's own composition keeps it while a provision depends on it.
func (h *Holder) judgeElsewhere(keys map[string]provisions.Provision, out *Composed) {
for _, key := range sortedKeys(keys) {
next := keys[key]
var have provisions.Provision
e, err := h.Store.Get(ProvisionsState, key)
if err == nil && e != nil {
if have, err = provisions.Decode(e.Value); err == nil {
next.Reference = have.Reference
}
}
h.outcome(key, &next)
if e != nil && same(have, next) {
continue
}
h.put(key, next)
out.Changed = append(out.Changed, key+" "+next.State)
}
}
// wantedOn is the fingerprint each want on a node names, by module.
func (h *Holder) wantedOn(node string) (map[string]string, error) {
out := map[string]string{}
keys, err := h.Store.Keys(WantsRead)
if err != nil {
return nil, fmt.Errorf("the wants cannot be read: %w", err)
}
for _, k := range keys {
n, module, err := delivery.ParseWantKey(k)
if err != nil || n != node {
continue
}
e, err := h.Store.Get(WantsRead, k)
if err != nil || e == nil {
continue
}
if w, err := delivery.DecodeWant(e.Value); err == nil && !w.Removed && w.FP != "" {
out[module] = w.FP
}
}
return out, nil
}
// runningOn is the fingerprint each module's have on a node says it runs, by module: what a module held without a want
// is composed at.
func (h *Holder) runningOn(node string) map[string]string {
out := map[string]string{}
keys, err := h.Store.Keys(HaveRead)
if err != nil {
return out
}
for _, k := range keys {
n, module, isNode, err := delivery.ParseHaveKey(k)
if err != nil || isNode || n != node {
continue
}
e, err := h.Store.Get(HaveRead, k)
if err != nil || e == nil {
continue
}
if have, err := delivery.DecodeHave(e.Value); err == nil && have.FP != "" {
out[module] = have.FP
}
}
return out
}
// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there.
func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error {
var problems []error
for _, module := range sortedKeys(plans) {
key, err := provisions.PlanKey(node, module)
if err != nil {
problems = append(problems, err)
continue
}
raw, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: plans[module]})
if err != nil {
problems = append(problems, err)
continue
}
if e, err := h.Store.Get(PlansState, key); err == nil && e != nil && bytes.Equal(e.Value, raw) {
continue
}
if err := h.Store.Put(PlansState, key, json.RawMessage(raw)); err != nil {
problems = append(problems, fmt.Errorf("the plan %s: %w", key, err))
continue
}
out.Plans = append(out.Plans, key)
}
keys, err := h.Store.Keys(PlansState)
if err != nil {
return errors.Join(append(problems, err)...)
}
for _, k := range keys {
n, module, ok := strings.Cut(k, ".")
if !ok || n != node {
continue
}
if _, still := plans[module]; still {
continue
}
if err := h.Store.Delete(PlansState, k); err != nil {
problems = append(problems, err)
continue
}
out.Gone = append(out.Gone, k)
}
return errors.Join(problems...)
}
// judgeNode brings the node's provision keys to what is needed and what each provider said.
func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, out *Composed) {
held := h.held(node)
// A key of this node another node's provision depends on is needed (a provider's one shared credential).
for _, dep := range h.dependedOn(node) {
if have, ok := held[dep]; ok {
if _, n := needed[dep]; !n {
have.State, have.Why, have.Ask, have.Warrant = "", "", "", nil
needed[dep] = have
}
}
}
for _, key := range unionKeys(needed, held) {
want, isNeeded := needed[key]
have, isHeld := held[key]
var next provisions.Provision
switch {
case isNeeded:
next = want
if isHeld {
next.Reference = have.Reference
}
h.outcome(key, &next)
case have.State == provisions.Retiring:
next = have
if next.Ask == "" && next.Warrant == nil {
if err := h.ask(&next, key); err != nil {
out.Problems = append(out.Problems, err.Error())
} else {
out.Asked = append(out.Asked, key)
}
}
if gone, why := h.retired(key, have); gone {
if err := h.Store.Delete(ProvisionsState, key); err != nil {
out.Problems = append(out.Problems, err.Error())
} else {
out.Changed = append(out.Changed, key+" (retired: "+why+")")
}
continue
}
default:
next = have
next.State, next.Why = provisions.Retiring, fmt.Sprintf("no build wanted or placed on %s requires %s of %s on %s "+
"any more", node, have.Provision, have.Provider.Module, have.Provider.Node)
next.Warrant = nil
if err := h.ask(&next, key); err != nil {
out.Problems = append(out.Problems, err.Error())
} else {
out.Asked = append(out.Asked, key)
}
}
if isHeld && same(have, next) {
continue
}
next.At = h.Now().UTC()
raw, err := provisions.Encode(next)
if err != nil {
out.Problems = append(out.Problems, err.Error())
continue
}
if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil {
out.Problems = append(out.Problems, fmt.Sprintf("%s: %v", key, err))
continue
}
out.Changed = append(out.Changed, key+" "+next.State)
}
}
// held is the node's provision keys as kept.
func (h *Holder) held(node string) map[string]provisions.Provision {
out := map[string]provisions.Provision{}
keys, err := h.Store.Keys(ProvisionsState)
if err != nil {
return out
}
for _, k := range keys {
if !strings.HasPrefix(k, node+".") {
continue
}
e, err := h.Store.Get(ProvisionsState, k)
if err != nil || e == nil {
continue
}
if p, err := provisions.Decode(e.Value); err == nil {
out[k] = p
}
}
return out
}
// same says two provisions say the same, their time aside.
func same(a, b provisions.Provision) bool {
a.At, b.At, a.V, b.V = time.Time{}, time.Time{}, 0, 0
ra, _ := json.Marshal(a)
rb, _ := json.Marshal(b)
return bytes.Equal(ra, rb)
}
// entry is a provider's outcome for one consumer, as its entry says it: the provider helper's Entry, read for what
// the holder needs and nothing more.
type entry struct {
// State and Why are the provider's outcome (provider.Entry, ADR 0570 clause 3): provided, failed or waits.
State string `json:"state,omitempty"`
Why string `json:"why,omitempty"`
Refused string `json:"refused,omitempty"`
Waits string `json:"waits,omitempty"`
VaultName string `json:"vault-name,omitempty"`
Generation int `json:"generation,omitempty"`
}
// outcome sets a needed provision's state from its provider's entry: provided, failed or waits; asked while it has
// none. The holder never sets provided by itself (clause 3): a provider whose entries it does not read keeps the key
// asked, said so.
func (h *Holder) outcome(key string, p *provisions.Provision) {
p.Ask, p.Warrant = "", nil
keeps, err := h.keepsEntries(p.Provider.Module)
if err != nil {
p.State, p.Why = provisions.Asked, "the manifest of "+p.Provider.Module+" cannot be read: "+err.Error()
return
}
if !keeps {
h.haveOutcome(p)
return
}
if !slices.Contains(BoundReads, p.Provider.Module) {
p.State, p.Why = provisions.Asked, p.Provider.Module+" keeps its outcome in its entries, which mesh-provisioner's "+
"manifest does not read yet"
return
}
e, err := h.Store.Get(boundRead(p.Provider.Module), key)
if err != nil {
p.State, p.Why = provisions.Asked, "the provider's entries cannot be read: "+err.Error()
return
}
if e == nil {
p.State, p.Why = provisions.Asked, ""
return
}
var en entry
if err := json.Unmarshal(e.Value, &en); err != nil {
p.State, p.Why = provisions.Asked, "the provider's entry is not readable"
return
}
switch {
case en.State == provisions.Failed:
p.State, p.Why = provisions.Failed, orSaid(en.Why, en.Refused, "the provider said it failed")
case en.State == provisions.Waits:
p.State, p.Why = provisions.Waits, orSaid(en.Why, en.Waits, "the provider waits for a person")
case en.State == "" && en.Refused != "":
p.State, p.Why = provisions.Failed, en.Refused
case en.State == "" && en.Waits != "":
p.State, p.Why = provisions.Waits, en.Waits
case en.State != "" && en.State != provisions.Provided:
p.State, p.Why = provisions.Asked, "the provider's entry says "+en.State+", which no state of a provision is"
return
default:
p.State, p.Why = provisions.Provided, ""
if en.VaultName != "" || en.Generation > 0 {
p.Reference = &provisions.Reference{ID: en.VaultName, Generation: en.Generation}
}
}
}
// haveOutcome is the outcome of a provider that makes no credential per consumer: its own have on its node (the lead's
// reading of ADR 0570 clause 3, recorded as an insight). Running with health ok at the build its want names is
// provided; failed is failed with the have's words; anything else is asked, waiting for the provider to run.
func (h *Holder) haveOutcome(p *provisions.Provision) {
where := p.Provider.Module + " on " + p.Provider.Node
// It makes what the consumer needs only once all it depends on is provided (its credential first).
for _, dep := range p.DependsOn {
state := "not composed yet"
if e, err := h.Store.Get(ProvisionsState, dep); err == nil && e != nil {
if d, err := provisions.Decode(e.Value); err == nil {
state = d.State
}
}
if state != provisions.Provided {
p.State, p.Why = provisions.Asked, "waits for "+dep+" ("+state+") before "+where+" can provide it"
return
}
}
key := p.Provider.Node + "." + p.Provider.Module
e, err := h.Store.Get(HaveRead, key)
if err != nil {
p.State, p.Why = provisions.Asked, "the have of "+where+" cannot be read: "+err.Error()
return
}
if e == nil {
p.State, p.Why = provisions.Asked, "waits for "+where+" to run: it says nothing yet"
return
}
have, err := delivery.DecodeHave(e.Value)
if err != nil {
p.State, p.Why = provisions.Asked, "the have of "+where+" is not readable"
return
}
wantFP := ""
if we, err := h.Store.Get(WantsRead, "want."+key); err == nil && we != nil {
if w, err := delivery.DecodeWant(we.Value); err == nil && !w.Removed {
wantFP = w.FP
}
}
switch {
case have.Phase == delivery.PhaseFailed || have.Health == delivery.HealthFailed:
p.State, p.Why = provisions.Failed, orSaid(have.Detail, where+" failed")
case have.Phase == delivery.PhaseRunning && have.Health == delivery.HealthOK && (wantFP == "" || have.FP == wantFP):
p.State, p.Why = provisions.Provided, ""
default:
p.State, p.Why = provisions.Asked, fmt.Sprintf("waits for %s to run healthy at the build its want names (it is %s, "+
"health %s)", where, have.Phase, have.Health)
}
}
// keepsEntries says a provider's manifest at the trunk declares the state `bound`: its outcome per consumer is its
// entries there; any other provider's is its own have.
func (h *Holder) keepsEntries(module string) (bool, error) {
mk, err := delivery.ModulesKey(module)
if err != nil {
return false, err
}
e, err := h.Store.Get(ModulesRead, mk)
if err != nil || e == nil {
return false, fmt.Errorf("the indexer holds no modules.%s (%v)", module, err)
}
m, err := delivery.DecodeModule(e.Value)
if err != nil || m.Manifest == "" {
return false, fmt.Errorf("the indexer's modules.%s names no manifest", module)
}
key, err := delivery.ManifestsKey(m.Manifest)
if err != nil {
return false, err
}
me, err := h.Store.Get(ManifestsRead, key)
if err != nil || me == nil {
return false, fmt.Errorf("the indexer holds no manifest of %s (%v)", module, err)
}
man, err := delivery.DecodeManifest(me.Value)
if err != nil {
return false, err
}
var decl struct {
State []json.RawMessage `json:"state"`
}
if err := json.Unmarshal([]byte(man.JSON), &decl); err != nil {
return false, err
}
for _, raw := range decl.State {
var name string
if json.Unmarshal(raw, &name) != nil {
var full struct {
Name string `json:"name"`
}
_ = json.Unmarshal(raw, &full)
name = full.Name
}
if name == "bound" {
return true, nil
}
}
return false, nil
}
// retired says a retiring key the operator approved is retired: its provider's entry is gone.
func (h *Holder) retired(key string, p provisions.Provision) (bool, string) {
if p.Warrant == nil || p.Warrant.Option != OptionApprove {
return false, ""
}
if keeps, err := h.keepsEntries(p.Provider.Module); err != nil || !keeps {
return false, ""
}
e, err := h.Store.Get(boundRead(p.Provider.Module), key)
if err != nil || e != nil {
return false, ""
}
return true, "its provider removed its credential on the operator's warrant"
}
// ---- the retiring ask ----------------------------------------------------------------------------------------------
// The options a retiring ask offers. The chosen one is the warrant's option in the key, and a provider lets its entry go
// only on "yes" (mesh-sdk provider.go, the pass's retiring keys), so the approving option is "yes" (review of
// 2026-10-30: the holder wrote "approve", which no provider acted on).
const (
OptionApprove = "yes"
OptionDecline = "no"
)
// askLasts is how long a retiring ask stays open.
const askLasts = asks.ApproveLasts - 10*time.Minute
// retiringAsk is the ask of one retiring provision, composed from the key alone.
func retiringAsk(key string, p provisions.Provision, at time.Time, id string) asks.Ask {
what := fmt.Sprintf("%s's access to %s on %s, for %s on %s", p.Consumer.Module, p.Provision, p.Provider.Node,
p.Consumer.Module, p.Consumer.Node)
a := asks.Ask{ID: id, Headline: "Remove " + p.Consumer.Module + "'s " + p.Provision + " access?",
Explanation: fmt.Sprintf("Nothing wanted or placed on %s needs %s any more. On Approve, %s removes %s: what it owns "+
"is handed to the resource's owner and its credential goes; the resource and its data stay.", p.Consumer.Node,
p.Provision, p.Provider.Module, what),
Who: asks.Operator, Expires: at.Add(askLasts), OnExpiry: "nothing is removed; it is asked again when it changes",
About: "provision." + key, Details: "The provision " + key + " of mesh-provisioner's state provisions."}
for _, o := range []struct{ id, label, does string }{
{OptionApprove, "Remove", "the provider removes the credential; the resource and its data stay"},
{OptionDecline, "Keep", "nothing is removed, and it is not asked again while nothing changes"},
} {
binds, _ := asks.ActDigest(retireAct(key, id, o.id))
a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds})
}
return a
}
func retireAct(key, id, option string) asks.Act {
return asks.Act{"verb": "mesh-provisioner.retire", "provision": key, "ask": id, "option": option}
}
// ask raises the retiring ask of a key, under mesh-provisioner's own name; one already raised stands.
func (h *Holder) ask(p *provisions.Provision, key string) error {
if p.Ask != "" {
return nil
}
if h.Asks == nil {
return errors.New("the operator cannot be asked from here, so " + key + " is retiring and nothing is removed")
}
now := h.Now().UTC()
id := "retire-" + strings.ReplaceAll(key, ".", "-") + "-" + fmt.Sprint(now.Unix())
a := retiringAsk(key, *p, now, id)
if err := a.Check(now); err != nil {
return err
}
if err := h.Asks.Publish(asks.AskSubject(Module), a, id); err != nil {
return fmt.Errorf("the retiring ask of %s could not be published: %w", key, err)
}
p.Ask = id
return nil
}
// Decide reads the router's record of each retiring key's ask and writes the warrant into the key (clause 5): the
// provider acts on an Approve; a Decline is kept, and not asked again while nothing changes.
func (h *Holder) Decide() []string {
h.mu.Lock()
defer h.mu.Unlock()
if h.Asks == nil {
return nil
}
keys, err := h.Store.Keys(ProvisionsState)
if err != nil {
return []string{"the provisions cannot be read: " + err.Error()}
}
var said []string
for _, k := range keys {
e, err := h.Store.Get(ProvisionsState, k)
if err != nil || e == nil {
continue
}
p, err := provisions.Decode(e.Value)
if err != nil || p.State != provisions.Retiring || p.Ask == "" || p.Warrant != nil {
continue
}
raw, found, err := h.Asks.Record(p.Ask)
if err != nil || !found {
continue
}
var rec struct {
State string `json:"state"`
Ask *asks.Ask `json:"ask"`
Warrant *asks.Warrant `json:"warrant"`
}
if json.Unmarshal(raw, &rec) != nil || rec.State == "open" || rec.Warrant == nil || rec.Ask == nil {
continue
}
w := *rec.Warrant
if w.Asker != Module || w.Ask != p.Ask {
continue
}
option := ""
if w.Outcome == asks.OutcomeChosen {
o, err := w.For(Module, *rec.Ask)
if err == nil {
err = o.Performs(retireAct(k, p.Ask, o.ID))
}
if err != nil {
said = append(said, k+": the answer is refused: "+err.Error())
continue
}
option = o.ID
} else {
// Ended with no choice: asked again at the next change.
p.Ask = ""
h.put(k, p)
said = append(said, k+": the ask "+string(w.Outcome)+"; asked again when it changes")
continue
}
by := "the operator"
if w.By != nil {
by = fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
}
p.Warrant = &provisions.Warrant{Ask: p.Ask, Option: option, By: by, At: w.At.UTC()}
h.put(k, p)
said = append(said, k+": "+option+" by "+by)
}
return said
}
func (h *Holder) put(key string, p provisions.Provision) {
p.At = h.Now().UTC()
if raw, err := provisions.Encode(p); err == nil {
if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil {
h.Logf("[%s] %s is not written: %v", Module, key, err)
}
}
}
// ---- what a change touches --------------------------------------------------------------------------------------
// NodesOfPlacement is the node a placement key names.
func NodesOfPlacement(key string) []string {
if n, err := delivery.ParsePlacementKey(key); err == nil {
return []string{n}
}
return nil
}
// NodesOfWant is the node a want key names.
func NodesOfWant(key string) []string {
if n, _, err := delivery.ParseWantKey(key); err == nil {
return []string{n}
}
return nil
}
// NodesOfModule is every node a module is placed on, and every node whose plans name it as a provider: what a change
// of the module's manifest touches.
func (h *Holder) NodesOfModule(module string) []string {
nodes := map[string]bool{}
for _, n := range h.placedOn(module) {
nodes[n] = true
}
if keys, err := h.Store.Keys(ProvisionsState); err == nil {
for _, k := range keys {
if e, err := h.Store.Get(ProvisionsState, k); err == nil && e != nil {
if p, err := provisions.Decode(e.Value); err == nil && p.Provider.Module == module {
nodes[p.Consumer.Node] = true
}
}
}
}
return sortedKeys(nodes)
}
// placedOn is every node a module is placed on, sorted.
func (h *Holder) placedOn(module string) []string {
nodes := map[string]bool{}
if keys, err := h.Store.Keys(PlacementRead); err == nil {
for _, k := range keys {
n, err := delivery.ParsePlacementKey(k)
if err != nil {
continue
}
e, err := h.Store.Get(PlacementRead, k)
if err != nil || e == nil {
continue
}
if p, err := delivery.DecodePlacement(e.Value); err == nil {
for _, m := range p.Modules {
if m.Module == module {
nodes[n] = true
}
}
}
}
}
return sortedKeys(nodes)
}
// NodeOfEntry is the consumer's node a provider's entry key names.
func NodeOfEntry(key string) []string {
if n, _, _, _, err := provisions.ParseKey(key); err == nil {
return []string{n}
}
return nil
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
func unionKeys[A, B any](a map[string]A, b map[string]B) []string {
seen := map[string]bool{}
for k := range a {
seen[k] = true
}
for k := range b {
seen[k] = true
}
return sortedKeys(seen)
}
// Rejudge sets one key's state again from its provider's outcome: what a change of a provider's entry touches. A
// retiring key whose provider removed its entry on the operator's warrant goes.
func (h *Holder) Rejudge(key string) (string, error) {
h.mu.Lock()
defer h.mu.Unlock()
e, err := h.Store.Get(ProvisionsState, key)
if err != nil || e == nil {
return "", err
}
have, err := provisions.Decode(e.Value)
if err != nil {
return "", err
}
if have.State == provisions.Retiring {
if gone, why := h.retired(key, have); gone {
return key + " retired: " + why, h.Store.Delete(ProvisionsState, key)
}
return "", nil
}
next := have
h.outcome(key, &next)
if same(have, next) {
return "", nil
}
h.put(key, next)
return key + " " + next.State, nil
}
// RetireNeeds deletes every key of the retired state `needs`, which the clients no longer write (clause 1): run once at
// the start.
func (h *Holder) RetireNeeds() (int, error) {
keys, err := h.Store.Keys(Needs)
if err != nil {
return 0, err
}
n := 0
for _, k := range keys {
if err := h.Store.Delete(Needs, k); err != nil {
return n, err
}
n++
}
return n, nil
}
// orSaid is the first of the words that says something.
func orSaid(words ...string) string {
for _, w := range words {
if strings.TrimSpace(w) != "" {
return w
}
}
return ""
}
// FromBus is the holder's composer over the bus: frombus's plans of a node, at the builds keptAt names, read through
// this module's runtime with its own `reads` (frombus.StatesReads). A plan names no value, so no own secret and no
// credential is read (compose.Planning).
func FromBus() Composer {
r := fromruntime.Reader()
return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) {
return frombus.Plans(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt})
}
}
@@ -0,0 +1,229 @@
//go:build integration
package main
// The holder against a real bus (novox/hq ADR 0570 clauses 1-3, 5; issue 1954 H5): its states and every input it reads
// are a real nats-server's key-value buckets, its plans composed by mesh-sdk's composition from bus state over the
// in-memory mesh frombustest gives (restic on n1, requiring postgres-database from postgres there). A placement and a
// want compose the key; the providers' outcomes (the vault's entry for the credential, postgres's own have) turn it
// provided; restic leaving the node makes it retiring and raises the ask.
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/compose/frombus/frombustest"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
// kvStore is the Store over a real bus: this module's states by their local names, another's as `<module>.<name>`.
type kvStore struct {
ctx context.Context
js jetstream.JetStream
}
func bucketOf(state string) string {
if owner, name, ok := strings.Cut(state, "."); ok {
return owner + "_" + name
}
return Module + "_" + state
}
func (s kvStore) kv(state string) (jetstream.KeyValue, error) {
kv, err := s.js.KeyValue(s.ctx, bucketOf(state))
if errors.Is(err, jetstream.ErrBucketNotFound) {
return s.js.CreateKeyValue(s.ctx, jetstream.KeyValueConfig{Bucket: bucketOf(state)})
}
return kv, err
}
func (s kvStore) Get(state, key string) (*Entry, error) {
kv, err := s.kv(state)
if err != nil {
return nil, err
}
e, err := kv.Get(s.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return &Entry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil
}
func (s kvStore) Put(state, key string, value any) error {
kv, err := s.kv(state)
if err != nil {
return err
}
raw, err := json.Marshal(value)
if err != nil {
return err
}
_, err = kv.Put(s.ctx, key, raw)
return err
}
func (s kvStore) Delete(state, key string) error {
kv, err := s.kv(state)
if err != nil {
return err
}
return kv.Delete(s.ctx, key)
}
func (s kvStore) Keys(state string) ([]string, error) {
kv, err := s.kv(state)
if err != nil {
return nil, err
}
keys, err := kv.Keys(s.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return keys, err
}
type asked struct{ ids []string }
func (a *asked) Publish(_ string, body any, id string) error {
a.ids = append(a.ids, id)
_ = body.(asks.Ask)
return nil
}
func (a *asked) Record(string) ([]byte, bool, error) { return nil, false, nil }
// 1954/H5.
func TestTheHolderComposesProvidesAndAsksOnTheBus(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
s := kvStore{ctx: ctx, js: js}
mesh, err := frombustest.Provisioned()
if err != nil {
t.Fatal(err)
}
node := frombustest.Node
// The bus as the holder reads it: the indexer's modules and manifests, a placement naming where the vault runs, the
// want of restic on n1.
for module, m := range mesh.Module {
m.V = 1
raw, _ := json.Marshal(m)
if err := s.Put(ModulesRead, module, json.RawMessage(raw)); err != nil {
t.Fatal(err)
}
}
for digest, raw := range mesh.Manifests {
var named struct {
Module string `json:"module"`
}
_ = json.Unmarshal(raw, &named)
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: named.Module, JSON: string(raw)})
key, _ := delivery.ManifestsKey(digest)
if err := s.Put(ManifestsRead, key, json.RawMessage(value)); err != nil {
t.Fatal(err)
}
}
// The vault keeps its outcomes as entries: its manifest declares `bound`.
vaultManifest := `{"module":"mesh-vault","version":"1","state":["bound"]}`
vaultDigest := delivery.ManifestDigest([]byte(vaultManifest))
vaultValue, _ := delivery.EncodeManifest(delivery.Manifest{Module: provisions.VaultModule, JSON: vaultManifest})
vaultKey, _ := delivery.ManifestsKey(vaultDigest)
if err := s.Put(ManifestsRead, vaultKey, json.RawMessage(vaultValue)); err != nil {
t.Fatal(err)
}
if err := s.Put(ModulesRead, provisions.VaultModule, map[string]any{"v": 1, "none2": "the test's", "manifest": vaultDigest,
"repository": "novox/mesh-catalog", "commit": "c1"}); err != nil {
t.Fatal(err)
}
vault, _ := json.Marshal(map[string]any{"v": 1, "node": "n2", "decided": map[string]any{},
"modules": []map[string]any{{"module": provisions.VaultModule}}})
if err := s.Put(PlacementRead, "placement.n2", json.RawMessage(vault)); err != nil {
t.Fatal(err)
}
fp := frombustest.FPs()["restic"]
if err := s.Put(WantsRead, "want."+node+".restic", map[string]any{"v": 1, "fp": fp, "self": true}); err != nil {
t.Fatal(err)
}
a := &asked{}
h := &Holder{Store: s, Asks: a, Now: time.Now, Logf: t.Logf,
Compose: func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) {
return frombus.Plans(ctx, mesh, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt})
}}
state := func(key string) provisions.Provision {
e, _ := s.Get(ProvisionsState, key)
if e == nil {
return provisions.Provision{}
}
p, _ := provisions.Decode(e.Value)
return p
}
key := node + ".restic.postgres-database"
if _, err := h.ComposeNode(ctx, node); err != nil {
t.Fatal(err)
}
if p := state(key); p.State != provisions.Asked || p.Provider.Module != "postgres" {
t.Fatalf("the placement and the want composed %+v", p)
}
if e, _ := s.Get(PlansState, node+".restic"); e == nil {
t.Fatal("restic's plan is not on the bus")
}
// The providers' outcomes: the vault provides the credential where restic has one, postgres runs healthy.
for _, dep := range state(key).DependsOn {
if err := s.Put(boundRead(provisions.VaultModule), dep, map[string]any{"state": "provided"}); err != nil {
t.Fatal(err)
}
}
if err := s.Put(HaveRead, node+".postgres", map[string]any{"v": 1, "fp": frombustest.FPs()["postgres"], "phase": "running",
"health": "ok"}); err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ { // the credential judged first, then what depends on it
if _, err := h.ComposeNode(ctx, node); err != nil {
t.Fatal(err)
}
}
if p := state(key); p.State != provisions.Provided {
t.Fatalf("the outcomes did not turn it provided: %+v", p)
}
// restic leaves the node and its want goes: retiring, one ask.
p := mesh.Placement[node]
var kept []delivery.PlacedModule
for _, m := range p.Modules {
if m.Module != "restic" {
kept = append(kept, m)
}
}
p.Modules = kept
mesh.Placement[node] = p
if err := s.Delete(WantsRead, "want."+node+".restic"); err != nil {
t.Fatal(err)
}
if _, err := h.ComposeNode(ctx, node); err != nil {
t.Fatal(err)
}
if r := state(key); r.State != provisions.Retiring || r.Ask == "" || len(a.ids) == 0 {
t.Fatalf("removing the want did not ask: %+v, asked %v", r, a.ids)
}
}
@@ -0,0 +1,362 @@
package main
import (
"context"
"encoding/json"
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// router is the operator's channel as the holder asks on it.
type router struct {
asked map[string]asks.Ask
records map[string][]byte
}
func (r *router) Publish(_ string, body any, id string) error {
r.asked[id] = body.(asks.Ask)
return nil
}
func (r *router) Record(id string) ([]byte, bool, error) {
raw, ok := r.records[id]
return raw, ok, nil
}
// answer is the operator's answer to an ask.
func (r *router) answer(t *testing.T, id, option string) {
t.Helper()
a := r.asked[id]
o, ok := a.Option(option)
if !ok {
t.Fatalf("the ask %s offers no %s", id, option)
}
w := asks.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label,
Level: o.Level, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42"}, Channel: "telegram",
At: time.Date(2026, 10, 29, 23, 0, 0, 0, time.UTC), AskDigest: a.Digest()}
r.records[id], _ = json.Marshal(map[string]any{"state": "closed", "ask": a, "warrant": w})
}
// mesh is the holder over memory, its composer answering each node from tables the test sets: the plans at the wanted
// builds and at the newest.
type mesh struct {
h *Holder
s *memStore
r *router
atWant map[string]map[string]catalogue.Plan
atNewest map[string]map[string]catalogue.Plan
composed []string
}
func newMesh() *mesh {
m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}},
atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}}
now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {},
Now: func() time.Time { now = now.Add(time.Second); return now },
Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) {
m.composed = append(m.composed, node)
if keptAt != nil {
return m.atWant[node], nil
}
return m.atNewest[node], nil
}}
// What each provider's manifest declares: postgres, keycloak and the vault keep entries, the others report by their
// have.
for module, states := range map[string]string{"postgres": `["bound"]`, "keycloak": `["bound"]`, "mesh-vault": `["bound"]`,
"systemd-resolved": `[]`, "minio": `[]`} {
raw := `{"module":"` + module + `","version":"1","state":` + states + `}`
digest := delivery.ManifestDigest([]byte(raw))
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
_ = m.s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value))
_ = m.s.Put(ModulesRead, mustKey(delivery.ModulesKey(module)),
json.RawMessage(`{"v":1,"none2":"a test","manifest":"`+digest+`","repository":"novox/mesh-catalog","commit":"c1"}`))
}
return m
}
func mustKey(k string, err error) string {
if err != nil {
panic(err)
}
return k
}
// needs makes a module on a node require provisions in its wanted build (and the same in its newest unless set).
func planOf(provs ...catalogue.PlanProvision) catalogue.Plan {
return catalogue.Plan{Provisions: provs}
}
var (
pgDB = catalogue.PlanProvision{Provision: "postgres-database", Provider: "ace", ProviderModule: "postgres"}
oidc = catalogue.PlanProvision{Provision: "oidc-client", Provider: "novox", ProviderModule: "keycloak"}
split = catalogue.PlanProvision{Provision: "split-dns", Provider: "g14", ProviderModule: "systemd-resolved"}
)
func (m *mesh) state(t *testing.T, key string) provisions.Provision {
t.Helper()
e, _ := m.s.Get(ProvisionsState, key)
if e == nil {
return provisions.Provision{}
}
p, err := provisions.Decode(e.Value)
if err != nil {
t.Fatal(err)
}
return p
}
func (m *mesh) compose(t *testing.T, node string) Composed {
t.Helper()
c, err := m.h.ComposeNode(context.Background(), node)
if err != nil {
t.Fatal(err)
}
return c
}
// A node composes its plans and provisions from what is wanted and placed; a key's state follows its provider's entry
// alone; the vault's generation reaches the reference; a
// quiet pass writes nothing (ADR 0570 clauses 1-3, 6, 7; issue 1954 H1, H2).
func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) {
m := newMesh()
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, split)}
m.atNewest["g14"] = m.atWant["g14"]
key := "g14.gitea.postgres-database"
steps := []struct {
name string
entry string // the provider's entry written before the step, "-" to delete
state string
why string
}{
{"composed: asked", "", provisions.Asked, ""},
{"the provider waits for a person", `{"waits":"the database's owner to accept"}`, provisions.Waits, "owner"},
{"the provider refused", `{"refused":"no room on ace"}`, provisions.Failed, "no room"},
{"the provider says failed in its state", `{"state":"failed","why":"the disk is full"}`, provisions.Failed, "disk"},
{"the provider made it", `{"state":"provided","vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""},
}
for _, s := range steps {
if s.entry != "" {
m.s.write(t, "postgres.bound", key, s.entry)
}
m.compose(t, "g14")
p := m.state(t, key)
if p.State != s.state || !strings.Contains(p.Why, s.why) || p.Provider.Module != "postgres" || p.Provider.Node != "ace" {
t.Errorf("%s: %+v", s.name, p)
}
}
if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 {
t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference)
}
// A provider making no credential per consumer reports through its own have (clause 3): never provided by the
// holder alone.
split := "g14.gitea.split-dns"
for _, row := range []struct{ have, want, state, why string }{
{"", "", provisions.Asked, "says nothing yet"},
{`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"applying","health":"unknown"}`, "", provisions.Asked, "applying"},
{`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Asked, "build its want"},
{`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"failed","health":"failed","detail":"the unit did not start"}`, "", provisions.Failed, "did not start"},
{`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Provided, ""},
} {
if row.have != "" {
m.s.write(t, HaveRead, "g14.systemd-resolved", row.have)
}
if row.want != "" {
m.s.write(t, WantsRead, "want.g14.systemd-resolved", row.want)
}
m.compose(t, "g14")
if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) {
t.Errorf("the provider's have %s: %+v, want %s", row.have, p, row.state)
}
}
if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil {
t.Error("the plan is not written")
}
if c := m.compose(t, "g14"); len(c.Plans)+len(c.Changed)+len(c.Asked) != 0 {
t.Errorf("a quiet pass wrote %+v", c)
}
// A provision's credential is the vault's own provision, on the consumer's node for a login of its own, and the
// provision depends on it (clauses 3, 6).
c := newMesh()
c.s.write(t, PlacementRead, "placement.novox", `{"v":1,"node":"novox","decided":{},"modules":[{"module":"mesh-vault"}]}`)
pair := pgDB
pair.CredentialForm = "pair"
c.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pair)}
c.atNewest["g14"] = c.atWant["g14"]
c.compose(t, "g14")
cred := c.state(t, "g14.gitea.secret-postgres-database")
if cred.Provider.Module != "mesh-vault" || cred.Provider.Node != "novox" || cred.State != provisions.Asked {
t.Errorf("the credential's provision is %+v", cred)
}
if d := c.state(t, key).DependsOn; !slices.Equal(d, []string{"g14.gitea.secret-postgres-database"}) {
t.Errorf("the provision depends on %v", d)
}
// A pair provider applying the vault's value through its templates (no entries) is provided only once its credential
// key is provided and it runs healthy (the lead's refinement of clause 3).
bucket := catalogue.PlanProvision{Provision: "s3-bucket", Provider: "g14", ProviderModule: "minio", CredentialForm: "pair"}
c.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(bucket)}
c.atNewest["g14"] = c.atWant["g14"]
c.s.write(t, HaveRead, "g14.minio", `{"v":1,"fp":"src2:`+strings.Repeat("cc", 32)+`","phase":"running","health":"ok"}`)
s3, s3cred := "g14.gitea.s3-bucket", "g14.gitea.secret-s3-bucket"
for _, row := range []struct {
name, entry, cred, state string
}{
{"its credential still asked", "", provisions.Asked, provisions.Asked},
{"its credential provided by the vault", `{"state":"provided","vault-name":"pair/g14/s3-bucket/g14/gitea","generation":1}`,
provisions.Provided, provisions.Provided},
} {
if row.entry != "" {
c.s.write(t, "mesh-vault.bound", s3cred, row.entry)
}
c.compose(t, "g14")
c.compose(t, "g14") // the credential judged first, then what depends on it
if got, dep := c.state(t, s3), c.state(t, s3cred); got.State != row.state || dep.State != row.cred {
t.Errorf("%s: %s is %s (%s), its credential %s", row.name, s3, got.State, got.Why, dep.State)
}
}
// An entry's change judges its key alone, no composition.
before := len(m.composed)
m.s.write(t, "postgres.bound", key, `{"refused":"the login is locked"}`)
if said, err := m.h.Rejudge(key); err != nil || !strings.Contains(said, provisions.Failed) || len(m.composed) != before {
t.Errorf("rejudged %q, %v, composed %d more", said, err, len(m.composed)-before)
}
}
// A provision no build wanted or placed needs is retiring with one ask naming provider, node and what goes; one composed
// ahead of its want raises none; a change removing one requirement and adding another retires only the removed one; the
// warrant goes into the key and the key goes once the provider removed its entry; a decline is kept and not asked again;
// a provision needed again returns from retiring (ADR 0570 clause 5; issue 1954 H3).
func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) {
m := newMesh()
pg, kc := "g14.gitea.postgres-database", "g14.gitea.oidc-client"
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, oidc)} // the newest build adds oidc ahead of its want
m.s.write(t, "postgres.bound", pg, `{}`)
steps := []struct {
name string
change func()
asked []string
states map[string]string
removed bool
}{
{"a key composed ahead of its want is asked, not retiring", func() {}, nil,
map[string]string{pg: provisions.Provided, kc: provisions.Asked}, false},
{"one requirement removed and another added: only the removed retires", func() {
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
}, []string{pg}, map[string]string{pg: provisions.Retiring, kc: provisions.Asked}, false},
{"composed again: no second ask", func() {}, nil, map[string]string{pg: provisions.Retiring}, false},
{"needed again before the warrant: back to its outcome", func() {
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)}
}, nil, map[string]string{pg: provisions.Provided}, false},
{"gone again: asked again", func() { m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} },
[]string{pg}, map[string]string{pg: provisions.Retiring}, false},
}
for _, s := range steps {
s.change()
c := m.compose(t, "g14")
if !slices.Equal(c.Asked, s.asked) {
t.Errorf("%s: asked %v, want %v", s.name, c.Asked, s.asked)
}
for k, st := range s.states {
if p := m.state(t, k); p.State != st {
t.Errorf("%s: %s is %s, want %s (%s)", s.name, k, p.State, st, p.Why)
}
}
}
p := m.state(t, pg)
a := m.r.asked[p.Ask]
if !strings.Contains(a.Explanation, "postgres") || !strings.Contains(a.Explanation, "ace") ||
!strings.Contains(a.Explanation, "data stay") {
t.Errorf("the ask does not say what goes from which provider on which node: %+v", a)
}
// Nothing before the warrant: the provider's entry stands and the key stays.
if said := m.h.Decide(); len(said) != 0 || m.state(t, pg).Warrant != nil {
t.Errorf("decided before an answer: %v", said)
}
m.r.answer(t, p.Ask, OptionApprove)
m.h.Decide()
if w := m.state(t, pg).Warrant; w == nil || w.Option != OptionApprove || !strings.Contains(w.By, "42") {
t.Fatalf("the warrant is not in the key: %+v", w)
}
// The word the SDK's provider lets an entry go on (provider.go): any other, and no provider ever retires it.
if w := m.state(t, pg).Warrant; w.Option != "yes" {
t.Fatalf("the warrant says %q, and a provider lets go only on \"yes\"", w.Option)
}
if said, _ := m.h.Rejudge(pg); said != "" || m.state(t, pg).State != provisions.Retiring {
t.Errorf("retired while the provider's entry stands: %q", said)
}
_ = m.s.Delete("postgres.bound", pg)
if said, _ := m.h.Rejudge(pg); !strings.Contains(said, "retired") || m.state(t, pg).State != "" {
t.Errorf("not retired once the provider removed its entry: %q", said)
}
// A decline is kept, and nothing is asked again while nothing changes.
d := newMesh()
d.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
d.compose(t, "g14")
d.atWant["g14"] = map[string]catalogue.Plan{}
d.compose(t, "g14")
d.r.answer(t, d.state(t, pg).Ask, OptionDecline)
d.h.Decide()
if c := d.compose(t, "g14"); len(c.Asked) != 0 || d.state(t, pg).Warrant.Option != OptionDecline {
t.Errorf("a decline was asked again: %+v", c)
}
}
// Review of 2026-10-30: a module held on a node with no want (the bus on the control-node) is composed at the build
// its have says it runs, so the holder writes it a plan and its client keeps filling it.
func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) {
m := newMesh()
fp := "src2:" + strings.Repeat("dd", 32)
m.s.write(t, HaveRead, "novox.nats", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`)
m.s.write(t, HaveRead, "g14.dunst", `{"v":1,"fp":"src2:`+strings.Repeat("ee", 32)+`","phase":"running","health":"ok"}`)
var keptAt map[string]string
m.h.Compose = func(_ context.Context, node string, at map[string]string) (map[string]catalogue.Plan, error) {
if at != nil {
keptAt = at
return map[string]catalogue.Plan{"nats": {}}, nil
}
return nil, nil
}
c := m.compose(t, "novox")
if keptAt["nats"] != fp || len(keptAt) != 1 {
t.Fatalf("composed at %v, want nats at the build it runs and nothing of another node", keptAt)
}
if !slices.Contains(c.Plans, "novox.nats") {
t.Fatalf("no plan written for a module held without a want: %+v", c)
}
}
// The holder holds no ticker: it composes on change events alone (ADR 0570 clause 2; issue 1954 H1).
func TestTheHolderHoldsNoTimer(t *testing.T) {
files, _ := filepath.Glob("*.go")
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
file, err := parser.ParseFile(token.NewFileSet(), f, nil, 0)
if err != nil {
t.Fatal(err)
}
ast.Inspect(file, func(n ast.Node) bool {
if sel, ok := n.(*ast.SelectorExpr); ok {
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "time" &&
slices.Contains([]string{"NewTicker", "Tick", "AfterFunc", "NewTimer"}, sel.Sel.Name) {
t.Errorf("%s uses time.%s: the holder composes on events alone", f, sel.Sel.Name)
}
}
return true
})
}
}
@@ -0,0 +1,104 @@
package main
// The holder's tools (novox/hq ADR 0570 Consequences, ADR 0564 §12's rule): a read of every key of `provisions` and
// `plans`, one key whole, each state's fill, composing one node now, and reading the operator's answers now.
import (
"context"
"encoding/json"
"fmt"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// The holder's tools.
const (
ProvisionsTool = "provisioner_provisions"
PlansTool = "provisioner_plans"
ComposeTool = "provisioner_compose"
DecideTool = "provisioner_decide"
)
func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
func holderTools(h *Holder) []stdio.Tool {
arg := func(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: ProvisionsTool,
Description: "Every provision the seat keeps (novox/hq ADR 0570): its key <node>.<consumer>.<provision>[.<local>], " +
"provider, consumer and state (asked, provided, waits, failed, retiring, with why), narrowed by node and " +
"state, with how many keys and bytes the state holds; or one key whole.",
Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one key, whole"),
"node": str("a consumer's node"), "state": str("asked, provided, waits, failed or retiring")}},
Run: func(a map[string]any) (any, error) {
return h.read(ProvisionsState, arg(a, "key"), arg(a, "node"), arg(a, "state"))
}},
{Name: PlansTool,
Description: "Every module's provisioning plan the seat composed, <node>.<module>, narrowed by node, with how many " +
"keys and bytes the state holds; or one plan whole.",
Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, <node>.<module>"),
"node": str("a node")}},
Run: func(a map[string]any) (any, error) { return h.read(PlansState, arg(a, "key"), arg(a, "node"), "") }},
{Name: ComposeTool,
Description: "Compose one node's plans and provisions now from the bus, as a change event would, and say what " +
"was written, removed, changed and asked.",
Input: map[string]any{"type": "object", "properties": map[string]any{"node": str("the node")},
"required": []string{"node"}},
Run: func(a map[string]any) (any, error) {
if arg(a, "node") == "" {
return nil, fmt.Errorf("compose names the node")
}
return h.ComposeNode(context.Background(), arg(a, "node"))
}},
{Name: DecideTool,
Description: "Read the operator's answers to every retiring provision's ask now, writing each warrant into its " +
"key, as the router's word on an ask would.",
Run: func(map[string]any) (any, error) { return map[string]any{"said": h.Decide()}, nil }},
}
}
// read is a state's keys narrowed, or one whole, with the state's fill.
func (h *Holder) read(state, key, node, want string) (any, error) {
if key != "" {
e, err := h.Store.Get(state, key)
if err != nil {
return nil, err
}
if e == nil {
return nil, fmt.Errorf("%s holds no %s", state, key)
}
return json.RawMessage(e.Value), nil
}
keys, err := h.Store.Keys(state)
if err != nil {
return nil, err
}
rows := []map[string]any{}
bytes := 0
for _, k := range keys {
e, err := h.Store.Get(state, k)
if err != nil || e == nil {
continue
}
bytes += len(e.Value)
if node != "" && !strings.HasPrefix(k, node+".") {
continue
}
row := map[string]any{"key": k}
if state == ProvisionsState {
p, err := provisions.Decode(e.Value)
if err != nil {
row["unreadable"] = err.Error()
} else {
if want != "" && p.State != want {
continue
}
row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why
}
}
rows = append(rows, row)
}
return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil
}
@@ -1,23 +1,28 @@
// mesh-provisioner: the provisioner's server, holder of the mesh-provisioner seat once for the mesh (novox/hq ADR
// 0349 clause 3, design 57 P4). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR
// 0259 §8), keeping four buckets: mesh-provisioner_registry (which domain answers which kind of placeholder, its
// alone), and mesh-provisioner_needs, _applied and _collect, which each machine's provisioner client writes for its
// own machine (the controller grants the client those keys alone, issue 670). stdout is the MCP channel; what this
// module says, it says on stderr.
// 0349 clause 3, ADR 0570). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR 0259
// §8), keeping mesh-provisioner_provisions and _plans (every provision and every plan, composed from the bus on change
// events, its alone), _registry (which domain answers which kind of placeholder, its alone), and _applied and
// _collect, which each machine's provisioner client writes for its own machine; _needs is retired and emptied at its
// start. stdout is the MCP channel; what this module says, it says on stderr.
package main
import (
"context"
"fmt"
"os"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Seat is the role this module holds, once for the mesh, and Verb its one verb.
const (
Seat = "mesh-provisioner"
Verb = "provisioning"
Seat = "mesh-provisioner"
Module = "mesh-provisioner"
Verb = "provisioning"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
@@ -42,30 +47,149 @@ func (runtimeStore) Delete(state, key string) error { return stdio.State(state).
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
// runtimeAsks is the operator's channel through the runtime: asks published under this module's own name, and the
// router's record of one read under that name.
type runtimeAsks struct{}
func (runtimeAsks) Publish(subject string, body any, id string) error {
return stdio.SeatPublish(subject, body, id)
}
func (runtimeAsks) Record(id string) ([]byte, bool, error) {
e, err := stdio.SeatRecord("", id)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}
// work is the nodes a change touched, composed one at a time by one worker: no timer, a change's event is the wake.
type work struct {
mu sync.Mutex
pending map[string]bool
wake chan struct{}
}
func (w *work) add(nodes ...string) {
if len(nodes) == 0 {
return
}
w.mu.Lock()
for _, n := range nodes {
w.pending[n] = true
}
w.mu.Unlock()
select {
case w.wake <- struct{}{}:
default:
}
}
func (w *work) run(h *Holder) {
for range w.wake {
w.mu.Lock()
nodes := sortedKeys(w.pending)
w.pending = map[string]bool{}
w.mu.Unlock()
for _, n := range nodes {
c, err := h.ComposeNode(context.Background(), n)
switch {
case err != nil:
logf("[%s] %s is not composed: %v", Module, n, err)
case len(c.Plans)+len(c.Gone)+len(c.Changed)+len(c.Asked)+len(c.Problems) > 0:
logf("[%s] composed %s: %+v", Module, n, c)
}
}
}
}
// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs.
func watched(state string, handle func(stdio.StateChange) error) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
time.Sleep(wait)
if err := stdio.State(state).Watch("", handle); err != nil {
logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait)
continue
}
return
}
}
func main() {
go publishing(runtimeStore{})
if err := stdio.Serve("", Tools(runtimeStore{})); err != nil {
s := runtimeStore{}
h := &Holder{Store: s, Compose: FromBus(), Asks: runtimeAsks{}, Now: time.Now, Logf: logf}
w := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)}
go w.run(h)
go starting(h)
go watched(PlacementRead, func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil })
go watched(WantsRead, func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil })
go watched(ModulesRead, func(c stdio.StateChange) error {
if module, err := delivery.ParseModulesKey(c.Key); err == nil {
w.add(h.NodesOfModule(module)...)
}
return nil
})
for _, provider := range BoundReads {
go watched(boundRead(provider), func(c stdio.StateChange) error {
if said, err := h.Rejudge(c.Key); err != nil {
logf("[%s] %s is not judged again: %v", Module, c.Key, err)
} else if said != "" {
logf("[%s] %s", Module, said)
}
return nil
})
}
go watched(Registry, func(c stdio.StateChange) error {
if c.Op != "put" {
if _, _, err := Publish(s); err != nil {
logf("[%s] the registry is not written back: %v", Module, err)
}
}
return nil
})
go func() {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
time.Sleep(wait)
err := stdio.Subscribe(asks.Seat+".decided."+Module, func(stdio.Envelope) error {
for _, said := range h.Decide() {
logf("[%s] %s", Module, said)
}
return nil
})
if err == nil {
return
}
logf("[%s] does not hear the operator's answers yet (%v); asking again in %s", Module, err, wait)
}
}()
if err := stdio.Serve("", append(Tools(s), holderTools(h)...)); err != nil {
logf("%v", err)
os.Exit(1)
}
}
// publishing writes the registry once the runtime answers, and again every hour, so a registry emptied by hand or
// a bucket made again is written back without a restart. The runtime answers only once Serve runs, so the first
// pass is tried again until it lands.
func publishing(s Store) {
for wait := 2 * time.Second; ; {
time.Sleep(wait)
written, deleted, err := Publish(s)
switch {
case err != nil:
logf("[mesh-provisioner] the registry is not written yet (%v); trying again in %s", err, wait)
wait = min(wait*2, time.Minute)
continue
case len(written) > 0 || len(deleted) > 0:
logf("[mesh-provisioner] the registry: wrote %v, deleted %v", written, deleted)
// starting writes the registry, deletes the retired needs and reads the operator's answers once the runtime answers,
// each tried again until it lands.
func starting(h *Holder) {
<-stdio.Served()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
written, deleted, err := Publish(h.Store)
if err == nil {
if len(written)+len(deleted) > 0 {
logf("[%s] the registry: wrote %v, deleted %v", Module, written, deleted)
}
break
}
wait = time.Hour
logf("[%s] the registry is not written yet (%v); trying again in %s", Module, err, wait)
time.Sleep(wait)
}
if n, err := h.RetireNeeds(); err != nil {
logf("[%s] the retired needs are not all deleted: %v", Module, err)
} else if n > 0 {
logf("[%s] deleted %d key(s) of the retired needs", Module, n)
}
for _, said := range h.Decide() {
logf("[%s] %s", Module, said)
}
}
@@ -8,6 +8,8 @@ import (
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
)
// memStore is the bus as mesh-provisioner reaches it, in memory: the states it keeps.
@@ -106,9 +108,9 @@ func TestTheRegistryAnswersEveryKindATemplateMayName(t *testing.T) {
for kind, bucket := range map[string]string{"config": "mesh-config_resolved", "collected": "mesh-provisioner_" + Collect,
"shell": "mesh-provisioner_" + Collect, "environment": "mesh-provisioner_" + Collect,
"contribution": "mesh-provisioner_" + Collect,
"dir": "mesh-controller_assignments", "access": "mesh-controller_assignments",
"port": "mesh-controller_assignments", "seat": "mesh-controller_assignments",
"machine": "mesh-controller_assignments", "backup": "mesh-controller_assignments"} {
"dir": "mesh-provisioner_plans", "access": "mesh-provisioner_plans",
"port": "mesh-provisioner_plans", "seat": "mesh-provisioner_plans",
"machine": "mesh-provisioner_plans", "backup": "mesh-provisioner_plans"} {
i := slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == kind })
if TheRegistry[i].Bucket != bucket {
t.Errorf("%s is found in %s, want %s", kind, TheRegistry[i].Bucket, bucket)
@@ -288,6 +290,8 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) {
Serves []string `json:"serves"`
} `json:"claims"`
State []string `json:"state"`
Reads []string `json:"reads"`
Tools []string `json:"tools"`
OwnSecrets map[string]string `json:"own-secrets"`
}
if err := json.Unmarshal(raw, &m); err != nil {
@@ -300,7 +304,39 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) {
!slices.Equal(m.Claims[0].Serves, []string{Verb}) {
t.Errorf("the claim: %+v", m.Claims)
}
if want := []string{Registry, Needs, Applied, Collect}; !slices.Equal(m.State, want) {
// needs stays declared this release so its first start can empty it (ADR 0570 clause 1).
if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState}; !slices.Equal(m.State, want) {
t.Errorf("the state is %v, want %v", m.State, want)
}
// Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools
// its own and the composition it does on a change.
served := map[string]bool{}
for _, tool := range append(Tools(newStore()), holderTools(&Holder{})...) {
served[tool.Name] = true
}
for what, tool := range map[string]string{Registry: Seat + "." + Verb, Needs: Seat + "." + Verb,
Applied: Seat + "." + Verb, Collect: Seat + "." + Verb, ProvisionsState: ProvisionsTool, PlansState: PlansTool,
"a composition on a change": ComposeTool, "a decision on an answer": DecideTool} {
if !served[tool] {
t.Errorf("%s has no tool: %s is not served", what, tool)
}
}
for _, tool := range m.Tools {
if !served[tool] {
t.Errorf("the manifest lists %s, which is not served", tool)
}
}
for _, r := range append([]string{WantsRead}, frombus.StatesReads...) { // what fromruntime.Reader reads
if !slices.Contains(m.Reads, r) {
t.Errorf("the manifest does not read %s, which the composition reads", r)
}
}
for _, p := range BoundReads {
if !slices.Contains(m.Reads, boundRead(p)) {
t.Errorf("the manifest does not read %s's entries", p)
}
}
if slices.Contains(m.Reads, "mesh-controller.assignments") {
t.Error("the manifest reads the controller's assignments (ADR 0570 clause 2)")
}
}
@@ -0,0 +1,2 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
modules/mesh-provisioner/module.json
@@ -48,16 +48,15 @@ type Answer struct {
Decision string `json:"decision"`
}
// placement is the controller's: where a module runs, its directories, accesses, ports, seats and the machine's
// facts (ADR 0349 clause 6). On the bus, in the provisioner's plan of each module the controller handed to it, which
// its assignment carries (mesh-controller internal/inventory Provisioning, issue 671), or for a module only pulled in
// by another, its value of its own under the same key (issue 765): a client never reads the node's declaration,
// which the node-engine keeps for root alone.
// placement is the seat's: where a module runs, its directories, accesses, ports, seats and the machine's facts (ADR
// 0349 clause 6), in the provisioning plan this seat composes of each module on each node from the bus, its state
// `plans` (novox/hq ADR 0570 clauses 1, 2): a client never reads the node's declaration, which the node-engine keeps for
// root alone, nor the controller's assignments.
func placement(kind, what string) Answer {
return Answer{Kind: kind, Domain: "mesh-controller", From: FromBucket, Bucket: "mesh-controller_assignments",
return Answer{Kind: kind, Domain: Module, From: FromBucket, Bucket: Module + "_" + PlansState,
Key: "<node>.<module>",
Why: what + ": placement, the controller's, in the provisioner's plan its assignment carries (provisioning." +
"placement)", Decision: "novox/hq ADR 0349 clause 6"}
Why: what + ": placement, in the provisioning plan the seat mesh-provisioner composes of the module on the node " +
"(plans, plan.placement)", Decision: "novox/hq ADR 0570 clause 1"}
}
// oldCollection is a namespace read as a collection of its own name (novox/hq ADR 0349 clause 13, design 57 L1).