jschoubben
bb18c4d14f
mesh/delivery merged: source.main moved to novox/mesh-catalog@0e78909a75b9
mesh/check being carried to bb18c4d14fc3 on 0e78909a75b9 | from d412991c6992
1058 lines
36 KiB
Go
1058 lines
36 KiB
Go
package main
|
|
|
|
// The seat's holder (novox/hq ADR 0570 clauses 1 to 5, 7; issue 1954 H1-H3): mesh-provisioner composes every placed
|
|
// module's provisioning plan and every provision from bus state, on change events and never on a timer, and keeps them
|
|
// in its states `plans` (`<node>.<module>`) and `provisions` (`<node>.<consumer>.<provision>[.<local>]`), which it alone
|
|
// writes.
|
|
//
|
|
// - **Composed per node.** A change of a node's placement or of one of its wants composes that node; a change of the
|
|
// indexer's module composes the nodes the module is placed on. Each node is composed twice: at the builds its wants
|
|
// name (its plans, what its client applies) and at the newest build of each placed module, so a build that adds a
|
|
// requirement has its key before its want (clause 2). A provision is needed while either needs it (clause 5).
|
|
// - **A key's state comes from its provider's outcome alone** (clause 3, 7): the provider's own entry for that
|
|
// consumer, `<provider>_bound` under the same key: present is provided (the vault's generation carried into the
|
|
// reference, clause 6), refused is failed with why, waits is waits with what. The holder never sets provided by
|
|
// itself: a key of a provider that writes no outcome stays asked.
|
|
// - **Retiring is asked** (clause 5): a key no longer needed is `retiring`, one approve ask under mesh-provisioner's
|
|
// name says what goes from which provider on which node; the warrant goes into the key and the provider acts on it;
|
|
// a decline is kept and not asked again while nothing changes; a key needed again returns to its outcome's state.
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
|
"git.novox.be/novox/mesh-sdk/go/compose/fromruntime"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/provisions"
|
|
)
|
|
|
|
// The states it keeps, beside Registry, Needs, Applied and Collect. RefusedState holds, by `<node>.<module>`, why a
|
|
// placed module has no plan composed (issue 774): written while it is held, deleted once it composes again.
|
|
const (
|
|
ProvisionsState = provisions.ProvisionsState
|
|
PlansState = provisions.PlansState
|
|
RefusedState = "refused"
|
|
)
|
|
|
|
// Refusal is one module held on a node, as RefusedState keeps it.
|
|
type Refusal struct {
|
|
Node string `json:"node"`
|
|
Module string `json:"module"`
|
|
Why string `json:"why"`
|
|
Since time.Time `json:"since"`
|
|
}
|
|
|
|
// The reads it composes from.
|
|
const (
|
|
PlacementRead = delivery.PlacementRead
|
|
WantsRead = delivery.AssignerModule + ".wants"
|
|
ModulesRead = delivery.IndexerModule + ".modules"
|
|
ManifestsRead = delivery.IndexerModule + ".manifests"
|
|
BuildsRead = delivery.BuilderModule + ".builds"
|
|
HaveRead = "node-engine.have"
|
|
ResolvedRead = "mesh-config.resolved"
|
|
)
|
|
|
|
// BoundReads are the providers' entries this module's manifest reads, `<module>.bound`, whose changes it hears. Which
|
|
// path judges a provider is its manifest's word (keepsEntries), not this list: a provider that declares `bound` and is
|
|
// not read here leaves its keys asked, saying its entries are not read. route-proxy (mesh-control) and node-tools
|
|
// (mesh-tools) answer route and mcp-endpoint by the provider helper from outside this catalogue (issue 774, sweep of
|
|
// 2026-10-30 20:10: their keys stood asked on every node).
|
|
var BoundReads = []string{"gitea", "keycloak", "postgres", "route-adapter", "route-proxy", "node-tools", "mesh-vault"}
|
|
|
|
// boundRead is a provider's entries as this module reads them.
|
|
func boundRead(module string) string { return module + ".bound" }
|
|
|
|
// Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names
|
|
// (module → fingerprint), the newest build of every other placed module. A module whose build or plan refuses is held
|
|
// (frombus.PlansHeld; issue 774, one bad module must not drop its node): in held with why, none of its plan in plans.
|
|
// err is a refusal that is no single module's.
|
|
type Composer func(ctx context.Context, node string, keptAt map[string]string) (plans map[string]catalogue.Plan,
|
|
held map[string]string, err error)
|
|
|
|
// Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's
|
|
// record of it.
|
|
type Asker interface {
|
|
Publish(subject string, body any, id string) error
|
|
Record(id string) ([]byte, bool, error)
|
|
}
|
|
|
|
// Holder is the seat's holder.
|
|
type Holder struct {
|
|
Store Store
|
|
Compose Composer
|
|
Asks Asker
|
|
Now func() time.Time
|
|
Logf func(format string, args ...any)
|
|
|
|
mu sync.Mutex
|
|
}
|
|
|
|
// Composed is what one node's composition did.
|
|
type Composed struct {
|
|
Node string `json:"node"`
|
|
Plans []string `json:"plans-written,omitempty"`
|
|
Gone []string `json:"plans-removed,omitempty"`
|
|
Changed []string `json:"provisions-changed,omitempty"`
|
|
Asked []string `json:"retiring-asked,omitempty"`
|
|
// Held is each module placed on the node whose plan is not composed, with why: its last plan and its provisions
|
|
// stay as they were, and every other module is composed (issue 774).
|
|
Held map[string]string `json:"modules-held,omitempty"`
|
|
Problems []string `json:"problems,omitempty"`
|
|
}
|
|
|
|
// ComposeNode composes one node: its plans and its provisions.
|
|
func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
out := Composed{Node: node}
|
|
wanted, err := h.wantedOn(node)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
// A module held on the node with no want (the bus on the control-node) is composed at the build its have says it
|
|
// runs, so it has a plan too and its client keeps filling it (review of 2026-10-30).
|
|
for module, fp := range h.runningOn(node) {
|
|
if _, ok := wanted[module]; !ok {
|
|
wanted[module] = fp
|
|
}
|
|
}
|
|
atWants, heldAtWants, err := h.Compose(ctx, node, wanted)
|
|
if err != nil {
|
|
return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err)
|
|
}
|
|
newest, heldNewest, err := h.Compose(ctx, node, nil)
|
|
if err != nil {
|
|
return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err)
|
|
}
|
|
// A module held at its wanted build has no plan composed: its last plan and its provisions are kept, said in
|
|
// `refused`. One held at its newest build alone keeps its plan at its want; what its newest build needs ahead waits.
|
|
if len(heldAtWants) > 0 {
|
|
out.Held = heldAtWants
|
|
}
|
|
for _, m := range sortedKeys(heldNewest) {
|
|
if _, ok := heldAtWants[m]; !ok {
|
|
out.Problems = append(out.Problems, fmt.Sprintf("%s at its newest build is not composed: %s", m, heldNewest[m]))
|
|
}
|
|
}
|
|
if err := h.writeRefused(node, heldAtWants); err != nil {
|
|
out.Problems = append(out.Problems, err.Error())
|
|
}
|
|
if err := h.writePlans(node, atWants, heldAtWants, &out); err != nil {
|
|
out.Problems = append(out.Problems, err.Error())
|
|
}
|
|
needed := map[string]provisions.Provision{}
|
|
elsewhere := map[string]provisions.Provision{}
|
|
vaultNode := h.vaultNode()
|
|
for _, plans := range []map[string]catalogue.Plan{atWants, newest} {
|
|
for module, plan := range plans {
|
|
for _, p := range plan.Provisions {
|
|
consumer := provisions.End{Module: module, Node: node}
|
|
provider := provisions.End{Module: p.ProviderModule, Node: p.Provider}
|
|
pv := provisions.Provision{Provision: p.Provision, Local: p.Local, Consumer: consumer, Provider: provider,
|
|
DependsOn: provisions.DependsOn(consumer, p, h.providerPlan(p, node, atWants))}
|
|
key, err := pv.Key()
|
|
if err != nil {
|
|
out.Problems = append(out.Problems, err.Error())
|
|
continue
|
|
}
|
|
if _, seen := needed[key]; !seen {
|
|
needed[key] = pv
|
|
}
|
|
// Its credential is a provision of its own, the vault's (clause 6): on the consumer's node for a value of the
|
|
// consumer's own, on the provider's for the provider's one shared value.
|
|
ck, ok := provisions.CredentialKey(p.CredentialForm, consumer, provider, p.Provision, p.Local)
|
|
if !ok {
|
|
continue
|
|
}
|
|
cn, cm, cp, cl, err := provisions.ParseKey(ck)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if vaultNode == "" {
|
|
out.Problems = append(out.Problems, "no node runs mesh-vault, so the credential "+ck+" has no provider")
|
|
continue
|
|
}
|
|
cred := provisions.Provision{Provision: cp, Local: cl, Consumer: provisions.End{Module: cm, Node: cn},
|
|
Provider: provisions.End{Module: provisions.VaultModule, Node: vaultNode}}
|
|
if cn == node {
|
|
if _, seen := needed[ck]; !seen {
|
|
needed[ck] = cred
|
|
}
|
|
} else {
|
|
elsewhere[ck] = cred
|
|
}
|
|
}
|
|
}
|
|
}
|
|
h.judgeElsewhere(elsewhere, &out)
|
|
h.judgeNode(node, needed, heldAtWants, &out)
|
|
return out, nil
|
|
}
|
|
|
|
// writeRefused keeps, for each module held on node, why, and deletes the refusal of one that composes again: a change
|
|
// of the state is the event a watcher hears.
|
|
func (h *Holder) writeRefused(node string, held map[string]string) error {
|
|
var problems []error
|
|
for _, module := range sortedKeys(held) {
|
|
key, err := provisions.PlanKey(node, module)
|
|
if err != nil {
|
|
problems = append(problems, err)
|
|
continue
|
|
}
|
|
r := Refusal{Node: node, Module: module, Why: held[module], Since: h.Now().UTC()}
|
|
if e, err := h.Store.Get(RefusedState, key); err == nil && e != nil {
|
|
var was Refusal
|
|
if json.Unmarshal(e.Value, &was) == nil && was.Why == r.Why {
|
|
continue
|
|
}
|
|
}
|
|
raw, err := json.Marshal(r)
|
|
if err != nil {
|
|
problems = append(problems, err)
|
|
continue
|
|
}
|
|
if err := h.Store.Put(RefusedState, key, json.RawMessage(raw)); err != nil {
|
|
problems = append(problems, fmt.Errorf("the refusal %s: %w", key, err))
|
|
continue
|
|
}
|
|
h.Logf("[%s] %s on %s has no plan composed: %s", Module, module, node, r.Why)
|
|
}
|
|
keys, err := h.Store.Keys(RefusedState)
|
|
if err != nil {
|
|
return errors.Join(append(problems, err)...)
|
|
}
|
|
for _, k := range keys {
|
|
n, module, ok := strings.Cut(k, ".")
|
|
if !ok || n != node {
|
|
continue
|
|
}
|
|
if _, still := held[module]; still {
|
|
continue
|
|
}
|
|
if err := h.Store.Delete(RefusedState, k); err != nil {
|
|
problems = append(problems, err)
|
|
continue
|
|
}
|
|
h.Logf("[%s] %s on %s composes again", Module, module, node)
|
|
}
|
|
return errors.Join(problems...)
|
|
}
|
|
|
|
// providerPlan is the plan of a provision's provider on its node: from this composition when it is on this node, else
|
|
// as kept in `plans`; nil when the provider is not handed.
|
|
func (h *Holder) providerPlan(p catalogue.PlanProvision, node string, here map[string]catalogue.Plan) *catalogue.Plan {
|
|
if p.Provider == node {
|
|
if plan, ok := here[p.ProviderModule]; ok {
|
|
return &plan
|
|
}
|
|
return nil
|
|
}
|
|
key, err := provisions.PlanKey(p.Provider, p.ProviderModule)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
e, err := h.Store.Get(PlansState, key)
|
|
if err != nil || e == nil {
|
|
return nil
|
|
}
|
|
plan, err := provisions.DecodePlan(e.Value)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return &plan.Plan
|
|
}
|
|
|
|
// vaultNode is the node mesh-vault is placed on: the provider of every credential.
|
|
func (h *Holder) vaultNode() string {
|
|
nodes := h.placedOn(provisions.VaultModule)
|
|
if len(nodes) == 0 {
|
|
return ""
|
|
}
|
|
return nodes[0]
|
|
}
|
|
|
|
// dependedOn is every key of a node that a provision of another node depends on.
|
|
func (h *Holder) dependedOn(node string) []string {
|
|
var out []string
|
|
keys, err := h.Store.Keys(ProvisionsState)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
for _, k := range keys {
|
|
if strings.HasPrefix(k, node+".") {
|
|
continue
|
|
}
|
|
e, err := h.Store.Get(ProvisionsState, k)
|
|
if err != nil || e == nil {
|
|
continue
|
|
}
|
|
if p, err := provisions.Decode(e.Value); err == nil {
|
|
for _, d := range p.DependsOn {
|
|
if strings.HasPrefix(d, node+".") {
|
|
out = append(out, d)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// judgeElsewhere writes each needed key of another node (a provider's shared credential), its state from its outcome;
|
|
// that node's own composition keeps it while a provision depends on it.
|
|
func (h *Holder) judgeElsewhere(keys map[string]provisions.Provision, out *Composed) {
|
|
for _, key := range sortedKeys(keys) {
|
|
next := keys[key]
|
|
var have provisions.Provision
|
|
e, err := h.Store.Get(ProvisionsState, key)
|
|
if err == nil && e != nil {
|
|
if have, err = provisions.Decode(e.Value); err == nil {
|
|
next.Reference = have.Reference
|
|
}
|
|
}
|
|
h.outcome(key, &next)
|
|
if e != nil && same(have, next) {
|
|
continue
|
|
}
|
|
h.put(key, next)
|
|
out.Changed = append(out.Changed, key+" "+next.State)
|
|
}
|
|
}
|
|
|
|
// wantedOn is the fingerprint each want on a node names, by module.
|
|
func (h *Holder) wantedOn(node string) (map[string]string, error) {
|
|
out := map[string]string{}
|
|
keys, err := h.Store.Keys(WantsRead)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the wants cannot be read: %w", err)
|
|
}
|
|
for _, k := range keys {
|
|
n, module, err := delivery.ParseWantKey(k)
|
|
if err != nil || n != node {
|
|
continue
|
|
}
|
|
e, err := h.Store.Get(WantsRead, k)
|
|
if err != nil || e == nil {
|
|
continue
|
|
}
|
|
if w, err := delivery.DecodeWant(e.Value); err == nil && !w.Removed && w.FP != "" {
|
|
out[module] = w.FP
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// runningOn is the fingerprint each module's have on a node says it runs, by module: what a module held without a want
|
|
// is composed at.
|
|
func (h *Holder) runningOn(node string) map[string]string {
|
|
out := map[string]string{}
|
|
keys, err := h.Store.Keys(HaveRead)
|
|
if err != nil {
|
|
return out
|
|
}
|
|
for _, k := range keys {
|
|
n, module, isNode, err := delivery.ParseHaveKey(k)
|
|
if err != nil || isNode || n != node {
|
|
continue
|
|
}
|
|
e, err := h.Store.Get(HaveRead, k)
|
|
if err != nil || e == nil {
|
|
continue
|
|
}
|
|
if have, err := delivery.DecodeHave(e.Value); err == nil && have.FP != "" {
|
|
out[module] = have.FP
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there; a held
|
|
// module's plan is kept as it was (issue 774).
|
|
func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, held map[string]string, out *Composed) error {
|
|
var problems []error
|
|
for _, module := range sortedKeys(plans) {
|
|
key, err := provisions.PlanKey(node, module)
|
|
if err != nil {
|
|
problems = append(problems, err)
|
|
continue
|
|
}
|
|
raw, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: plans[module]})
|
|
if err != nil {
|
|
problems = append(problems, err)
|
|
continue
|
|
}
|
|
if e, err := h.Store.Get(PlansState, key); err == nil && e != nil && bytes.Equal(e.Value, raw) {
|
|
continue
|
|
}
|
|
if err := h.Store.Put(PlansState, key, json.RawMessage(raw)); err != nil {
|
|
problems = append(problems, fmt.Errorf("the plan %s: %w", key, err))
|
|
continue
|
|
}
|
|
out.Plans = append(out.Plans, key)
|
|
}
|
|
keys, err := h.Store.Keys(PlansState)
|
|
if err != nil {
|
|
return errors.Join(append(problems, err)...)
|
|
}
|
|
for _, k := range keys {
|
|
n, module, ok := strings.Cut(k, ".")
|
|
if !ok || n != node {
|
|
continue
|
|
}
|
|
if _, still := plans[module]; still {
|
|
continue
|
|
}
|
|
if _, kept := held[module]; kept {
|
|
continue
|
|
}
|
|
if err := h.Store.Delete(PlansState, k); err != nil {
|
|
problems = append(problems, err)
|
|
continue
|
|
}
|
|
out.Gone = append(out.Gone, k)
|
|
}
|
|
return errors.Join(problems...)
|
|
}
|
|
|
|
// judgeNode brings the node's provision keys to what is needed and what each provider said. A key of a module held on
|
|
// the node (heldModules) that its composition no longer names is left as it is: not composed is not unneeded.
|
|
func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, heldModules map[string]string, out *Composed) {
|
|
held := h.held(node)
|
|
for key, have := range held {
|
|
if _, isNeeded := needed[key]; !isNeeded && have.Consumer.Node == node {
|
|
if _, notComposed := heldModules[have.Consumer.Module]; notComposed {
|
|
delete(held, key)
|
|
}
|
|
}
|
|
}
|
|
// A key of this node another node's provision depends on is needed (a provider's one shared credential).
|
|
for _, dep := range h.dependedOn(node) {
|
|
if have, ok := held[dep]; ok {
|
|
if _, n := needed[dep]; !n {
|
|
have.State, have.Why, have.Ask, have.Warrant = "", "", "", nil
|
|
needed[dep] = have
|
|
}
|
|
}
|
|
}
|
|
for _, key := range unionKeys(needed, held) {
|
|
want, isNeeded := needed[key]
|
|
have, isHeld := held[key]
|
|
var next provisions.Provision
|
|
switch {
|
|
case isNeeded:
|
|
next = want
|
|
if isHeld {
|
|
next.Reference = have.Reference
|
|
}
|
|
h.outcome(key, &next)
|
|
case have.State == provisions.Retiring:
|
|
next = have
|
|
if next.Ask == "" && next.Warrant == nil {
|
|
if err := h.ask(&next, key); err != nil {
|
|
out.Problems = append(out.Problems, err.Error())
|
|
} else {
|
|
out.Asked = append(out.Asked, key)
|
|
}
|
|
}
|
|
if gone, why := h.retired(key, have); gone {
|
|
if err := h.Store.Delete(ProvisionsState, key); err != nil {
|
|
out.Problems = append(out.Problems, err.Error())
|
|
} else {
|
|
out.Changed = append(out.Changed, key+" (retired: "+why+")")
|
|
}
|
|
continue
|
|
}
|
|
default:
|
|
next = have
|
|
next.State, next.Why = provisions.Retiring, fmt.Sprintf("no build wanted or placed on %s requires %s of %s on %s "+
|
|
"any more", node, have.Provision, have.Provider.Module, have.Provider.Node)
|
|
next.Warrant = nil
|
|
if err := h.ask(&next, key); err != nil {
|
|
out.Problems = append(out.Problems, err.Error())
|
|
} else {
|
|
out.Asked = append(out.Asked, key)
|
|
}
|
|
}
|
|
if isHeld && same(have, next) {
|
|
continue
|
|
}
|
|
next.At = h.Now().UTC()
|
|
raw, err := provisions.Encode(next)
|
|
if err != nil {
|
|
out.Problems = append(out.Problems, err.Error())
|
|
continue
|
|
}
|
|
if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil {
|
|
out.Problems = append(out.Problems, fmt.Sprintf("%s: %v", key, err))
|
|
continue
|
|
}
|
|
out.Changed = append(out.Changed, key+" "+next.State)
|
|
}
|
|
}
|
|
|
|
// held is the node's provision keys as kept.
|
|
func (h *Holder) held(node string) map[string]provisions.Provision {
|
|
out := map[string]provisions.Provision{}
|
|
keys, err := h.Store.Keys(ProvisionsState)
|
|
if err != nil {
|
|
return out
|
|
}
|
|
for _, k := range keys {
|
|
if !strings.HasPrefix(k, node+".") {
|
|
continue
|
|
}
|
|
e, err := h.Store.Get(ProvisionsState, k)
|
|
if err != nil || e == nil {
|
|
continue
|
|
}
|
|
if p, err := provisions.Decode(e.Value); err == nil {
|
|
out[k] = p
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// same says two provisions say the same, their time aside.
|
|
func same(a, b provisions.Provision) bool {
|
|
a.At, b.At, a.V, b.V = time.Time{}, time.Time{}, 0, 0
|
|
ra, _ := json.Marshal(a)
|
|
rb, _ := json.Marshal(b)
|
|
return bytes.Equal(ra, rb)
|
|
}
|
|
|
|
// entry is a provider's outcome for one consumer, as its entry says it: the provider helper's Entry, read for what
|
|
// the holder needs and nothing more.
|
|
type entry struct {
|
|
// State and Why are the provider's outcome (provider.Entry, ADR 0570 clause 3): provided, failed or waits.
|
|
State string `json:"state,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
Refused string `json:"refused,omitempty"`
|
|
Waits string `json:"waits,omitempty"`
|
|
VaultName string `json:"vault-name,omitempty"`
|
|
Generation int `json:"generation,omitempty"`
|
|
}
|
|
|
|
// outcome sets a needed provision's state from its provider's entry: provided, failed or waits; asked while it has
|
|
// none. The holder never sets provided by itself (clause 3): a provider whose entries it does not read keeps the key
|
|
// asked, said so.
|
|
func (h *Holder) outcome(key string, p *provisions.Provision) {
|
|
p.Ask, p.Warrant = "", nil
|
|
keeps, err := h.keepsEntries(p.Provider.Module, p.Provision)
|
|
if err != nil {
|
|
p.State, p.Why = provisions.Asked, "the manifest of "+p.Provider.Module+" cannot be read: "+err.Error()
|
|
return
|
|
}
|
|
if !keeps {
|
|
h.haveOutcome(p)
|
|
return
|
|
}
|
|
if !slices.Contains(BoundReads, p.Provider.Module) {
|
|
p.State, p.Why = provisions.Asked, p.Provider.Module+" keeps its outcome in its entries, which mesh-provisioner's "+
|
|
"manifest does not read yet"
|
|
return
|
|
}
|
|
e, err := h.Store.Get(boundRead(p.Provider.Module), key)
|
|
if err != nil {
|
|
p.State, p.Why = provisions.Asked, "the provider's entries cannot be read: "+err.Error()
|
|
return
|
|
}
|
|
if e == nil {
|
|
p.State, p.Why = provisions.Asked, h.unread(p.Provider.Module)
|
|
return
|
|
}
|
|
var en entry
|
|
if err := json.Unmarshal(e.Value, &en); err != nil {
|
|
p.State, p.Why = provisions.Asked, "the provider's entry is not readable"
|
|
return
|
|
}
|
|
switch {
|
|
case en.State == provisions.Failed:
|
|
p.State, p.Why = provisions.Failed, orSaid(en.Why, en.Refused, "the provider said it failed")
|
|
case en.State == provisions.Waits:
|
|
p.State, p.Why = provisions.Waits, orSaid(en.Why, en.Waits, "the provider waits for a person")
|
|
case en.State == "" && en.Refused != "":
|
|
p.State, p.Why = provisions.Failed, en.Refused
|
|
case en.State == "" && en.Waits != "":
|
|
p.State, p.Why = provisions.Waits, en.Waits
|
|
case en.State != "" && en.State != provisions.Provided:
|
|
p.State, p.Why = provisions.Asked, "the provider's entry says "+en.State+", which no state of a provision is"
|
|
return
|
|
default:
|
|
p.State, p.Why = provisions.Provided, ""
|
|
if en.VaultName != "" || en.Generation > 0 {
|
|
p.Reference = &provisions.Reference{ID: en.VaultName, Generation: en.Generation}
|
|
}
|
|
}
|
|
}
|
|
|
|
// unread is why a provider whose helper keeps its outcome in its entries has written none, when its manifest says it:
|
|
// it reads no plans of the seat (novox/hq ADR 0570 clause 2), so its helper never hears a bind and the key stays asked
|
|
// for good (issue 774, 2026-10-31: route-proxy's helper read only the controller's assignments, and 38 routes the seat
|
|
// keyed for modules not handed stood asked with no why while the assigner withheld their consumers' builds). "" when
|
|
// it reads them, or its manifest cannot say: then it has not answered yet.
|
|
func (h *Holder) unread(module string) string {
|
|
manifest, err := h.providerManifest(module)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
var decl struct {
|
|
Reads []string `json:"reads"`
|
|
}
|
|
if json.Unmarshal([]byte(manifest), &decl) != nil || slices.Contains(decl.Reads, provisions.PlansRead) {
|
|
return ""
|
|
}
|
|
return module + " reads no plans of the seat (its manifest's reads do not name " + provisions.PlansRead +
|
|
"), so its helper never answers this key"
|
|
}
|
|
|
|
// haveOutcome is the outcome of a provider that makes no credential per consumer: its own have on its node (the lead's
|
|
// reading of ADR 0570 clause 3, recorded as an insight). Running with health ok at the build its want names is
|
|
// provided; failed is failed with the have's words; anything else is asked, waiting for the provider to run.
|
|
func (h *Holder) haveOutcome(p *provisions.Provision) {
|
|
where := p.Provider.Module + " on " + p.Provider.Node
|
|
// It makes what the consumer needs only once all it depends on is provided (its credential first).
|
|
for _, dep := range p.DependsOn {
|
|
state := "not composed yet"
|
|
if e, err := h.Store.Get(ProvisionsState, dep); err == nil && e != nil {
|
|
if d, err := provisions.Decode(e.Value); err == nil {
|
|
state = d.State
|
|
}
|
|
}
|
|
if state != provisions.Provided {
|
|
p.State, p.Why = provisions.Asked, "waits for "+dep+" ("+state+") before "+where+" can provide it"
|
|
return
|
|
}
|
|
}
|
|
key := p.Provider.Node + "." + p.Provider.Module
|
|
e, err := h.Store.Get(HaveRead, key)
|
|
if err != nil {
|
|
p.State, p.Why = provisions.Asked, "the have of "+where+" cannot be read: "+err.Error()
|
|
return
|
|
}
|
|
if e == nil {
|
|
p.State, p.Why = provisions.Asked, "waits for "+where+" to run: it says nothing yet"
|
|
return
|
|
}
|
|
have, err := delivery.DecodeHave(e.Value)
|
|
if err != nil {
|
|
p.State, p.Why = provisions.Asked, "the have of "+where+" is not readable"
|
|
return
|
|
}
|
|
wantFP := ""
|
|
if we, err := h.Store.Get(WantsRead, "want."+key); err == nil && we != nil {
|
|
if w, err := delivery.DecodeWant(we.Value); err == nil && !w.Removed {
|
|
wantFP = w.FP
|
|
}
|
|
}
|
|
switch {
|
|
case have.Phase == delivery.PhaseFailed || have.Health == delivery.HealthFailed:
|
|
p.State, p.Why = provisions.Failed, orSaid(have.Detail, where+" failed")
|
|
case have.Phase == delivery.PhaseRunning && have.Health == delivery.HealthOK && (wantFP == "" || have.FP == wantFP):
|
|
p.State, p.Why = provisions.Provided, ""
|
|
default:
|
|
p.State, p.Why = provisions.Asked, fmt.Sprintf("waits for %s to run healthy at the build its want names (it is %s, "+
|
|
"health %s)", where, have.Phase, have.Health)
|
|
}
|
|
}
|
|
|
|
// keepsEntries says whether a provider's outcome for one provision is its entries (its manifest at the trunk declares the
|
|
// state `bound`) rather than its own have. When any offer of the provider says `answered-by: helper`, only those offers
|
|
// are answered by its entries: a module running the helper for one provision and making another some other way (gitea:
|
|
// the helper answers forge-account, the vault issues gitea-read, the package harness makes npm-package-registry) is
|
|
// judged per offer, so the helper's "does not serve" for an offer it does not run never becomes the provision's outcome
|
|
// (issue 774, sweep of 2026-10-30 20:10). A provider whose offers say nothing keeps every outcome in its entries.
|
|
func (h *Holder) keepsEntries(module, provision string) (bool, error) {
|
|
man, err := h.providerManifest(module)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return keepsEntriesFor(man, provision)
|
|
}
|
|
|
|
// keepsEntriesFor is keepsEntries over a provider's manifest JSON.
|
|
func keepsEntriesFor(manifest, provision string) (bool, error) {
|
|
var decl struct {
|
|
State []json.RawMessage `json:"state"`
|
|
Provides []json.RawMessage `json:"provides"`
|
|
}
|
|
if err := json.Unmarshal([]byte(manifest), &decl); err != nil {
|
|
return false, err
|
|
}
|
|
bound := false
|
|
for _, raw := range decl.State {
|
|
var name string
|
|
if json.Unmarshal(raw, &name) != nil {
|
|
var full struct {
|
|
Name string `json:"name"`
|
|
}
|
|
_ = json.Unmarshal(raw, &full)
|
|
name = full.Name
|
|
}
|
|
if name == "bound" {
|
|
bound = true
|
|
}
|
|
}
|
|
if !bound {
|
|
return false, nil
|
|
}
|
|
helped, says := false, false
|
|
for _, raw := range decl.Provides {
|
|
var full struct {
|
|
Name string `json:"name"`
|
|
AnsweredBy string `json:"answered-by"`
|
|
}
|
|
if json.Unmarshal(raw, &full) != nil || full.AnsweredBy != catalogue.AnsweredByHelper {
|
|
continue
|
|
}
|
|
says = true
|
|
if full.Name == provision {
|
|
helped = true
|
|
}
|
|
}
|
|
return helped || !says, nil
|
|
}
|
|
|
|
// providerManifest is a provider's manifest JSON at the trunk, as the indexer holds it.
|
|
func (h *Holder) providerManifest(module string) (string, error) {
|
|
mk, err := delivery.ModulesKey(module)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
e, err := h.Store.Get(ModulesRead, mk)
|
|
if err != nil || e == nil {
|
|
return "", fmt.Errorf("the indexer holds no modules.%s (%v)", module, err)
|
|
}
|
|
m, err := delivery.DecodeModule(e.Value)
|
|
if err != nil || m.Manifest == "" {
|
|
return "", fmt.Errorf("the indexer's modules.%s names no manifest", module)
|
|
}
|
|
key, err := delivery.ManifestsKey(m.Manifest)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
me, err := h.Store.Get(ManifestsRead, key)
|
|
if err != nil || me == nil {
|
|
return "", fmt.Errorf("the indexer holds no manifest of %s (%v)", module, err)
|
|
}
|
|
man, err := delivery.DecodeManifest(me.Value)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return man.JSON, nil
|
|
}
|
|
|
|
// retired says a retiring key the operator approved is retired: its provider's entry is gone.
|
|
func (h *Holder) retired(key string, p provisions.Provision) (bool, string) {
|
|
if p.Warrant == nil || p.Warrant.Option != OptionApprove {
|
|
return false, ""
|
|
}
|
|
if keeps, err := h.keepsEntries(p.Provider.Module, p.Provision); err != nil || !keeps {
|
|
return false, ""
|
|
}
|
|
e, err := h.Store.Get(boundRead(p.Provider.Module), key)
|
|
if err != nil || e != nil {
|
|
return false, ""
|
|
}
|
|
return true, "its provider removed its credential on the operator's warrant"
|
|
}
|
|
|
|
// ---- the retiring ask ----------------------------------------------------------------------------------------------
|
|
|
|
// The options a retiring ask offers. The chosen one is the warrant's option in the key, and a provider lets its entry go
|
|
// only on "yes" (mesh-sdk provider.go, the pass's retiring keys), so the approving option is "yes" (review of
|
|
// 2026-10-30: the holder wrote "approve", which no provider acted on).
|
|
const (
|
|
OptionApprove = "yes"
|
|
OptionDecline = "no"
|
|
)
|
|
|
|
// askLasts is how long a retiring ask stays open.
|
|
const askLasts = asks.ApproveLasts - 10*time.Minute
|
|
|
|
// retiringAsk is the ask of one retiring provision, composed from the key alone.
|
|
func retiringAsk(key string, p provisions.Provision, at time.Time, id string) asks.Ask {
|
|
what := fmt.Sprintf("%s's access to %s on %s, for %s on %s", p.Consumer.Module, p.Provision, p.Provider.Node,
|
|
p.Consumer.Module, p.Consumer.Node)
|
|
a := asks.Ask{ID: id, Headline: "Remove " + p.Consumer.Module + "'s " + p.Provision + " access?",
|
|
Explanation: fmt.Sprintf("Nothing wanted or placed on %s needs %s any more. On Approve, %s removes %s: what it owns "+
|
|
"is handed to the resource's owner and its credential goes; the resource and its data stay.", p.Consumer.Node,
|
|
p.Provision, p.Provider.Module, what),
|
|
Who: asks.Operator, Expires: at.Add(askLasts), OnExpiry: "nothing is removed; it is asked again when it changes",
|
|
About: "provision." + key, Details: "The provision " + key + " of mesh-provisioner's state provisions."}
|
|
for _, o := range []struct{ id, label, does string }{
|
|
{OptionApprove, "Remove", "the provider removes the credential; the resource and its data stay"},
|
|
{OptionDecline, "Keep", "nothing is removed, and it is not asked again while nothing changes"},
|
|
} {
|
|
binds, _ := asks.ActDigest(retireAct(key, id, o.id))
|
|
a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds})
|
|
}
|
|
return a
|
|
}
|
|
|
|
func retireAct(key, id, option string) asks.Act {
|
|
return asks.Act{"verb": "mesh-provisioner.retire", "provision": key, "ask": id, "option": option}
|
|
}
|
|
|
|
// ask raises the retiring ask of a key, under mesh-provisioner's own name; one already raised stands.
|
|
func (h *Holder) ask(p *provisions.Provision, key string) error {
|
|
if p.Ask != "" {
|
|
return nil
|
|
}
|
|
if h.Asks == nil {
|
|
return errors.New("the operator cannot be asked from here, so " + key + " is retiring and nothing is removed")
|
|
}
|
|
now := h.Now().UTC()
|
|
id := "retire-" + strings.ReplaceAll(key, ".", "-") + "-" + fmt.Sprint(now.Unix())
|
|
a := retiringAsk(key, *p, now, id)
|
|
if err := a.Check(now); err != nil {
|
|
return err
|
|
}
|
|
if err := h.Asks.Publish(asks.AskSubject(Module), a, id); err != nil {
|
|
return fmt.Errorf("the retiring ask of %s could not be published: %w", key, err)
|
|
}
|
|
p.Ask = id
|
|
return nil
|
|
}
|
|
|
|
// Decide reads the router's record of each retiring key's ask and writes the warrant into the key (clause 5): the
|
|
// provider acts on an Approve; a Decline is kept, and not asked again while nothing changes.
|
|
func (h *Holder) Decide() []string {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if h.Asks == nil {
|
|
return nil
|
|
}
|
|
keys, err := h.Store.Keys(ProvisionsState)
|
|
if err != nil {
|
|
return []string{"the provisions cannot be read: " + err.Error()}
|
|
}
|
|
var said []string
|
|
for _, k := range keys {
|
|
e, err := h.Store.Get(ProvisionsState, k)
|
|
if err != nil || e == nil {
|
|
continue
|
|
}
|
|
p, err := provisions.Decode(e.Value)
|
|
if err != nil || p.State != provisions.Retiring || p.Ask == "" || p.Warrant != nil {
|
|
continue
|
|
}
|
|
raw, found, err := h.Asks.Record(p.Ask)
|
|
if err != nil || !found {
|
|
continue
|
|
}
|
|
var rec struct {
|
|
State string `json:"state"`
|
|
Ask *asks.Ask `json:"ask"`
|
|
Warrant *asks.Warrant `json:"warrant"`
|
|
}
|
|
if json.Unmarshal(raw, &rec) != nil || rec.State == "open" || rec.Warrant == nil || rec.Ask == nil {
|
|
continue
|
|
}
|
|
w := *rec.Warrant
|
|
if w.Asker != Module || w.Ask != p.Ask {
|
|
continue
|
|
}
|
|
option := ""
|
|
if w.Outcome == asks.OutcomeChosen {
|
|
o, err := w.For(Module, *rec.Ask)
|
|
if err == nil {
|
|
err = o.Performs(retireAct(k, p.Ask, o.ID))
|
|
}
|
|
if err != nil {
|
|
said = append(said, k+": the answer is refused: "+err.Error())
|
|
continue
|
|
}
|
|
option = o.ID
|
|
} else {
|
|
// Ended with no choice: asked again at the next change.
|
|
p.Ask = ""
|
|
h.put(k, p)
|
|
said = append(said, k+": the ask "+string(w.Outcome)+"; asked again when it changes")
|
|
continue
|
|
}
|
|
by := "the operator"
|
|
if w.By != nil {
|
|
by = fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
|
|
}
|
|
p.Warrant = &provisions.Warrant{Ask: p.Ask, Option: option, By: by, At: w.At.UTC()}
|
|
h.put(k, p)
|
|
said = append(said, k+": "+option+" by "+by)
|
|
}
|
|
return said
|
|
}
|
|
|
|
func (h *Holder) put(key string, p provisions.Provision) {
|
|
p.At = h.Now().UTC()
|
|
if raw, err := provisions.Encode(p); err == nil {
|
|
if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil {
|
|
h.Logf("[%s] %s is not written: %v", Module, key, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- what a change touches --------------------------------------------------------------------------------------
|
|
|
|
// NodesOfPlacement is the node a placement key names.
|
|
func NodesOfPlacement(key string) []string {
|
|
if n, err := delivery.ParsePlacementKey(key); err == nil {
|
|
return []string{n}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// NodesOfWant is the node a want key names.
|
|
func NodesOfWant(key string) []string {
|
|
if n, _, err := delivery.ParseWantKey(key); err == nil {
|
|
return []string{n}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// NodesOfModule is every node a module is placed on, and every node whose plans name it as a provider: what a change
|
|
// of the module's manifest touches.
|
|
func (h *Holder) NodesOfModule(module string) []string {
|
|
nodes := map[string]bool{}
|
|
for _, n := range h.placedOn(module) {
|
|
nodes[n] = true
|
|
}
|
|
if keys, err := h.Store.Keys(ProvisionsState); err == nil {
|
|
for _, k := range keys {
|
|
if e, err := h.Store.Get(ProvisionsState, k); err == nil && e != nil {
|
|
if p, err := provisions.Decode(e.Value); err == nil && p.Provider.Module == module {
|
|
nodes[p.Consumer.Node] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return sortedKeys(nodes)
|
|
}
|
|
|
|
// placedOn is every node a module is placed on, sorted.
|
|
func (h *Holder) placedOn(module string) []string {
|
|
nodes := map[string]bool{}
|
|
if keys, err := h.Store.Keys(PlacementRead); err == nil {
|
|
for _, k := range keys {
|
|
n, err := delivery.ParsePlacementKey(k)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
e, err := h.Store.Get(PlacementRead, k)
|
|
if err != nil || e == nil {
|
|
continue
|
|
}
|
|
if p, err := delivery.DecodePlacement(e.Value); err == nil {
|
|
for _, m := range p.Modules {
|
|
if m.Module == module {
|
|
nodes[n] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return sortedKeys(nodes)
|
|
}
|
|
|
|
// NodeOfEntry is the consumer's node a provider's entry key names.
|
|
func NodeOfEntry(key string) []string {
|
|
if n, _, _, _, err := provisions.ParseKey(key); err == nil {
|
|
return []string{n}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func sortedKeys[V any](m map[string]V) []string {
|
|
out := make([]string, 0, len(m))
|
|
for k := range m {
|
|
out = append(out, k)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
func unionKeys[A, B any](a map[string]A, b map[string]B) []string {
|
|
seen := map[string]bool{}
|
|
for k := range a {
|
|
seen[k] = true
|
|
}
|
|
for k := range b {
|
|
seen[k] = true
|
|
}
|
|
return sortedKeys(seen)
|
|
}
|
|
|
|
// Rejudge sets one key's state again from its provider's outcome: what a change of a provider's entry touches. A
|
|
// retiring key whose provider removed its entry on the operator's warrant goes.
|
|
func (h *Holder) Rejudge(key string) (string, error) {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
e, err := h.Store.Get(ProvisionsState, key)
|
|
if err != nil || e == nil {
|
|
return "", err
|
|
}
|
|
have, err := provisions.Decode(e.Value)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if have.State == provisions.Retiring {
|
|
if gone, why := h.retired(key, have); gone {
|
|
return key + " retired: " + why, h.Store.Delete(ProvisionsState, key)
|
|
}
|
|
return "", nil
|
|
}
|
|
next := have
|
|
h.outcome(key, &next)
|
|
if same(have, next) {
|
|
return "", nil
|
|
}
|
|
h.put(key, next)
|
|
return key + " " + next.State, nil
|
|
}
|
|
|
|
// RetireNeeds deletes every key of the retired state `needs`, which the clients no longer write (clause 1): run once at
|
|
// the start.
|
|
func (h *Holder) RetireNeeds() (int, error) {
|
|
keys, err := h.Store.Keys(Needs)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
n := 0
|
|
for _, k := range keys {
|
|
if err := h.Store.Delete(Needs, k); err != nil {
|
|
return n, err
|
|
}
|
|
n++
|
|
}
|
|
return n, nil
|
|
}
|
|
|
|
// orSaid is the first of the words that says something.
|
|
func orSaid(words ...string) string {
|
|
for _, w := range words {
|
|
if strings.TrimSpace(w) != "" {
|
|
return w
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// FromBus is the holder's composer over the bus: frombus's plans of a node, at the builds keptAt names, read through
|
|
// this module's runtime with its own `reads` (frombus.StatesReads). A plan names no value, so no own secret and no
|
|
// credential is read (compose.Planning).
|
|
func FromBus() Composer {
|
|
r := fromruntime.Reader()
|
|
return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, map[string]string, error) {
|
|
return frombus.PlansHeld(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt})
|
|
}
|
|
}
|