jschoubben
333ab9de22
A provider whose build does not offer the provision it was asked for must not be counted as having delivered it; the provisioner reads the build's manifest (the builds read serves only this offer check and makes no plan, ADR 0593 section 8) before it takes the provider's have as the outcome.
787 lines
37 KiB
Go
787 lines
37 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"go/ast"
|
|
"go/parser"
|
|
"go/token"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/provisions"
|
|
)
|
|
|
|
// router is the operator's channel as the holder asks on it: it holds an ask open, or, like the router past its open
|
|
// asks per asker, holds no record of it (drop).
|
|
type router struct {
|
|
asked map[string]asks.Ask
|
|
records map[string][]byte
|
|
drop bool
|
|
}
|
|
|
|
func (r *router) Publish(_ string, body any, id string) error {
|
|
r.asked[id] = body.(asks.Ask)
|
|
if !r.drop {
|
|
r.records[id], _ = json.Marshal(map[string]any{"state": "open", "ask": body})
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *router) Record(id string) ([]byte, bool, error) {
|
|
raw, ok := r.records[id]
|
|
return raw, ok, nil
|
|
}
|
|
|
|
// answer is the operator's answer to an ask.
|
|
func (r *router) answer(t *testing.T, id, option string) {
|
|
t.Helper()
|
|
a := r.asked[id]
|
|
o, ok := a.Option(option)
|
|
if !ok {
|
|
t.Fatalf("the ask %s offers no %s", id, option)
|
|
}
|
|
w := asks.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label,
|
|
Level: o.Level, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42"}, Channel: "telegram",
|
|
At: time.Date(2026, 10, 29, 23, 0, 0, 0, time.UTC), AskDigest: a.Digest()}
|
|
r.records[id], _ = json.Marshal(map[string]any{"state": "closed", "ask": a, "warrant": w})
|
|
}
|
|
|
|
// mesh is the holder over memory, its composer answering each node from tables the test sets: the plans at the wanted
|
|
// builds and at the newest.
|
|
type mesh struct {
|
|
h *Holder
|
|
s *memStore
|
|
r *router
|
|
atWant map[string]map[string]catalogue.Plan
|
|
atNewest map[string]map[string]catalogue.Plan
|
|
heldAt map[string]map[string]string // by node, the modules its composition at the wanted builds holds
|
|
// unpublished is, by node, the modules whose composition is not published yet.
|
|
unpublished map[string][]string
|
|
composed []string
|
|
}
|
|
|
|
func newMesh() *mesh {
|
|
m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}},
|
|
atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{},
|
|
heldAt: map[string]map[string]string{}, unpublished: map[string][]string{}}
|
|
now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
|
|
m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {},
|
|
Now: func() time.Time { now = now.Add(time.Second); return now },
|
|
Compose: func(_ context.Context, node string, keptAt map[string]string) Composition {
|
|
m.composed = append(m.composed, node)
|
|
return Composition{AtWants: HeldPlans{Plans: m.atWant[node], Held: m.heldAt[node]},
|
|
Newest: HeldPlans{Plans: m.atNewest[node]}, Unpublished: m.unpublished[node]}
|
|
}}
|
|
// What each provider's manifest declares: postgres, keycloak and the vault keep entries, the others report by their
|
|
// have.
|
|
for module, states := range map[string]string{"postgres": `["bound"]`, "keycloak": `["bound"]`, "mesh-vault": `["bound"]`,
|
|
"systemd-resolved": `[]`, "minio": `[]`} {
|
|
raw := `{"module":"` + module + `","version":"1","state":` + states + `}`
|
|
digest := delivery.ManifestDigest([]byte(raw))
|
|
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
|
|
_ = m.s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value))
|
|
_ = m.s.Put(ModulesRead, mustKey(delivery.ModulesKey(module)),
|
|
json.RawMessage(`{"v":1,"none2":"a test","manifest":"`+digest+`","repository":"novox/mesh-catalog","commit":"c1"}`))
|
|
}
|
|
return m
|
|
}
|
|
|
|
func mustKey(k string, err error) string {
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return k
|
|
}
|
|
|
|
// needs makes a module on a node require provisions in its wanted build (and the same in its newest unless set).
|
|
func planOf(provs ...catalogue.PlanProvision) catalogue.Plan {
|
|
return catalogue.Plan{Provisions: provs}
|
|
}
|
|
|
|
var (
|
|
pgDB = catalogue.PlanProvision{Provision: "postgres-database", Provider: "ace", ProviderModule: "postgres"}
|
|
oidc = catalogue.PlanProvision{Provision: "oidc-client", Provider: "novox", ProviderModule: "keycloak"}
|
|
split = catalogue.PlanProvision{Provision: "split-dns", Provider: "g14", ProviderModule: "systemd-resolved"}
|
|
)
|
|
|
|
func (m *mesh) state(t *testing.T, key string) provisions.Provision {
|
|
t.Helper()
|
|
e, _ := m.s.Get(ProvisionsState, key)
|
|
if e == nil {
|
|
return provisions.Provision{}
|
|
}
|
|
p, err := provisions.Decode(e.Value)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
|
|
func (m *mesh) compose(t *testing.T, node string) Composed {
|
|
t.Helper()
|
|
c, err := m.h.ComposeNode(context.Background(), node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return c
|
|
}
|
|
|
|
// A node composes its plans and provisions from what is wanted and placed; a key's state follows its provider's entry
|
|
// alone; the vault's generation reaches the reference; a
|
|
// quiet pass writes nothing (ADR 0570 clauses 1-3, 6, 7; issue 1954 H1, H2).
|
|
func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) {
|
|
m := newMesh()
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, split)}
|
|
m.atNewest["g14"] = m.atWant["g14"]
|
|
key := "g14.gitea.postgres-database"
|
|
steps := []struct {
|
|
name string
|
|
entry string // the provider's entry written before the step, "-" to delete
|
|
state string
|
|
why string
|
|
}{
|
|
{"composed: asked", "", provisions.Asked, ""},
|
|
{"the provider waits for a person", `{"waits":"the database's owner to accept"}`, provisions.Waits, "owner"},
|
|
{"the provider refused", `{"refused":"no room on ace"}`, provisions.Failed, "no room"},
|
|
{"the provider says failed in its state", `{"state":"failed","why":"the disk is full"}`, provisions.Failed, "disk"},
|
|
{"the provider made it", `{"state":"provided","vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""},
|
|
}
|
|
for _, s := range steps {
|
|
if s.entry != "" {
|
|
m.s.write(t, "postgres.bound", key, s.entry)
|
|
}
|
|
m.compose(t, "g14")
|
|
p := m.state(t, key)
|
|
if p.State != s.state || !strings.Contains(p.Why, s.why) || p.Provider.Module != "postgres" || p.Provider.Node != "ace" {
|
|
t.Errorf("%s: %+v", s.name, p)
|
|
}
|
|
}
|
|
if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 {
|
|
t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference)
|
|
}
|
|
// A provider making no credential per consumer reports through its own have (clause 3): never provided by the
|
|
// holder alone.
|
|
split := "g14.gitea.split-dns"
|
|
for _, row := range []struct{ have, want, state, why string }{
|
|
{"", "", provisions.Asked, "says nothing yet"},
|
|
{`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"applying","health":"unknown"}`, "", provisions.Asked, "applying"},
|
|
{`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Asked, "build its want"},
|
|
{`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"failed","health":"failed","detail":"the unit did not start"}`, "", provisions.Failed, "did not start"},
|
|
{`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Provided, ""},
|
|
} {
|
|
if row.have != "" {
|
|
m.s.write(t, HaveRead, "g14.systemd-resolved", row.have)
|
|
}
|
|
if row.want != "" {
|
|
m.s.write(t, WantsRead, "want.g14.systemd-resolved", row.want)
|
|
}
|
|
m.compose(t, "g14")
|
|
if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) {
|
|
t.Errorf("the provider's have %s: %+v, want %s", row.have, p, row.state)
|
|
}
|
|
}
|
|
// A have counts as the provider's outcome only from a build whose manifest offers the provision (ADR 0603 clause 5):
|
|
// refused when the build's manifest is known and does not offer it, provided when it does.
|
|
bfp := "src2:" + strings.Repeat("bb", 32)
|
|
buildKey := mustKey(delivery.BuilderBuildKey("systemd-resolved", bfp))
|
|
for _, row := range []struct{ name, provides, state, why string }{
|
|
{"a build whose manifest does not offer it", `["other"]`, provisions.Asked, "offers split-dns"},
|
|
{"a build whose manifest offers it", `[{"name":"split-dns"}]`, provisions.Provided, ""},
|
|
} {
|
|
raw := `{"module":"systemd-resolved","version":"1","provides":` + row.provides + `}`
|
|
digest := delivery.ManifestDigest([]byte(raw))
|
|
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: "systemd-resolved", JSON: raw})
|
|
_ = m.s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value))
|
|
build, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = m.s.Put(BuildsRead, buildKey, json.RawMessage(build))
|
|
m.compose(t, "g14")
|
|
if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) {
|
|
t.Errorf("%s: %+v, want %s", row.name, p, row.state)
|
|
}
|
|
}
|
|
if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil {
|
|
t.Error("the plan is not written")
|
|
}
|
|
if c := m.compose(t, "g14"); len(c.Plans)+len(c.Changed)+len(c.Asked) != 0 {
|
|
t.Errorf("a quiet pass wrote %+v", c)
|
|
}
|
|
// A provision's credential is the vault's own provision, on the consumer's node for a login of its own, and the
|
|
// provision depends on it (clauses 3, 6).
|
|
c := newMesh()
|
|
// The placement in its keys per module and node (research 072 row 37).
|
|
novox, err := delivery.PlacementValues(delivery.Placement{Node: "novox", Modules: []delivery.PlacedModule{{Module: "mesh-vault"}}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for k, v := range novox {
|
|
c.s.write(t, PlacementRead, k, string(v))
|
|
if n := NodesOfPlacement(k); !slices.Equal(n, []string{"novox"}) {
|
|
t.Errorf("NodesOfPlacement(%q) = %v", k, n)
|
|
}
|
|
}
|
|
pair := pgDB
|
|
pair.CredentialForm = "pair"
|
|
c.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pair)}
|
|
c.atNewest["g14"] = c.atWant["g14"]
|
|
c.compose(t, "g14")
|
|
cred := c.state(t, "g14.gitea.secret-postgres-database")
|
|
if cred.Provider.Module != "mesh-vault" || cred.Provider.Node != "novox" || cred.State != provisions.Asked {
|
|
t.Errorf("the credential's provision is %+v", cred)
|
|
}
|
|
if d := c.state(t, key).DependsOn; !slices.Equal(d, []string{"g14.gitea.secret-postgres-database"}) {
|
|
t.Errorf("the provision depends on %v", d)
|
|
}
|
|
// A pair provider applying the vault's value through its templates (no entries) is provided only once its credential
|
|
// key is provided and it runs healthy (the lead's refinement of clause 3).
|
|
bucket := catalogue.PlanProvision{Provision: "s3-bucket", Provider: "g14", ProviderModule: "minio", CredentialForm: "pair"}
|
|
c.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(bucket)}
|
|
c.atNewest["g14"] = c.atWant["g14"]
|
|
c.s.write(t, HaveRead, "g14.minio", `{"v":1,"fp":"src2:`+strings.Repeat("cc", 32)+`","phase":"running","health":"ok"}`)
|
|
s3, s3cred := "g14.gitea.s3-bucket", "g14.gitea.secret-s3-bucket"
|
|
for _, row := range []struct {
|
|
name, entry, cred, state string
|
|
}{
|
|
{"its credential still asked", "", provisions.Asked, provisions.Asked},
|
|
{"its credential provided by the vault", `{"state":"provided","vault-name":"pair/g14/s3-bucket/g14/gitea","generation":1}`,
|
|
provisions.Provided, provisions.Provided},
|
|
} {
|
|
if row.entry != "" {
|
|
c.s.write(t, "mesh-vault.bound", s3cred, row.entry)
|
|
}
|
|
c.compose(t, "g14")
|
|
c.compose(t, "g14") // the credential judged first, then what depends on it
|
|
if got, dep := c.state(t, s3), c.state(t, s3cred); got.State != row.state || dep.State != row.cred {
|
|
t.Errorf("%s: %s is %s (%s), its credential %s", row.name, s3, got.State, got.Why, dep.State)
|
|
}
|
|
}
|
|
// An entry's change judges its key alone, no composition.
|
|
before := len(m.composed)
|
|
m.s.write(t, "postgres.bound", key, `{"refused":"the login is locked"}`)
|
|
if said, err := m.h.Rejudge(key); err != nil || !strings.Contains(said, provisions.Failed) || len(m.composed) != before {
|
|
t.Errorf("rejudged %q, %v, composed %d more", said, err, len(m.composed)-before)
|
|
}
|
|
}
|
|
|
|
// A provision no build wanted or placed needs is retiring with one ask naming provider, node and what goes; one composed
|
|
// ahead of its want raises none; a change removing one requirement and adding another retires only the removed one; the
|
|
// warrant goes into the key and the key goes once the provider removed its entry; a decline is kept and not asked again;
|
|
// a provision needed again returns from retiring (ADR 0570 clause 5; issue 1954 H3).
|
|
func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) {
|
|
m := newMesh()
|
|
pg, kc := "g14.gitea.postgres-database", "g14.gitea.oidc-client"
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
|
|
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, oidc)} // the newest build adds oidc ahead of its want
|
|
m.s.write(t, "postgres.bound", pg, `{}`)
|
|
steps := []struct {
|
|
name string
|
|
change func()
|
|
asked []string
|
|
states map[string]string
|
|
removed bool
|
|
}{
|
|
{"a key composed ahead of its want is asked, not retiring", func() {}, nil,
|
|
map[string]string{pg: provisions.Provided, kc: provisions.Asked}, false},
|
|
{"one requirement removed and another added: only the removed retires", func() {
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
|
|
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
|
|
}, []string{pg}, map[string]string{pg: provisions.Retiring, kc: provisions.Asked}, false},
|
|
{"composed again: no second ask", func() {}, nil, map[string]string{pg: provisions.Retiring}, false},
|
|
{"needed again before the warrant: back to its outcome", func() {
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)}
|
|
}, nil, map[string]string{pg: provisions.Provided}, false},
|
|
{"gone again: asked again", func() { m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} },
|
|
[]string{pg}, map[string]string{pg: provisions.Retiring}, false},
|
|
}
|
|
for _, s := range steps {
|
|
s.change()
|
|
c := m.compose(t, "g14")
|
|
if !slices.Equal(c.Asked, s.asked) {
|
|
t.Errorf("%s: asked %v, want %v", s.name, c.Asked, s.asked)
|
|
}
|
|
for k, st := range s.states {
|
|
if p := m.state(t, k); p.State != st {
|
|
t.Errorf("%s: %s is %s, want %s (%s)", s.name, k, p.State, st, p.Why)
|
|
}
|
|
}
|
|
}
|
|
p := m.state(t, pg)
|
|
a := m.r.asked[p.Ask]
|
|
if !strings.Contains(a.Explanation, "postgres") || !strings.Contains(a.Explanation, "ace") ||
|
|
!strings.Contains(a.Explanation, "data stay") {
|
|
t.Errorf("the ask does not say what goes from which provider on which node: %+v", a)
|
|
}
|
|
// Nothing before the warrant: the provider's entry stands and the key stays.
|
|
if said := m.h.Decide(); len(said) != 0 || m.state(t, pg).Warrant != nil {
|
|
t.Errorf("decided before an answer: %v", said)
|
|
}
|
|
m.r.answer(t, p.Ask, OptionApprove)
|
|
m.h.Decide()
|
|
if w := m.state(t, pg).Warrant; w == nil || w.Option != OptionApprove || !strings.Contains(w.By, "42") {
|
|
t.Fatalf("the warrant is not in the key: %+v", w)
|
|
}
|
|
// The word the SDK's provider lets an entry go on (provider.go): any other, and no provider ever retires it.
|
|
if w := m.state(t, pg).Warrant; w.Option != "yes" {
|
|
t.Fatalf("the warrant says %q, and a provider lets go only on \"yes\"", w.Option)
|
|
}
|
|
if said, _ := m.h.Rejudge(pg); said != "" || m.state(t, pg).State != provisions.Retiring {
|
|
t.Errorf("retired while the provider's entry stands: %q", said)
|
|
}
|
|
_ = m.s.Delete("postgres.bound", pg)
|
|
if said, _ := m.h.Rejudge(pg); !strings.Contains(said, "retired") || m.state(t, pg).State != "" {
|
|
t.Errorf("not retired once the provider removed its entry: %q", said)
|
|
}
|
|
// A decline is kept, and nothing is asked again while nothing changes.
|
|
d := newMesh()
|
|
d.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
|
|
d.compose(t, "g14")
|
|
d.atWant["g14"] = map[string]catalogue.Plan{}
|
|
d.compose(t, "g14")
|
|
d.r.answer(t, d.state(t, pg).Ask, OptionDecline)
|
|
d.h.Decide()
|
|
if c := d.compose(t, "g14"); len(c.Asked) != 0 || d.state(t, pg).Warrant.Option != OptionDecline {
|
|
t.Errorf("a decline was asked again: %+v", c)
|
|
}
|
|
}
|
|
|
|
// Issue 774, one bad module must not drop its node: a module whose manifest or plan refuses is held, named in `refused`
|
|
// with why, its last plan and its provisions kept; every other module on the node is composed and written; once it
|
|
// composes again its refusal goes.
|
|
func TestOneBadModuleDoesNotDropItsNode(t *testing.T) {
|
|
m := newMesh()
|
|
why := "the built manifest of nats at src2:6258f3 cannot be used"
|
|
steps := []struct {
|
|
name string
|
|
plans map[string]catalogue.Plan
|
|
held map[string]string
|
|
written []string // plans the step writes
|
|
refused bool // g14.nats in refused
|
|
}{
|
|
{"a node new to the seat with one bad module: the others are composed",
|
|
map[string]catalogue.Plan{"gitea": planOf(pgDB)}, map[string]string{"nats": why}, []string{"g14.gitea"}, true},
|
|
{"the bad one composes: its plan is written and its refusal goes",
|
|
map[string]catalogue.Plan{"gitea": planOf(pgDB), "nats": planOf(oidc)}, nil, []string{"g14.nats"}, false},
|
|
{"it refuses again: its last plan and its provision are kept, the others composed",
|
|
map[string]catalogue.Plan{"gitea": planOf(pgDB, split)}, map[string]string{"nats": why}, []string{"g14.gitea"}, true},
|
|
}
|
|
for _, s := range steps {
|
|
m.atWant["g14"], m.atNewest["g14"], m.heldAt["g14"] = s.plans, s.plans, s.held
|
|
c := m.compose(t, "g14")
|
|
if !slices.Equal(c.Plans, s.written) || len(c.Gone) != 0 || len(c.Asked) != 0 {
|
|
t.Errorf("%s: %+v", s.name, c)
|
|
}
|
|
e, _ := m.s.Get(RefusedState, "g14.nats")
|
|
var r Refusal
|
|
if e != nil {
|
|
_ = json.Unmarshal(e.Value, &r)
|
|
}
|
|
if (e != nil) != s.refused || (s.refused && (r.Module != "nats" || r.Why != why || c.Held["nats"] != why)) {
|
|
t.Errorf("%s: refused %+v, held %v", s.name, r, c.Held)
|
|
}
|
|
if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil {
|
|
t.Errorf("%s: gitea has no plan", s.name)
|
|
}
|
|
}
|
|
if e, _ := m.s.Get(PlansState, "g14.nats"); e == nil {
|
|
t.Error("the held module's last plan was removed")
|
|
}
|
|
if p := m.state(t, "g14.nats.oidc-client"); p.State != provisions.Asked {
|
|
t.Errorf("the held module's provision is %s (%s), not kept", p.State, p.Why)
|
|
}
|
|
got, err := m.h.read(RefusedState, "", "g14", "")
|
|
if rows := got.(map[string]any)[RefusedState].([]map[string]any); err != nil || len(rows) != 1 || rows[0]["why"] != why {
|
|
t.Errorf("the refusal reads %v, %v", got, err)
|
|
}
|
|
}
|
|
|
|
// Review of 2026-10-30: a module held on a node with no want (the bus on the control-node) is composed at the build
|
|
// its have says it runs, so the holder writes it a plan and its client keeps filling it.
|
|
func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) {
|
|
m := newMesh()
|
|
fp := "src2:" + strings.Repeat("dd", 32)
|
|
m.s.write(t, HaveRead, "novox.nats", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`)
|
|
m.s.write(t, HaveRead, "g14.dunst", `{"v":1,"fp":"src2:`+strings.Repeat("ee", 32)+`","phase":"running","health":"ok"}`)
|
|
var keptAt map[string]string
|
|
m.h.Compose = func(_ context.Context, node string, at map[string]string) Composition {
|
|
keptAt = at
|
|
return Composition{AtWants: HeldPlans{Plans: map[string]catalogue.Plan{"nats": {}}}}
|
|
}
|
|
// ADR 0580 clause 2: the reads come from one watched copy, each state watched once and none read key by key from the
|
|
// bus, and a change the copy takes wakes the node it touches (a published composition its node).
|
|
bus := &readsOf{Store: m.s, read: map[string]int{}}
|
|
handles, watches := map[string]func(stdio.StateChange) error{}, map[string]int{}
|
|
ws := stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
|
|
watches[state]++
|
|
handles[state] = handle
|
|
keys, _ := m.s.Keys(state)
|
|
for _, k := range keys {
|
|
e, _ := m.s.Get(state, k)
|
|
_ = handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: e.Value, Current: true})
|
|
}
|
|
return nil
|
|
}, func(string, string) ([]byte, bool, error) { return nil, false, nil })
|
|
m.h.Store = Watching(bus, ws)
|
|
wk := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)}
|
|
for _, r := range []string{ComposedRead, HaveRead} {
|
|
ws.Follow(r, func(c stdio.StateChange) error { wk.add(NodesOfNodeKey(c.Key)...); return nil })
|
|
}
|
|
for i := range 2 {
|
|
c := m.compose(t, "novox")
|
|
if keptAt["nats"] != fp || len(keptAt) != 1 {
|
|
t.Fatalf("composed at %v, want nats at the build it runs and nothing of another node", keptAt)
|
|
}
|
|
if i == 0 && !slices.Contains(c.Plans, "novox.nats") {
|
|
t.Fatalf("no plan written for a module held without a want: %+v", c)
|
|
}
|
|
// Issue 1954: its own states come from the copy too, its own writes read back at once (nothing is written
|
|
// again), and the result says where the time went, its whole last.
|
|
if i == 1 && len(c.Plans)+len(c.Changed) > 0 {
|
|
t.Fatalf("composed again unchanged, it wrote %+v: its own writes were not read back", c)
|
|
}
|
|
if n := len(c.Took); n == 0 || c.Took[n-1].Phase != "whole" || c.Took[0].Phase != "wants" {
|
|
t.Fatalf("the composition says its phases as %v", c.Took)
|
|
}
|
|
}
|
|
for _, r := range append(slices.Clone(WatchedReads), OwnStates...) {
|
|
if bus.read[r] != 0 || watches[r] > 1 {
|
|
t.Errorf("%s was read %d times from the bus and watched %d times", r, bus.read[r], watches[r])
|
|
}
|
|
}
|
|
// The current haves queued their nodes when watched; a later change of a published composition or a have
|
|
// queues its node.
|
|
wk.pending = map[string]bool{}
|
|
_ = handles[ComposedRead](stdio.StateChange{Key: "g14.dunst." + strings.Repeat("ee", 32), Op: stdio.OpPut,
|
|
Value: []byte("{}")})
|
|
_ = handles[HaveRead](stdio.StateChange{Key: "ace.restic", Op: stdio.OpDelete})
|
|
if !wk.pending["g14"] || !wk.pending["ace"] || len(wk.pending) != 2 {
|
|
t.Errorf("a published composition's and a have's change queued %v, want g14 and ace", wk.pending)
|
|
}
|
|
}
|
|
|
|
// readsOf counts the reads that reach the bus, by state.
|
|
type readsOf struct {
|
|
Store
|
|
read map[string]int
|
|
}
|
|
|
|
func (r *readsOf) Get(state, key string) (*Entry, error) {
|
|
r.read[state]++
|
|
return r.Store.Get(state, key)
|
|
}
|
|
func (r *readsOf) Keys(state string) ([]string, error) { r.read[state]++; return r.Store.Keys(state) }
|
|
|
|
// The holder holds no ticker: it composes on change events alone (ADR 0570 clause 2; issue 1954 H1).
|
|
func TestTheHolderHoldsNoTimer(t *testing.T) {
|
|
files, _ := filepath.Glob("*.go")
|
|
for _, f := range files {
|
|
if strings.HasSuffix(f, "_test.go") {
|
|
continue
|
|
}
|
|
file, err := parser.ParseFile(token.NewFileSet(), f, nil, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ast.Inspect(file, func(n ast.Node) bool {
|
|
if sel, ok := n.(*ast.SelectorExpr); ok {
|
|
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "time" &&
|
|
slices.Contains([]string{"NewTicker", "Tick", "AfterFunc", "NewTimer"}, sel.Sel.Name) {
|
|
t.Errorf("%s uses time.%s: the holder composes on events alone", f, sel.Sel.Name)
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
}
|
|
|
|
// manifestOf puts a provider's manifest into the indexer's reads as the holder finds it.
|
|
func (m *mesh) manifestOf(module, raw string) {
|
|
digest := delivery.ManifestDigest([]byte(raw))
|
|
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
|
|
_ = m.s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value))
|
|
_ = m.s.Put(ModulesRead, mustKey(delivery.ModulesKey(module)),
|
|
json.RawMessage(`{"v":1,"none2":"a test","manifest":"`+digest+`","repository":"novox/mesh-catalog","commit":"c1"}`))
|
|
}
|
|
|
|
// A provider's outcome is read per offer: from its entries where the provider helper answers that offer, from its have
|
|
// where it does not, and the entries of route-proxy and node-tools are read (issue 774, the sweep of 2026-10-30 20:10:
|
|
// <node>.claude-code.gitea-read failed "gitea does not serve" on all four nodes while gitea's helper answers only
|
|
// forge-account, and novox.gitea.route and <node>.claude-code.mcp-endpoint stood asked, their providers' entries unread).
|
|
func TestAProvidersOutcomeIsReadPerOffer(t *testing.T) {
|
|
const (
|
|
gitea = `{"module":"gitea","version":"1","state":["bound","announced"],"provides":[{"name":"npm-package-registry","credential":{"form":"pair"}},{"name":"gitea-read","credential":{"issued-by":"software","tool":"gitea_agent_token"}},{"name":"forge-account","credential":{"form":"none"},"answered-by":"helper"}]}`
|
|
routes = `{"module":"route-proxy","version":"1","state":["bound"],"provides":[{"name":"route","scope":"mesh","identity":false,"answered-by":"helper"}]}`
|
|
nodeTools = `{"module":"node-tools","version":"1","state":["bound"],"provides":[{"name":"mcp-endpoint","scope":"node","answered-by":"helper"}]}`
|
|
running = `{"v":1,"fp":"src2:` + "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + `","phase":"running","health":"ok"}`
|
|
refused = `{"state":"failed","refused":"claude-code on ace is bound to gitea on novox for gitea-read, which gitea does not serve"}`
|
|
)
|
|
for _, row := range []struct {
|
|
name string
|
|
node string
|
|
plan map[string]catalogue.Plan
|
|
key string
|
|
bucket string // the provider's entries, written as entry
|
|
entry string
|
|
have string // the provider's have key, written running
|
|
state string
|
|
why string // a part of the provision's why, when it says one
|
|
}{
|
|
{"gitea-read: the helper's refusal is not its outcome; gitea runs, so provided", "ace",
|
|
map[string]catalogue.Plan{"claude-code": planOf(catalogue.PlanProvision{Provision: "gitea-read", Provider: "novox", ProviderModule: "gitea"})},
|
|
"ace.claude-code.gitea-read", "gitea.bound", refused, "novox.gitea", provisions.Provided, ""},
|
|
{"forge-account: still the helper's entry", "ace",
|
|
map[string]catalogue.Plan{"mesh-build-go": planOf(catalogue.PlanProvision{Provision: "forge-account", Provider: "novox", ProviderModule: "gitea"})},
|
|
"ace.mesh-build-go.forge-account", "gitea.bound", `{"state":"provided"}`, "", provisions.Provided, ""},
|
|
{"route: route-proxy's entry is read", "novox",
|
|
map[string]catalogue.Plan{"gitea": planOf(catalogue.PlanProvision{Provision: "route", Provider: "novox", ProviderModule: "route-proxy"})},
|
|
"novox.gitea.route", "route-proxy.bound", `{"state":"provided"}`, "", provisions.Provided, ""},
|
|
{"mcp-endpoint: node-tools' entry is read", "ace",
|
|
map[string]catalogue.Plan{"claude-code": planOf(catalogue.PlanProvision{Provision: "mcp-endpoint", Provider: "ace", ProviderModule: "node-tools"})},
|
|
"ace.claude-code.mcp-endpoint", "node-tools.bound", `{"state":"provided"}`, "", provisions.Provided, ""},
|
|
// Issue 774, 2026-10-31: route-proxy's helper read the controller's assignments, never the seat's plans, so 38
|
|
// routes the seat keys stood asked with no why while the assigner withheld their consumers' builds.
|
|
{"route: no entry from a provider that reads no plans of the seat is said so", "ace",
|
|
map[string]catalogue.Plan{"grafana": planOf(catalogue.PlanProvision{Provision: "route", Provider: "ace", ProviderModule: "route-proxy"})},
|
|
"ace.grafana.route", "", "", "", provisions.Asked, "reads no plans of the seat"},
|
|
} {
|
|
m := newMesh()
|
|
m.manifestOf("gitea", gitea)
|
|
m.manifestOf("route-proxy", routes)
|
|
m.manifestOf("node-tools", nodeTools)
|
|
m.atWant[row.node], m.atNewest[row.node] = row.plan, row.plan
|
|
if row.bucket != "" {
|
|
m.s.write(t, row.bucket, row.key, row.entry)
|
|
}
|
|
if row.have != "" {
|
|
m.s.write(t, HaveRead, row.have, running)
|
|
}
|
|
m.compose(t, row.node)
|
|
if p := m.state(t, row.key); p.State != row.state || !strings.Contains(p.Why, row.why) {
|
|
t.Errorf("%s: %s is %s (%s)", row.name, row.key, p.State, p.Why)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Issue 1126 review 1 (mcp-endpoint asked on ace and shanks, 2026-11-02 11:56): a requirement the consumer's newest build
|
|
// adds is keyed before its want, and the plan the seat keeps is its wanted build's, which names it not; the key carries
|
|
// the plan entry it was composed from, so the provider binds it, and its why says no plan names it yet. A key its
|
|
// wanted plan names carries none.
|
|
func TestAKeyAheadOfItsWantCarriesItsPlanEntryAndSaysWhy(t *testing.T) {
|
|
m := newMesh()
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
|
|
ahead := oidc
|
|
ahead.As = "gitea-g14"
|
|
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, ahead)}
|
|
m.compose(t, "g14")
|
|
kc := m.state(t, "g14.gitea.oidc-client")
|
|
if kc.Plan == nil || kc.Plan.Provision != "oidc-client" || kc.Plan.ProviderModule != "keycloak" || kc.Plan.As != "gitea-g14" {
|
|
t.Errorf("the key ahead of its want carries %+v", kc.Plan)
|
|
}
|
|
if kc.State != provisions.Asked || !strings.Contains(kc.Why, "no plan names it") {
|
|
t.Errorf("the key ahead of its want: %s %q", kc.State, kc.Why)
|
|
}
|
|
// A key a plan names carries no entry: a reader on an older mesh-sdk, which refused a field it did not know, reads it
|
|
// still, and skips only a key ahead of its want, which it could never bind (review 1).
|
|
if pg := m.state(t, "g14.gitea.postgres-database"); pg.Plan != nil || strings.Contains(pg.Why, "no plan names it") {
|
|
t.Errorf("the planned key: %+v", pg)
|
|
}
|
|
}
|
|
|
|
// Issue 940 (met before as 937 and 1332): a retirement is judged from a whole picture only. At 2026-11-02 22:55:58 a
|
|
// first pass after a new membership read a part, counted the modules missing from it as handing nothing, and asked the
|
|
// operator to remove the routes of 21 running web apps; 3 s later the next pass put every plan back.
|
|
func TestARetirementIsJudgedOnlyFromAWholePicture(t *testing.T) {
|
|
fp := "src2:" + strings.Repeat("ab", 32)
|
|
route := func(n int) catalogue.PlanProvision {
|
|
return catalogue.PlanProvision{Provision: "route", Provider: "novox", ProviderModule: "keycloak", Local: fmt.Sprint(n)}
|
|
}
|
|
apps := []string{"de-spiegel", "invoicing", "link2pay", "nextcloud", "photos", "website"}
|
|
for _, row := range []struct {
|
|
name string
|
|
consumer int // how many of apps hold a key, each one
|
|
gone int // how many of them the pass no longer composes
|
|
change func(m *mesh) // what else the pass sees
|
|
many bool // the operator's retire-many
|
|
retiring int // keys retiring after the pass
|
|
asked int // asks raised by the pass
|
|
problem string // a part of a problem the pass says
|
|
fault string // a part of the health's why, "" for healthy
|
|
after func(*testing.T, *mesh) // a further check
|
|
}{
|
|
{name: "a read not loaded once: nothing judged", consumer: 6, gone: 6,
|
|
change: func(m *mesh) { m.h.Loaded = func() string { return WantsRead } }},
|
|
{name: "another module's composition not published: a removed consumer still retires", consumer: 2, gone: 1,
|
|
change: func(m *mesh) {
|
|
m.unpublished["novox"] = []string{"gitea"}
|
|
m.heldAt["novox"] = map[string]string{"gitea": "the composition of gitea at " + fp + " is not published yet"}
|
|
}, retiring: 1, asked: 1, after: func(t *testing.T, m *mesh) {
|
|
// Held at one build: healthy over many passes within the bound (a build takes minutes), unhealthy past it,
|
|
// naming the module, its build and since when; another build starts the clock again.
|
|
now := m.h.Now()
|
|
m.h.Now = func() time.Time { return now }
|
|
for _, row := range []struct {
|
|
name string
|
|
after time.Duration
|
|
healthy bool
|
|
}{
|
|
{"many passes within the bound", 0, true},
|
|
{"still within the bound", unpublishedFor - time.Minute, true},
|
|
{"past the bound", unpublishedFor + time.Minute, false},
|
|
} {
|
|
now = now.Add(row.after)
|
|
for range 5 {
|
|
m.compose(t, "novox")
|
|
}
|
|
a := m.h.health()
|
|
if a.Healthy != row.healthy || (!row.healthy && (!strings.Contains(a.Why, "novox.gitea") ||
|
|
!strings.Contains(a.Why, fp) || !strings.Contains(a.Why, "held since"))) {
|
|
t.Errorf("%s: %+v", row.name, a)
|
|
}
|
|
}
|
|
m.heldAt["novox"] = map[string]string{"gitea": "the composition of gitea at src2:" + strings.Repeat("cd", 32) +
|
|
" is not published yet"}
|
|
m.compose(t, "novox")
|
|
if a := m.h.health(); !a.Healthy {
|
|
t.Errorf("another build did not start the clock again: %+v", a)
|
|
}
|
|
m.unpublished["novox"], m.heldAt["novox"] = nil, nil
|
|
m.compose(t, "novox")
|
|
if a := m.h.health(); !a.Healthy {
|
|
t.Errorf("the hold is said after it ended: %+v", a)
|
|
}
|
|
}},
|
|
{name: "a consumer that still has a want: not retired", consumer: 2, gone: 1,
|
|
change: func(m *mesh) {
|
|
m.s.write(t, WantsRead, "want.novox.de-spiegel", `{"v":1,"fp":"`+fp+`","self":true}`)
|
|
}},
|
|
{name: "a consumer that still has a have: not retired", consumer: 2, gone: 1,
|
|
change: func(m *mesh) {
|
|
m.s.write(t, HaveRead, "novox.de-spiegel", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`)
|
|
}},
|
|
{name: "one consumer of two leaving: retired and asked", consumer: 2, gone: 1, retiring: 1, asked: 1},
|
|
{name: "most consumers at once: a fault, not a question", consumer: 6, gone: 6, problem: "a read stopped listing them",
|
|
fault: "6 of the 6 consumers"},
|
|
{name: "more than 3 at once, though fewer than half", consumer: 6, gone: 4, fault: "4 of the 6"},
|
|
{name: "the operator's retire-many: retired, at most 3 asked at once", consumer: 6, gone: 6, many: true,
|
|
retiring: 6, asked: askOpenAtMost, after: func(t *testing.T, m *mesh) {
|
|
var waiting int
|
|
for _, a := range apps {
|
|
if strings.Contains(m.state(t, "novox."+a+".route.1").Why, askWaits) {
|
|
waiting++
|
|
}
|
|
}
|
|
if waiting != 6-askOpenAtMost {
|
|
t.Errorf("%d keys say their ask waits, want %d", waiting, 6-askOpenAtMost)
|
|
}
|
|
}},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
m := newMesh()
|
|
plans := map[string]catalogue.Plan{}
|
|
for i := range row.consumer {
|
|
plans[apps[i]] = planOf(route(1))
|
|
}
|
|
m.atWant["novox"], m.atNewest["novox"] = plans, plans
|
|
m.compose(t, "novox")
|
|
left := map[string]catalogue.Plan{}
|
|
for i := row.gone; i < row.consumer; i++ {
|
|
left[apps[i]] = plans[apps[i]]
|
|
}
|
|
m.atWant["novox"], m.atNewest["novox"] = left, left
|
|
if row.change != nil {
|
|
row.change(m)
|
|
}
|
|
c, err := m.h.ComposeNodeWith(context.Background(), "novox", row.many)
|
|
if m.h.Loaded != nil {
|
|
if err == nil || !strings.Contains(err.Error(), WantsRead) {
|
|
t.Errorf("a pass before every read loaded: %v", err)
|
|
}
|
|
} else if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
retiring := 0
|
|
for i := range row.consumer {
|
|
if m.state(t, "novox."+apps[i]+".route.1").State == provisions.Retiring {
|
|
retiring++
|
|
}
|
|
}
|
|
if retiring != row.retiring || len(c.Asked) != row.asked || len(m.r.asked) != row.asked {
|
|
t.Errorf("%d retiring, %d asked (%d published), want %d and %d: %+v", retiring, len(c.Asked),
|
|
len(m.r.asked), row.retiring, row.asked, c)
|
|
}
|
|
if row.problem != "" && !strings.Contains(strings.Join(c.Problems, " "), row.problem) {
|
|
t.Errorf("the problems %v do not say %q", c.Problems, row.problem)
|
|
}
|
|
if a := m.h.health(); (row.fault == "") != a.Healthy || !strings.Contains(a.Why, row.fault) ||
|
|
(row.fault != "" && !strings.Contains(a.Why, "retire-many")) {
|
|
t.Errorf("the health is %+v, want a fault saying %q", a, row.fault)
|
|
}
|
|
if row.fault != "" {
|
|
// The read whole again: the fault clears.
|
|
m.atWant["novox"], m.atNewest["novox"] = plans, plans
|
|
m.compose(t, "novox")
|
|
if a := m.h.health(); !a.Healthy {
|
|
t.Errorf("the fault stands once the picture is whole: %+v", a)
|
|
}
|
|
}
|
|
if row.after != nil {
|
|
row.after(t, m)
|
|
}
|
|
})
|
|
}
|
|
// A placed module no build of which is known is held, not counted as handing nothing; one unpublished says so.
|
|
var c today
|
|
readJSON(t, "testdata/today-composed.json", &c)
|
|
s := goldenMesh(t, c)
|
|
published(t, s, c)
|
|
s.write(t, PlacementRead, "placement."+c.Node, `{"v":1,"node":"`+c.Node+`","decided":{},"modules":[{"module":"store"},`+
|
|
`{"module":"postgres"},{"module":"mesh-provisioner-client"},{"module":"restic"},{"module":"de-spiegel"}]}`)
|
|
got := FromCompositions(s)(context.Background(), c.Node, map[string]string{"photos": fp})
|
|
if !strings.Contains(got.AtWants.Held["de-spiegel"], "no build") || !slices.Equal(got.Unpublished, []string{"photos"}) {
|
|
t.Errorf("held %v, unpublished %v", got.AtWants.Held, got.Unpublished)
|
|
}
|
|
}
|
|
|
|
// Issue 940: an ask the router does not hold is not counted asked; it is raised again once an open ask ends.
|
|
func TestAnAskTheRouterDoesNotHoldIsRaisedAgain(t *testing.T) {
|
|
m := newMesh()
|
|
key := "g14.gitea.postgres-database"
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB), "dunst": planOf(oidc)}
|
|
m.compose(t, "g14")
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"dunst": planOf(oidc)}
|
|
m.r.drop = true
|
|
m.compose(t, "g14")
|
|
first := m.state(t, key).Ask
|
|
if first == "" {
|
|
t.Fatal("not asked")
|
|
}
|
|
m.r.drop = false
|
|
steps := []struct {
|
|
name string
|
|
later time.Duration
|
|
held bool // the router holds an ask of the key after the step
|
|
}{
|
|
{"within a minute the router may not have taken it yet", 0, false},
|
|
{"a minute on, with no record: raised again", 2 * time.Minute, true},
|
|
}
|
|
for _, s := range steps {
|
|
now := m.h.Now()
|
|
m.h.Now = func() time.Time { now = now.Add(time.Second); return now.Add(s.later) }
|
|
m.h.Decide()
|
|
p := m.state(t, key)
|
|
if _, held := m.r.records[p.Ask]; held != s.held || (s.held && p.Ask == first) {
|
|
t.Errorf("%s: the key's ask is %q, the router holds it: %v", s.name, p.Ask, held)
|
|
}
|
|
}
|
|
}
|