jschoubben
695ac0356d
mesh-sdk holds neither asks nor asker, nor the vault's answer types, and mesh-delivery's sdk/views is mesh-sdk's go/views (a1-progress.md, the rewrite map for the join): thirteen modules switch to the operatorchannel contract and client, mesh-sdk's channel package and meshvault, with go.mod requiring each at the group pin (nats: meshbroker too). The gate holds the checks mesh-sdk dropped (an ask's id is its key's asking, at least one option), with a row that fails without them; claude-code refuses a router record whose times do not parse, as its JSON decode did, with a row. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
788 lines
37 KiB
Go
788 lines
37 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"go/ast"
|
|
"go/parser"
|
|
"go/token"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
|
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/provisions"
|
|
)
|
|
|
|
// router is the operator's channel as the holder asks on it: it holds an ask open, or, like the router past its open
|
|
// asks per asker, holds no record of it (drop).
|
|
type router struct {
|
|
asked map[string]oc.Ask
|
|
records map[string][]byte
|
|
drop bool
|
|
}
|
|
|
|
func (r *router) Publish(_ string, body any, id string) error {
|
|
r.asked[id] = body.(oc.Ask)
|
|
if !r.drop {
|
|
r.records[id], _ = json.Marshal(map[string]any{"state": "open", "ask": body})
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *router) Record(id string) ([]byte, bool, error) {
|
|
raw, ok := r.records[id]
|
|
return raw, ok, nil
|
|
}
|
|
|
|
// answer is the operator's answer to an ask.
|
|
func (r *router) answer(t *testing.T, id, option string) {
|
|
t.Helper()
|
|
a := r.asked[id]
|
|
o, ok := occ.OptionOf(a, option)
|
|
if !ok {
|
|
t.Fatalf("the ask %s offers no %s", id, option)
|
|
}
|
|
w := oc.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: occ.OutcomeChosen, Option: o.ID, Label: o.Label,
|
|
Level: o.Level, By: &oc.Person{Who: occ.Operator, Kind: "telegram", Identity: "42"}, Channel: "telegram",
|
|
At: occ.FormatTime(time.Date(2026, 10, 29, 23, 0, 0, 0, time.UTC)), AskDigest: occ.Digest(a)}
|
|
r.records[id], _ = json.Marshal(map[string]any{"state": "closed", "ask": a, "warrant": w})
|
|
}
|
|
|
|
// mesh is the holder over memory, its composer answering each node from tables the test sets: the plans at the wanted
|
|
// builds and at the newest.
|
|
type mesh struct {
|
|
h *Holder
|
|
s *memStore
|
|
r *router
|
|
atWant map[string]map[string]catalogue.Plan
|
|
atNewest map[string]map[string]catalogue.Plan
|
|
heldAt map[string]map[string]string // by node, the modules its composition at the wanted builds holds
|
|
// unpublished is, by node, the modules whose composition is not published yet.
|
|
unpublished map[string][]string
|
|
composed []string
|
|
}
|
|
|
|
func newMesh() *mesh {
|
|
m := &mesh{s: newStore(), r: &router{asked: map[string]oc.Ask{}, records: map[string][]byte{}},
|
|
atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{},
|
|
heldAt: map[string]map[string]string{}, unpublished: map[string][]string{}}
|
|
now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
|
|
m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {},
|
|
Now: func() time.Time { now = now.Add(time.Second); return now },
|
|
Compose: func(_ context.Context, node string, keptAt map[string]string) Composition {
|
|
m.composed = append(m.composed, node)
|
|
return Composition{AtWants: HeldPlans{Plans: m.atWant[node], Held: m.heldAt[node]},
|
|
Newest: HeldPlans{Plans: m.atNewest[node]}, Unpublished: m.unpublished[node]}
|
|
}}
|
|
// What each provider's manifest declares: postgres, keycloak and the vault keep entries, the others report by their
|
|
// have.
|
|
for module, states := range map[string]string{"postgres": `["bound"]`, "keycloak": `["bound"]`, "mesh-vault": `["bound"]`,
|
|
"systemd-resolved": `[]`, "minio": `[]`} {
|
|
raw := `{"module":"` + module + `","version":"1","state":` + states + `}`
|
|
digest := delivery.ManifestDigest([]byte(raw))
|
|
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
|
|
_ = m.s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value))
|
|
_ = m.s.Put(ModulesRead, mustKey(delivery.ModulesKey(module)),
|
|
json.RawMessage(`{"v":1,"none2":"a test","manifest":"`+digest+`","repository":"novox/mesh-catalog","commit":"c1"}`))
|
|
}
|
|
return m
|
|
}
|
|
|
|
func mustKey(k string, err error) string {
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return k
|
|
}
|
|
|
|
// needs makes a module on a node require provisions in its wanted build (and the same in its newest unless set).
|
|
func planOf(provs ...catalogue.PlanProvision) catalogue.Plan {
|
|
return catalogue.Plan{Provisions: provs}
|
|
}
|
|
|
|
var (
|
|
pgDB = catalogue.PlanProvision{Provision: "postgres-database", Provider: "ace", ProviderModule: "postgres"}
|
|
oidc = catalogue.PlanProvision{Provision: "oidc-client", Provider: "novox", ProviderModule: "keycloak"}
|
|
split = catalogue.PlanProvision{Provision: "split-dns", Provider: "g14", ProviderModule: "systemd-resolved"}
|
|
)
|
|
|
|
func (m *mesh) state(t *testing.T, key string) provisions.Provision {
|
|
t.Helper()
|
|
e, _ := m.s.Get(ProvisionsState, key)
|
|
if e == nil {
|
|
return provisions.Provision{}
|
|
}
|
|
p, err := provisions.Decode(e.Value)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
|
|
func (m *mesh) compose(t *testing.T, node string) Composed {
|
|
t.Helper()
|
|
c, err := m.h.ComposeNode(context.Background(), node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return c
|
|
}
|
|
|
|
// A node composes its plans and provisions from what is wanted and placed; a key's state follows its provider's entry
|
|
// alone; the vault's generation reaches the reference; a
|
|
// quiet pass writes nothing (ADR 0570 clauses 1-3, 6, 7; issue 1954 H1, H2).
|
|
func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) {
|
|
m := newMesh()
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, split)}
|
|
m.atNewest["g14"] = m.atWant["g14"]
|
|
key := "g14.gitea.postgres-database"
|
|
steps := []struct {
|
|
name string
|
|
entry string // the provider's entry written before the step, "-" to delete
|
|
state string
|
|
why string
|
|
}{
|
|
{"composed: asked", "", provisions.Asked, ""},
|
|
{"the provider waits for a person", `{"waits":"the database's owner to accept"}`, provisions.Waits, "owner"},
|
|
{"the provider refused", `{"refused":"no room on ace"}`, provisions.Failed, "no room"},
|
|
{"the provider says failed in its state", `{"state":"failed","why":"the disk is full"}`, provisions.Failed, "disk"},
|
|
{"the provider made it", `{"state":"provided","vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""},
|
|
}
|
|
for _, s := range steps {
|
|
if s.entry != "" {
|
|
m.s.write(t, "postgres.bound", key, s.entry)
|
|
}
|
|
m.compose(t, "g14")
|
|
p := m.state(t, key)
|
|
if p.State != s.state || !strings.Contains(p.Why, s.why) || p.Provider.Module != "postgres" || p.Provider.Node != "ace" {
|
|
t.Errorf("%s: %+v", s.name, p)
|
|
}
|
|
}
|
|
if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 {
|
|
t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference)
|
|
}
|
|
// A provider making no credential per consumer reports through its own have (clause 3): never provided by the
|
|
// holder alone.
|
|
split := "g14.gitea.split-dns"
|
|
for _, row := range []struct{ have, want, state, why string }{
|
|
{"", "", provisions.Asked, "says nothing yet"},
|
|
{`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"applying","health":"unknown"}`, "", provisions.Asked, "applying"},
|
|
{`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Asked, "build its want"},
|
|
{`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"failed","health":"failed","detail":"the unit did not start"}`, "", provisions.Failed, "did not start"},
|
|
{`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Provided, ""},
|
|
} {
|
|
if row.have != "" {
|
|
m.s.write(t, HaveRead, "g14.systemd-resolved", row.have)
|
|
}
|
|
if row.want != "" {
|
|
m.s.write(t, WantsRead, "want.g14.systemd-resolved", row.want)
|
|
}
|
|
m.compose(t, "g14")
|
|
if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) {
|
|
t.Errorf("the provider's have %s: %+v, want %s", row.have, p, row.state)
|
|
}
|
|
}
|
|
// A have counts as the provider's outcome only from a build whose manifest offers the provision (ADR 0603 clause 5):
|
|
// refused when the build's manifest is known and does not offer it, provided when it does.
|
|
bfp := "src2:" + strings.Repeat("bb", 32)
|
|
buildKey := mustKey(delivery.BuilderBuildKey("systemd-resolved", bfp))
|
|
for _, row := range []struct{ name, provides, state, why string }{
|
|
{"a build whose manifest does not offer it", `["other"]`, provisions.Asked, "offers split-dns"},
|
|
{"a build whose manifest offers it", `[{"name":"split-dns"}]`, provisions.Provided, ""},
|
|
} {
|
|
raw := `{"module":"systemd-resolved","version":"1","provides":` + row.provides + `}`
|
|
digest := delivery.ManifestDigest([]byte(raw))
|
|
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: "systemd-resolved", JSON: raw})
|
|
_ = m.s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value))
|
|
build, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = m.s.Put(BuildsRead, buildKey, json.RawMessage(build))
|
|
m.compose(t, "g14")
|
|
if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) {
|
|
t.Errorf("%s: %+v, want %s", row.name, p, row.state)
|
|
}
|
|
}
|
|
if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil {
|
|
t.Error("the plan is not written")
|
|
}
|
|
if c := m.compose(t, "g14"); len(c.Plans)+len(c.Changed)+len(c.Asked) != 0 {
|
|
t.Errorf("a quiet pass wrote %+v", c)
|
|
}
|
|
// A provision's credential is the vault's own provision, on the consumer's node for a login of its own, and the
|
|
// provision depends on it (clauses 3, 6).
|
|
c := newMesh()
|
|
// The placement in its keys per module and node (research 072 row 37).
|
|
novox, err := delivery.PlacementValues(delivery.Placement{Node: "novox", Modules: []delivery.PlacedModule{{Module: "mesh-vault"}}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for k, v := range novox {
|
|
c.s.write(t, PlacementRead, k, string(v))
|
|
if n := NodesOfPlacement(k); !slices.Equal(n, []string{"novox"}) {
|
|
t.Errorf("NodesOfPlacement(%q) = %v", k, n)
|
|
}
|
|
}
|
|
pair := pgDB
|
|
pair.CredentialForm = "pair"
|
|
c.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pair)}
|
|
c.atNewest["g14"] = c.atWant["g14"]
|
|
c.compose(t, "g14")
|
|
cred := c.state(t, "g14.gitea.secret-postgres-database")
|
|
if cred.Provider.Module != "mesh-vault" || cred.Provider.Node != "novox" || cred.State != provisions.Asked {
|
|
t.Errorf("the credential's provision is %+v", cred)
|
|
}
|
|
if d := c.state(t, key).DependsOn; !slices.Equal(d, []string{"g14.gitea.secret-postgres-database"}) {
|
|
t.Errorf("the provision depends on %v", d)
|
|
}
|
|
// A pair provider applying the vault's value through its templates (no entries) is provided only once its credential
|
|
// key is provided and it runs healthy (the lead's refinement of clause 3).
|
|
bucket := catalogue.PlanProvision{Provision: "s3-bucket", Provider: "g14", ProviderModule: "minio", CredentialForm: "pair"}
|
|
c.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(bucket)}
|
|
c.atNewest["g14"] = c.atWant["g14"]
|
|
c.s.write(t, HaveRead, "g14.minio", `{"v":1,"fp":"src2:`+strings.Repeat("cc", 32)+`","phase":"running","health":"ok"}`)
|
|
s3, s3cred := "g14.gitea.s3-bucket", "g14.gitea.secret-s3-bucket"
|
|
for _, row := range []struct {
|
|
name, entry, cred, state string
|
|
}{
|
|
{"its credential still asked", "", provisions.Asked, provisions.Asked},
|
|
{"its credential provided by the vault", `{"state":"provided","vault-name":"pair/g14/s3-bucket/g14/gitea","generation":1}`,
|
|
provisions.Provided, provisions.Provided},
|
|
} {
|
|
if row.entry != "" {
|
|
c.s.write(t, "mesh-vault.bound", s3cred, row.entry)
|
|
}
|
|
c.compose(t, "g14")
|
|
c.compose(t, "g14") // the credential judged first, then what depends on it
|
|
if got, dep := c.state(t, s3), c.state(t, s3cred); got.State != row.state || dep.State != row.cred {
|
|
t.Errorf("%s: %s is %s (%s), its credential %s", row.name, s3, got.State, got.Why, dep.State)
|
|
}
|
|
}
|
|
// An entry's change judges its key alone, no composition.
|
|
before := len(m.composed)
|
|
m.s.write(t, "postgres.bound", key, `{"refused":"the login is locked"}`)
|
|
if said, err := m.h.Rejudge(key); err != nil || !strings.Contains(said, provisions.Failed) || len(m.composed) != before {
|
|
t.Errorf("rejudged %q, %v, composed %d more", said, err, len(m.composed)-before)
|
|
}
|
|
}
|
|
|
|
// A provision no build wanted or placed needs is retiring with one ask naming provider, node and what goes; one composed
|
|
// ahead of its want raises none; a change removing one requirement and adding another retires only the removed one; the
|
|
// warrant goes into the key and the key goes once the provider removed its entry; a decline is kept and not asked again;
|
|
// a provision needed again returns from retiring (ADR 0570 clause 5; issue 1954 H3).
|
|
func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) {
|
|
m := newMesh()
|
|
pg, kc := "g14.gitea.postgres-database", "g14.gitea.oidc-client"
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
|
|
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, oidc)} // the newest build adds oidc ahead of its want
|
|
m.s.write(t, "postgres.bound", pg, `{}`)
|
|
steps := []struct {
|
|
name string
|
|
change func()
|
|
asked []string
|
|
states map[string]string
|
|
removed bool
|
|
}{
|
|
{"a key composed ahead of its want is asked, not retiring", func() {}, nil,
|
|
map[string]string{pg: provisions.Provided, kc: provisions.Asked}, false},
|
|
{"one requirement removed and another added: only the removed retires", func() {
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
|
|
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
|
|
}, []string{pg}, map[string]string{pg: provisions.Retiring, kc: provisions.Asked}, false},
|
|
{"composed again: no second ask", func() {}, nil, map[string]string{pg: provisions.Retiring}, false},
|
|
{"needed again before the warrant: back to its outcome", func() {
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)}
|
|
}, nil, map[string]string{pg: provisions.Provided}, false},
|
|
{"gone again: asked again", func() { m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} },
|
|
[]string{pg}, map[string]string{pg: provisions.Retiring}, false},
|
|
}
|
|
for _, s := range steps {
|
|
s.change()
|
|
c := m.compose(t, "g14")
|
|
if !slices.Equal(c.Asked, s.asked) {
|
|
t.Errorf("%s: asked %v, want %v", s.name, c.Asked, s.asked)
|
|
}
|
|
for k, st := range s.states {
|
|
if p := m.state(t, k); p.State != st {
|
|
t.Errorf("%s: %s is %s, want %s (%s)", s.name, k, p.State, st, p.Why)
|
|
}
|
|
}
|
|
}
|
|
p := m.state(t, pg)
|
|
a := m.r.asked[p.Ask]
|
|
if !strings.Contains(a.Explanation, "postgres") || !strings.Contains(a.Explanation, "ace") ||
|
|
!strings.Contains(a.Explanation, "data stay") {
|
|
t.Errorf("the ask does not say what goes from which provider on which node: %+v", a)
|
|
}
|
|
// Nothing before the warrant: the provider's entry stands and the key stays.
|
|
if said := m.h.Decide(); len(said) != 0 || m.state(t, pg).Warrant != nil {
|
|
t.Errorf("decided before an answer: %v", said)
|
|
}
|
|
m.r.answer(t, p.Ask, OptionApprove)
|
|
m.h.Decide()
|
|
if w := m.state(t, pg).Warrant; w == nil || w.Option != OptionApprove || !strings.Contains(w.By, "42") {
|
|
t.Fatalf("the warrant is not in the key: %+v", w)
|
|
}
|
|
// The word the SDK's provider lets an entry go on (provider.go): any other, and no provider ever retires it.
|
|
if w := m.state(t, pg).Warrant; w.Option != "yes" {
|
|
t.Fatalf("the warrant says %q, and a provider lets go only on \"yes\"", w.Option)
|
|
}
|
|
if said, _ := m.h.Rejudge(pg); said != "" || m.state(t, pg).State != provisions.Retiring {
|
|
t.Errorf("retired while the provider's entry stands: %q", said)
|
|
}
|
|
_ = m.s.Delete("postgres.bound", pg)
|
|
if said, _ := m.h.Rejudge(pg); !strings.Contains(said, "retired") || m.state(t, pg).State != "" {
|
|
t.Errorf("not retired once the provider removed its entry: %q", said)
|
|
}
|
|
// A decline is kept, and nothing is asked again while nothing changes.
|
|
d := newMesh()
|
|
d.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
|
|
d.compose(t, "g14")
|
|
d.atWant["g14"] = map[string]catalogue.Plan{}
|
|
d.compose(t, "g14")
|
|
d.r.answer(t, d.state(t, pg).Ask, OptionDecline)
|
|
d.h.Decide()
|
|
if c := d.compose(t, "g14"); len(c.Asked) != 0 || d.state(t, pg).Warrant.Option != OptionDecline {
|
|
t.Errorf("a decline was asked again: %+v", c)
|
|
}
|
|
}
|
|
|
|
// Issue 774, one bad module must not drop its node: a module whose manifest or plan refuses is held, named in `refused`
|
|
// with why, its last plan and its provisions kept; every other module on the node is composed and written; once it
|
|
// composes again its refusal goes.
|
|
func TestOneBadModuleDoesNotDropItsNode(t *testing.T) {
|
|
m := newMesh()
|
|
why := "the built manifest of nats at src2:6258f3 cannot be used"
|
|
steps := []struct {
|
|
name string
|
|
plans map[string]catalogue.Plan
|
|
held map[string]string
|
|
written []string // plans the step writes
|
|
refused bool // g14.nats in refused
|
|
}{
|
|
{"a node new to the seat with one bad module: the others are composed",
|
|
map[string]catalogue.Plan{"gitea": planOf(pgDB)}, map[string]string{"nats": why}, []string{"g14.gitea"}, true},
|
|
{"the bad one composes: its plan is written and its refusal goes",
|
|
map[string]catalogue.Plan{"gitea": planOf(pgDB), "nats": planOf(oidc)}, nil, []string{"g14.nats"}, false},
|
|
{"it refuses again: its last plan and its provision are kept, the others composed",
|
|
map[string]catalogue.Plan{"gitea": planOf(pgDB, split)}, map[string]string{"nats": why}, []string{"g14.gitea"}, true},
|
|
}
|
|
for _, s := range steps {
|
|
m.atWant["g14"], m.atNewest["g14"], m.heldAt["g14"] = s.plans, s.plans, s.held
|
|
c := m.compose(t, "g14")
|
|
if !slices.Equal(c.Plans, s.written) || len(c.Gone) != 0 || len(c.Asked) != 0 {
|
|
t.Errorf("%s: %+v", s.name, c)
|
|
}
|
|
e, _ := m.s.Get(RefusedState, "g14.nats")
|
|
var r Refusal
|
|
if e != nil {
|
|
_ = json.Unmarshal(e.Value, &r)
|
|
}
|
|
if (e != nil) != s.refused || (s.refused && (r.Module != "nats" || r.Why != why || c.Held["nats"] != why)) {
|
|
t.Errorf("%s: refused %+v, held %v", s.name, r, c.Held)
|
|
}
|
|
if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil {
|
|
t.Errorf("%s: gitea has no plan", s.name)
|
|
}
|
|
}
|
|
if e, _ := m.s.Get(PlansState, "g14.nats"); e == nil {
|
|
t.Error("the held module's last plan was removed")
|
|
}
|
|
if p := m.state(t, "g14.nats.oidc-client"); p.State != provisions.Asked {
|
|
t.Errorf("the held module's provision is %s (%s), not kept", p.State, p.Why)
|
|
}
|
|
got, err := m.h.read(RefusedState, "", "g14", "")
|
|
if rows := got.(map[string]any)[RefusedState].([]map[string]any); err != nil || len(rows) != 1 || rows[0]["why"] != why {
|
|
t.Errorf("the refusal reads %v, %v", got, err)
|
|
}
|
|
}
|
|
|
|
// Review of 2026-10-30: a module held on a node with no want (the bus on the control-node) is composed at the build
|
|
// its have says it runs, so the holder writes it a plan and its client keeps filling it.
|
|
func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) {
|
|
m := newMesh()
|
|
fp := "src2:" + strings.Repeat("dd", 32)
|
|
m.s.write(t, HaveRead, "novox.nats", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`)
|
|
m.s.write(t, HaveRead, "g14.dunst", `{"v":1,"fp":"src2:`+strings.Repeat("ee", 32)+`","phase":"running","health":"ok"}`)
|
|
var keptAt map[string]string
|
|
m.h.Compose = func(_ context.Context, node string, at map[string]string) Composition {
|
|
keptAt = at
|
|
return Composition{AtWants: HeldPlans{Plans: map[string]catalogue.Plan{"nats": {}}}}
|
|
}
|
|
// ADR 0580 clause 2: the reads come from one watched copy, each state watched once and none read key by key from the
|
|
// bus, and a change the copy takes wakes the node it touches (a published composition its node).
|
|
bus := &readsOf{Store: m.s, read: map[string]int{}}
|
|
handles, watches := map[string]func(stdio.StateChange) error{}, map[string]int{}
|
|
ws := stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
|
|
watches[state]++
|
|
handles[state] = handle
|
|
keys, _ := m.s.Keys(state)
|
|
for _, k := range keys {
|
|
e, _ := m.s.Get(state, k)
|
|
_ = handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: e.Value, Current: true})
|
|
}
|
|
return nil
|
|
}, func(string, string) ([]byte, bool, error) { return nil, false, nil })
|
|
m.h.Store = Watching(bus, ws)
|
|
wk := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)}
|
|
for _, r := range []string{ComposedRead, HaveRead} {
|
|
ws.Follow(r, func(c stdio.StateChange) error { wk.add(NodesOfNodeKey(c.Key)...); return nil })
|
|
}
|
|
for i := range 2 {
|
|
c := m.compose(t, "novox")
|
|
if keptAt["nats"] != fp || len(keptAt) != 1 {
|
|
t.Fatalf("composed at %v, want nats at the build it runs and nothing of another node", keptAt)
|
|
}
|
|
if i == 0 && !slices.Contains(c.Plans, "novox.nats") {
|
|
t.Fatalf("no plan written for a module held without a want: %+v", c)
|
|
}
|
|
// Issue 1954: its own states come from the copy too, its own writes read back at once (nothing is written
|
|
// again), and the result says where the time went, its whole last.
|
|
if i == 1 && len(c.Plans)+len(c.Changed) > 0 {
|
|
t.Fatalf("composed again unchanged, it wrote %+v: its own writes were not read back", c)
|
|
}
|
|
if n := len(c.Took); n == 0 || c.Took[n-1].Phase != "whole" || c.Took[0].Phase != "wants" {
|
|
t.Fatalf("the composition says its phases as %v", c.Took)
|
|
}
|
|
}
|
|
for _, r := range append(slices.Clone(WatchedReads), OwnStates...) {
|
|
if bus.read[r] != 0 || watches[r] > 1 {
|
|
t.Errorf("%s was read %d times from the bus and watched %d times", r, bus.read[r], watches[r])
|
|
}
|
|
}
|
|
// The current haves queued their nodes when watched; a later change of a published composition or a have
|
|
// queues its node.
|
|
wk.pending = map[string]bool{}
|
|
_ = handles[ComposedRead](stdio.StateChange{Key: "g14.dunst." + strings.Repeat("ee", 32), Op: stdio.OpPut,
|
|
Value: []byte("{}")})
|
|
_ = handles[HaveRead](stdio.StateChange{Key: "ace.restic", Op: stdio.OpDelete})
|
|
if !wk.pending["g14"] || !wk.pending["ace"] || len(wk.pending) != 2 {
|
|
t.Errorf("a published composition's and a have's change queued %v, want g14 and ace", wk.pending)
|
|
}
|
|
}
|
|
|
|
// readsOf counts the reads that reach the bus, by state.
|
|
type readsOf struct {
|
|
Store
|
|
read map[string]int
|
|
}
|
|
|
|
func (r *readsOf) Get(state, key string) (*Entry, error) {
|
|
r.read[state]++
|
|
return r.Store.Get(state, key)
|
|
}
|
|
func (r *readsOf) Keys(state string) ([]string, error) { r.read[state]++; return r.Store.Keys(state) }
|
|
|
|
// The holder holds no ticker: it composes on change events alone (ADR 0570 clause 2; issue 1954 H1).
|
|
func TestTheHolderHoldsNoTimer(t *testing.T) {
|
|
files, _ := filepath.Glob("*.go")
|
|
for _, f := range files {
|
|
if strings.HasSuffix(f, "_test.go") {
|
|
continue
|
|
}
|
|
file, err := parser.ParseFile(token.NewFileSet(), f, nil, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ast.Inspect(file, func(n ast.Node) bool {
|
|
if sel, ok := n.(*ast.SelectorExpr); ok {
|
|
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "time" &&
|
|
slices.Contains([]string{"NewTicker", "Tick", "AfterFunc", "NewTimer"}, sel.Sel.Name) {
|
|
t.Errorf("%s uses time.%s: the holder composes on events alone", f, sel.Sel.Name)
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
}
|
|
|
|
// manifestOf puts a provider's manifest into the indexer's reads as the holder finds it.
|
|
func (m *mesh) manifestOf(module, raw string) {
|
|
digest := delivery.ManifestDigest([]byte(raw))
|
|
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
|
|
_ = m.s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value))
|
|
_ = m.s.Put(ModulesRead, mustKey(delivery.ModulesKey(module)),
|
|
json.RawMessage(`{"v":1,"none2":"a test","manifest":"`+digest+`","repository":"novox/mesh-catalog","commit":"c1"}`))
|
|
}
|
|
|
|
// A provider's outcome is read per offer: from its entries where the provider helper answers that offer, from its have
|
|
// where it does not, and the entries of route-proxy and node-tools are read (issue 774, the sweep of 2026-10-30 20:10:
|
|
// <node>.claude-code.gitea-read failed "gitea does not serve" on all four nodes while gitea's helper answers only
|
|
// forge-account, and novox.gitea.route and <node>.claude-code.mcp-endpoint stood asked, their providers' entries unread).
|
|
func TestAProvidersOutcomeIsReadPerOffer(t *testing.T) {
|
|
const (
|
|
gitea = `{"module":"gitea","version":"1","state":["bound","announced"],"provides":[{"name":"npm-package-registry","credential":{"form":"pair"}},{"name":"gitea-read","credential":{"issued-by":"software","tool":"gitea_agent_token"}},{"name":"forge-account","credential":{"form":"none"},"answered-by":"helper"}]}`
|
|
routes = `{"module":"route-proxy","version":"1","state":["bound"],"provides":[{"name":"route","scope":"mesh","identity":false,"answered-by":"helper"}]}`
|
|
nodeTools = `{"module":"node-tools","version":"1","state":["bound"],"provides":[{"name":"mcp-endpoint","scope":"node","answered-by":"helper"}]}`
|
|
running = `{"v":1,"fp":"src2:` + "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + `","phase":"running","health":"ok"}`
|
|
refused = `{"state":"failed","refused":"claude-code on ace is bound to gitea on novox for gitea-read, which gitea does not serve"}`
|
|
)
|
|
for _, row := range []struct {
|
|
name string
|
|
node string
|
|
plan map[string]catalogue.Plan
|
|
key string
|
|
bucket string // the provider's entries, written as entry
|
|
entry string
|
|
have string // the provider's have key, written running
|
|
state string
|
|
why string // a part of the provision's why, when it says one
|
|
}{
|
|
{"gitea-read: the helper's refusal is not its outcome; gitea runs, so provided", "ace",
|
|
map[string]catalogue.Plan{"claude-code": planOf(catalogue.PlanProvision{Provision: "gitea-read", Provider: "novox", ProviderModule: "gitea"})},
|
|
"ace.claude-code.gitea-read", "gitea.bound", refused, "novox.gitea", provisions.Provided, ""},
|
|
{"forge-account: still the helper's entry", "ace",
|
|
map[string]catalogue.Plan{"mesh-build-go": planOf(catalogue.PlanProvision{Provision: "forge-account", Provider: "novox", ProviderModule: "gitea"})},
|
|
"ace.mesh-build-go.forge-account", "gitea.bound", `{"state":"provided"}`, "", provisions.Provided, ""},
|
|
{"route: route-proxy's entry is read", "novox",
|
|
map[string]catalogue.Plan{"gitea": planOf(catalogue.PlanProvision{Provision: "route", Provider: "novox", ProviderModule: "route-proxy"})},
|
|
"novox.gitea.route", "route-proxy.bound", `{"state":"provided"}`, "", provisions.Provided, ""},
|
|
{"mcp-endpoint: node-tools' entry is read", "ace",
|
|
map[string]catalogue.Plan{"claude-code": planOf(catalogue.PlanProvision{Provision: "mcp-endpoint", Provider: "ace", ProviderModule: "node-tools"})},
|
|
"ace.claude-code.mcp-endpoint", "node-tools.bound", `{"state":"provided"}`, "", provisions.Provided, ""},
|
|
// Issue 774, 2026-10-31: route-proxy's helper read the controller's assignments, never the seat's plans, so 38
|
|
// routes the seat keys stood asked with no why while the assigner withheld their consumers' builds.
|
|
{"route: no entry from a provider that reads no plans of the seat is said so", "ace",
|
|
map[string]catalogue.Plan{"grafana": planOf(catalogue.PlanProvision{Provision: "route", Provider: "ace", ProviderModule: "route-proxy"})},
|
|
"ace.grafana.route", "", "", "", provisions.Asked, "reads no plans of the seat"},
|
|
} {
|
|
m := newMesh()
|
|
m.manifestOf("gitea", gitea)
|
|
m.manifestOf("route-proxy", routes)
|
|
m.manifestOf("node-tools", nodeTools)
|
|
m.atWant[row.node], m.atNewest[row.node] = row.plan, row.plan
|
|
if row.bucket != "" {
|
|
m.s.write(t, row.bucket, row.key, row.entry)
|
|
}
|
|
if row.have != "" {
|
|
m.s.write(t, HaveRead, row.have, running)
|
|
}
|
|
m.compose(t, row.node)
|
|
if p := m.state(t, row.key); p.State != row.state || !strings.Contains(p.Why, row.why) {
|
|
t.Errorf("%s: %s is %s (%s)", row.name, row.key, p.State, p.Why)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Issue 1126 review 1 (mcp-endpoint asked on ace and shanks, 2026-11-02 11:56): a requirement the consumer's newest build
|
|
// adds is keyed before its want, and the plan the seat keeps is its wanted build's, which names it not; the key carries
|
|
// the plan entry it was composed from, so the provider binds it, and its why says no plan names it yet. A key its
|
|
// wanted plan names carries none.
|
|
func TestAKeyAheadOfItsWantCarriesItsPlanEntryAndSaysWhy(t *testing.T) {
|
|
m := newMesh()
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
|
|
ahead := oidc
|
|
ahead.As = "gitea-g14"
|
|
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, ahead)}
|
|
m.compose(t, "g14")
|
|
kc := m.state(t, "g14.gitea.oidc-client")
|
|
if kc.Plan == nil || kc.Plan.Provision != "oidc-client" || kc.Plan.ProviderModule != "keycloak" || kc.Plan.As != "gitea-g14" {
|
|
t.Errorf("the key ahead of its want carries %+v", kc.Plan)
|
|
}
|
|
if kc.State != provisions.Asked || !strings.Contains(kc.Why, "no plan names it") {
|
|
t.Errorf("the key ahead of its want: %s %q", kc.State, kc.Why)
|
|
}
|
|
// A key a plan names carries no entry: a reader on an older mesh-sdk, which refused a field it did not know, reads it
|
|
// still, and skips only a key ahead of its want, which it could never bind (review 1).
|
|
if pg := m.state(t, "g14.gitea.postgres-database"); pg.Plan != nil || strings.Contains(pg.Why, "no plan names it") {
|
|
t.Errorf("the planned key: %+v", pg)
|
|
}
|
|
}
|
|
|
|
// Issue 940 (met before as 937 and 1332): a retirement is judged from a whole picture only. At 2026-11-02 22:55:58 a
|
|
// first pass after a new membership read a part, counted the modules missing from it as handing nothing, and asked the
|
|
// operator to remove the routes of 21 running web apps; 3 s later the next pass put every plan back.
|
|
func TestARetirementIsJudgedOnlyFromAWholePicture(t *testing.T) {
|
|
fp := "src2:" + strings.Repeat("ab", 32)
|
|
route := func(n int) catalogue.PlanProvision {
|
|
return catalogue.PlanProvision{Provision: "route", Provider: "novox", ProviderModule: "keycloak", Local: fmt.Sprint(n)}
|
|
}
|
|
apps := []string{"de-spiegel", "invoicing", "link2pay", "nextcloud", "photos", "website"}
|
|
for _, row := range []struct {
|
|
name string
|
|
consumer int // how many of apps hold a key, each one
|
|
gone int // how many of them the pass no longer composes
|
|
change func(m *mesh) // what else the pass sees
|
|
many bool // the operator's retire-many
|
|
retiring int // keys retiring after the pass
|
|
asked int // asks raised by the pass
|
|
problem string // a part of a problem the pass says
|
|
fault string // a part of the health's why, "" for healthy
|
|
after func(*testing.T, *mesh) // a further check
|
|
}{
|
|
{name: "a read not loaded once: nothing judged", consumer: 6, gone: 6,
|
|
change: func(m *mesh) { m.h.Loaded = func() string { return WantsRead } }},
|
|
{name: "another module's composition not published: a removed consumer still retires", consumer: 2, gone: 1,
|
|
change: func(m *mesh) {
|
|
m.unpublished["novox"] = []string{"gitea"}
|
|
m.heldAt["novox"] = map[string]string{"gitea": "the composition of gitea at " + fp + " is not published yet"}
|
|
}, retiring: 1, asked: 1, after: func(t *testing.T, m *mesh) {
|
|
// Held at one build: healthy over many passes within the bound (a build takes minutes), unhealthy past it,
|
|
// naming the module, its build and since when; another build starts the clock again.
|
|
now := m.h.Now()
|
|
m.h.Now = func() time.Time { return now }
|
|
for _, row := range []struct {
|
|
name string
|
|
after time.Duration
|
|
healthy bool
|
|
}{
|
|
{"many passes within the bound", 0, true},
|
|
{"still within the bound", unpublishedFor - time.Minute, true},
|
|
{"past the bound", unpublishedFor + time.Minute, false},
|
|
} {
|
|
now = now.Add(row.after)
|
|
for range 5 {
|
|
m.compose(t, "novox")
|
|
}
|
|
a := m.h.health()
|
|
if a.Healthy != row.healthy || (!row.healthy && (!strings.Contains(a.Why, "novox.gitea") ||
|
|
!strings.Contains(a.Why, fp) || !strings.Contains(a.Why, "held since"))) {
|
|
t.Errorf("%s: %+v", row.name, a)
|
|
}
|
|
}
|
|
m.heldAt["novox"] = map[string]string{"gitea": "the composition of gitea at src2:" + strings.Repeat("cd", 32) +
|
|
" is not published yet"}
|
|
m.compose(t, "novox")
|
|
if a := m.h.health(); !a.Healthy {
|
|
t.Errorf("another build did not start the clock again: %+v", a)
|
|
}
|
|
m.unpublished["novox"], m.heldAt["novox"] = nil, nil
|
|
m.compose(t, "novox")
|
|
if a := m.h.health(); !a.Healthy {
|
|
t.Errorf("the hold is said after it ended: %+v", a)
|
|
}
|
|
}},
|
|
{name: "a consumer that still has a want: not retired", consumer: 2, gone: 1,
|
|
change: func(m *mesh) {
|
|
m.s.write(t, WantsRead, "want.novox.de-spiegel", `{"v":1,"fp":"`+fp+`","self":true}`)
|
|
}},
|
|
{name: "a consumer that still has a have: not retired", consumer: 2, gone: 1,
|
|
change: func(m *mesh) {
|
|
m.s.write(t, HaveRead, "novox.de-spiegel", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`)
|
|
}},
|
|
{name: "one consumer of two leaving: retired and asked", consumer: 2, gone: 1, retiring: 1, asked: 1},
|
|
{name: "most consumers at once: a fault, not a question", consumer: 6, gone: 6, problem: "a read stopped listing them",
|
|
fault: "6 of the 6 consumers"},
|
|
{name: "more than 3 at once, though fewer than half", consumer: 6, gone: 4, fault: "4 of the 6"},
|
|
{name: "the operator's retire-many: retired, at most 3 asked at once", consumer: 6, gone: 6, many: true,
|
|
retiring: 6, asked: askOpenAtMost, after: func(t *testing.T, m *mesh) {
|
|
var waiting int
|
|
for _, a := range apps {
|
|
if strings.Contains(m.state(t, "novox."+a+".route.1").Why, askWaits) {
|
|
waiting++
|
|
}
|
|
}
|
|
if waiting != 6-askOpenAtMost {
|
|
t.Errorf("%d keys say their ask waits, want %d", waiting, 6-askOpenAtMost)
|
|
}
|
|
}},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
m := newMesh()
|
|
plans := map[string]catalogue.Plan{}
|
|
for i := range row.consumer {
|
|
plans[apps[i]] = planOf(route(1))
|
|
}
|
|
m.atWant["novox"], m.atNewest["novox"] = plans, plans
|
|
m.compose(t, "novox")
|
|
left := map[string]catalogue.Plan{}
|
|
for i := row.gone; i < row.consumer; i++ {
|
|
left[apps[i]] = plans[apps[i]]
|
|
}
|
|
m.atWant["novox"], m.atNewest["novox"] = left, left
|
|
if row.change != nil {
|
|
row.change(m)
|
|
}
|
|
c, err := m.h.ComposeNodeWith(context.Background(), "novox", row.many)
|
|
if m.h.Loaded != nil {
|
|
if err == nil || !strings.Contains(err.Error(), WantsRead) {
|
|
t.Errorf("a pass before every read loaded: %v", err)
|
|
}
|
|
} else if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
retiring := 0
|
|
for i := range row.consumer {
|
|
if m.state(t, "novox."+apps[i]+".route.1").State == provisions.Retiring {
|
|
retiring++
|
|
}
|
|
}
|
|
if retiring != row.retiring || len(c.Asked) != row.asked || len(m.r.asked) != row.asked {
|
|
t.Errorf("%d retiring, %d asked (%d published), want %d and %d: %+v", retiring, len(c.Asked),
|
|
len(m.r.asked), row.retiring, row.asked, c)
|
|
}
|
|
if row.problem != "" && !strings.Contains(strings.Join(c.Problems, " "), row.problem) {
|
|
t.Errorf("the problems %v do not say %q", c.Problems, row.problem)
|
|
}
|
|
if a := m.h.health(); (row.fault == "") != a.Healthy || !strings.Contains(a.Why, row.fault) ||
|
|
(row.fault != "" && !strings.Contains(a.Why, "retire-many")) {
|
|
t.Errorf("the health is %+v, want a fault saying %q", a, row.fault)
|
|
}
|
|
if row.fault != "" {
|
|
// The read whole again: the fault clears.
|
|
m.atWant["novox"], m.atNewest["novox"] = plans, plans
|
|
m.compose(t, "novox")
|
|
if a := m.h.health(); !a.Healthy {
|
|
t.Errorf("the fault stands once the picture is whole: %+v", a)
|
|
}
|
|
}
|
|
if row.after != nil {
|
|
row.after(t, m)
|
|
}
|
|
})
|
|
}
|
|
// A placed module no build of which is known is held, not counted as handing nothing; one unpublished says so.
|
|
var c today
|
|
readJSON(t, "testdata/today-composed.json", &c)
|
|
s := goldenMesh(t, c)
|
|
published(t, s, c)
|
|
s.write(t, PlacementRead, "placement."+c.Node, `{"v":1,"node":"`+c.Node+`","decided":{},"modules":[{"module":"store"},`+
|
|
`{"module":"postgres"},{"module":"mesh-provisioner-client"},{"module":"restic"},{"module":"de-spiegel"}]}`)
|
|
got := FromCompositions(s)(context.Background(), c.Node, map[string]string{"photos": fp})
|
|
if !strings.Contains(got.AtWants.Held["de-spiegel"], "no build") || !slices.Equal(got.Unpublished, []string{"photos"}) {
|
|
t.Errorf("held %v, unpublished %v", got.AtWants.Held, got.Unpublished)
|
|
}
|
|
}
|
|
|
|
// Issue 940: an ask the router does not hold is not counted asked; it is raised again once an open ask ends.
|
|
func TestAnAskTheRouterDoesNotHoldIsRaisedAgain(t *testing.T) {
|
|
m := newMesh()
|
|
key := "g14.gitea.postgres-database"
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB), "dunst": planOf(oidc)}
|
|
m.compose(t, "g14")
|
|
m.atWant["g14"] = map[string]catalogue.Plan{"dunst": planOf(oidc)}
|
|
m.r.drop = true
|
|
m.compose(t, "g14")
|
|
first := m.state(t, key).Ask
|
|
if first == "" {
|
|
t.Fatal("not asked")
|
|
}
|
|
m.r.drop = false
|
|
steps := []struct {
|
|
name string
|
|
later time.Duration
|
|
held bool // the router holds an ask of the key after the step
|
|
}{
|
|
{"within a minute the router may not have taken it yet", 0, false},
|
|
{"a minute on, with no record: raised again", 2 * time.Minute, true},
|
|
}
|
|
for _, s := range steps {
|
|
now := m.h.Now()
|
|
m.h.Now = func() time.Time { now = now.Add(time.Second); return now.Add(s.later) }
|
|
m.h.Decide()
|
|
p := m.state(t, key)
|
|
if _, held := m.r.records[p.Ask]; held != s.held || (s.held && p.Ask == first) {
|
|
t.Errorf("%s: the key's ask is %q, the router holds it: %v", s.name, p.Ask, held)
|
|
}
|
|
}
|
|
}
|