Files
mesh-catalog/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go
T
jschoubben 6c61d9facf S6: the condition keeper reads the node-engine's findings
ace's node-engine wrote seat-list-unread in its state findings, and no
condition rose: the keeper's observers did not name the node-engine. It now
reads mesh-host.findings, so ADR 0636 §1's condition is raised and clears at
the first full read (issue 1966).

Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
2026-11-10 00:49:55 +01:00

421 lines
17 KiB
Go

package main
import (
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
meshhealers "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhealers"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// fakeStore is the bus's states in memory, with revisions, as the runtime answers them.
type fakeStore struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
next uint64
// ttls are the times to live the keys were created with.
ttls map[string]time.Duration
}
func newFakeStore() *fakeStore {
return &fakeStore{vals: map[string][]byte{}, revs: map[string]uint64{}}
}
func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
v, ok := f.vals[state+"/"+key]
return v, f.revs[state+"/"+key], ok, nil
}
func (f *fakeStore) put(state, key string, value any) uint64 {
raw, _ := json.Marshal(value)
f.next++
f.vals[state+"/"+key], f.revs[state+"/"+key] = raw, f.next
return f.next
}
func (f *fakeStore) Create(state, key string, value any) (uint64, error) {
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.vals[state+"/"+key]; ok {
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
}
return f.put(state, key, value), nil
}
func (f *fakeStore) Update(state, key string, value any, rev uint64) (uint64, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.revs[state+"/"+key] != rev {
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
}
return f.put(state, key, value), nil
}
func (f *fakeStore) Delete(state, key string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.vals, state+"/"+key)
delete(f.revs, state+"/"+key)
return nil
}
func (f *fakeStore) Keys(state string) ([]string, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []string
for k := range f.vals {
if s, key, _ := strings.Cut(k, "/"); s == state {
out = append(out, key)
}
}
sort.Strings(out)
return out, nil
}
func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
f.mu.Lock()
if f.ttls == nil {
f.ttls = map[string]time.Duration{}
}
f.ttls[state+"/"+key] = ttl
f.mu.Unlock()
_, err := f.Create(state, key, value)
return err
}
// rig is a keeper over a fake store, its clock and what it said.
type rig struct {
k *Keeper
st *fakeStore
now time.Time
said []Event
}
func newRig() *rig {
r := &rig{st: newFakeStore(), now: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)}
r.k = &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e Event) error {
r.said = append(r.said, e)
return nil
}}
return r
}
func (r *rig) changes() string {
var out []string
for _, e := range r.said {
out = append(out, e.Change+" "+e.Key)
}
r.said = nil
return strings.Join(out, ", ")
}
func (r *rig) open(key string) (Condition, bool) {
raw, _, ok, _ := r.st.Get(ConditionsState, key)
var c Condition
if ok {
// The seat's layout, read strictly: a field the seat does not name is a fault of the keeper's write.
v, err := seat.DecodeConditionsByKeyValue(raw)
if err != nil {
panic(fmt.Sprintf("the condition %s is not the seat's layout: %v: %s", key, err, raw))
}
c = conditionOf(v)
}
return c, ok
}
// encodeFinding writes a finding as an observer does: the seat's layout, checked.
func encodeFinding(f seat.FindingsByKeyValue) ([]byte, error) {
f.V = cc.FindingVersion
if f.Evidence == nil {
f.Evidence = []seat.Evidence{}
}
if err := cc.CheckFinding(f); err != nil {
return nil, err
}
return json.Marshal(f)
}
// finding is an observer's finding as mesh-sdk's layout writes it (encodeFinding).
func finding(t *testing.T, observer, scope, id, kind, source string, sev string, at time.Time) (string, []byte) {
t.Helper()
f := seat.FindingsByKeyValue{Key: cc.Key(scope, id, kind), Kind: kind, Subject: seat.Subject{Scope: scope, ID: id},
Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source,
Observer: observer, At: at, Evidence: []seat.Evidence{{At: at, Said: kind}}}
raw, err := encodeFinding(f)
if err != nil {
t.Fatal(err)
}
return f.Key, raw
}
// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer
// clears nothing; the healers' attempts reach the condition; a reopening within ten
// minutes is the same fault; a silence is a transition in history.
func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) {
r := newRig()
key, raw := finding(t, watchman, cc.ScopeMachine, "g14", "silent", "S1", cc.Warning, r.now)
steps := []struct {
name string
do func() error
said string
check func(c Condition, open bool) string
}{
{name: "a finding written raises", do: func() error { return r.k.Finding(watchman, key, raw, false) },
said: "raised machine.g14.silent", check: func(c Condition, open bool) string {
if !open || c.Severity != cc.Warning || c.Resolver != cc.ResolverSelf || c.Epoch != 0 || c.Count != 1 {
return fmt.Sprintf("%+v", c)
}
return ""
}},
{name: "the same finding again says nothing", do: func() error { return r.k.Finding(watchman, key, raw, false) }},
{name: "an observer gone blind clears nothing", do: func() error { r.k.Blind(watchman, "refused"); _, err := r.k.Resync(); return err },
check: func(c Condition, open bool) string {
if !open {
return "cleared by blindness"
}
return ""
}},
{name: "a healer's attempt reaches tried and the resolver", do: func() error {
a, _ := json.Marshal(meshhealers.AttemptsByAttemptValue{Healer: "H3", Condition: key, Act: "asserted", Outcome: meshhealers.AttemptsByAttemptValueOutcomeActed, Begun: r.now})
return r.k.Attempt("h3-1", a, false)
}, said: "resolver machine.g14.silent", check: func(c Condition, _ bool) string {
if c.Resolver != "healer:H3" || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" {
return fmt.Sprintf("%+v", c)
}
return ""
}},
{name: "an escalation makes it urgent, the operator's", do: func() error {
a, _ := json.Marshal(meshhealers.AttemptsByAttemptValue{Healer: "H3", Condition: key, Act: "handed over", Outcome: meshhealers.AttemptsByAttemptValueOutcomeEscalated,
Begun: r.now.Add(time.Minute)})
return r.k.Attempt("h3-2", a, false)
}, said: "severity machine.g14.silent, resolver machine.g14.silent", check: func(c Condition, _ bool) string {
if c.Severity != cc.Urgent || c.Resolver != cc.ResolverOperator || len(c.Tried) != 2 {
return fmt.Sprintf("%+v", c)
}
return ""
}},
{name: "a silence is a transition with who and why", do: func() error {
_, err := r.k.Silence(key, time.Hour, "jochen", "known, a cable")
return err
}, said: "silenced machine.g14.silent"},
{name: "the finding deleted clears", do: func() error { return r.k.Finding(watchman, key, nil, true) },
said: "cleared machine.g14.silent", check: func(_ Condition, open bool) string {
if open {
return "still open"
}
return ""
}},
{name: "raised again within ten minutes is a reopening", do: func() error {
r.now = r.now.Add(5 * time.Minute)
return r.k.Finding(watchman, key, raw, false)
}, said: "reopened machine.g14.silent", check: func(c Condition, _ bool) string {
if c.Count != 2 || len(c.Gaps) != 1 {
return fmt.Sprintf("%+v", c)
}
return ""
}},
}
for _, s := range steps {
if err := s.do(); err != nil {
t.Fatalf("%s: %v", s.name, err)
}
if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" {
t.Errorf("%s: said %q, want %q", s.name, got, s.said)
}
if s.check != nil {
c, open := r.open(key)
if why := s.check(c, open); why != "" {
t.Errorf("%s: %s", s.name, why)
}
}
}
// Every transition is kept in history, and each event carries the keeper's show.
keys, _ := r.st.Keys(HistoryState)
if len(keys) < 7 {
t.Errorf("history keeps %d transitions", len(keys))
}
raw0, _, _, _ := r.st.Get(HistoryState, keys[0])
if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) {
t.Errorf("the first transition is %s", raw0)
}
}
// 1126/A5/AC2: the keeper carries no controller condition: the controller's bucket, as Controller hands it, opens no
// condition and changes none (no escalation, no silence). What it carried while the controller ran, still open, is cleared
// once by EndCarried, said as cleared; a condition an observer's finding holds stays; a second EndCarried clears nothing.
func TestTheKeeperCarriesNoControllerConditionAndClearsWhatItCarriedOnce(t *testing.T) {
r := newRig()
key, raw := finding(t, watchman, cc.ScopeMachine, "g14", "silent", "S1", cc.Warning, r.now)
if err := r.k.Finding(watchman, key, raw, false); err != nil {
t.Fatal(err)
}
r.said = nil
// Carried while the controller ran: two of kinds that ended with it, still open in conditions.
for _, c := range []Condition{
{Key: "core.controller.deaf", Kind: "controller-deaf", Severity: cc.Urgent, Resolver: "self", Count: 1},
{Key: "seat.vault.ace.holder-silent", Kind: "holder-silent", Severity: cc.Warning, Resolver: "self", Count: 1},
} {
if _, err := r.st.Create(ConditionsState, c.Key, c.Stored()); err != nil {
t.Fatal(err)
}
}
n, err := r.k.EndCarried()
if err != nil || n != 2 || r.changes() != "cleared core.controller.deaf, cleared seat.vault.ace.holder-silent" {
t.Fatalf("EndCarried cleared %d (%v)", n, err)
}
if _, open := r.open(key); !open {
t.Error("a condition a finding holds was cleared")
}
if n, err := r.k.EndCarried(); err != nil || n != 0 || r.changes() != "" {
t.Errorf("a second EndCarried cleared %d (%v)", n, err)
}
}
// ADR 0564 (issue 1952 C5; the lead 2026-10-29): every kind the controller raised has its home in the table, and the
// kinds that ended with it are marked carried, which EndCarried clears once (issue 1126). compare, take-over and
// hand-back went with the controller (ADR 0590): there is no other side to compare and nothing to hand back to.
func TestEveryControllerKindHasItsHome(t *testing.T) {
// The lead's table of 2026-10-29: each kind has its home, lease-split and image-missing do not end, and the 22
// that end are carried; with the 11 kinds of the probes the operator dropped on 2026-10-30, 33.
carried := 0
for _, h := range Homes {
if h.Carried {
carried++
}
}
for kind, observer := range map[string]string{"needs-operator": watchman, "machine-network": watchman, "lease-split": selfCheck,
"data-shrank": selfCheck, "retire-waiting": provisioner, "asks-undelivered": messenger,
"image-missing": artifactStore, "build-failed": builder, "floor-silent": builder, "old-form-held": builder,
"tag-removal-not-done": builder, "seat-definition-unparsed": indexer, "generated-types-out-of-step": indexer,
"seat-defined-twice": indexer, "accepted-unsaid": indexer, "manifest-held": indexer, "packages-token-refused": gitea,
"publish-refused": builder, "publish-unanswered": builder, "seat-outcome-unread": builder} {
if h, ok := Homes[kind]; !ok || h.Observer != observer || h.Carried {
t.Errorf("%s's home is %+v", kind, h)
}
// The keeper reads every observer's findings that a kind's home names (the registry milestone's join).
if !slices.Contains(Observers, observer) {
t.Errorf("%s's observer %s is not read by the keeper", kind, observer)
}
}
for _, kind := range []string{"resolver-wrong", "holder-silent", "archives-missing", "archives-unheld", "binding-kept",
"binding-moved", "binding-moving", "grant-composes-nothing", "root-not-free", "cleanup-waiting", "data-held-twice"} {
if h := Homes[kind]; !h.Carried || h.Observer != "" || h.Why != Dropped {
t.Errorf("the dropped probe's kind %s is %+v", kind, h)
}
}
// And the controller's own bus upgrade, carried (the operator, 2026-10-31): 34; and the bus-consumer probe's three
// kinds, ended with the controller (issue 1126): 37.
for _, kind := range []string{"consumer-wrong", "consumer-lost", "consumer-behind"} {
if h := Homes[kind]; !h.Carried || h.Observer != "" || h.Why != EndedD6 {
t.Errorf("the ended probe's kind %s is %+v", kind, h)
}
}
// The readers' unanswered first view and the owners' views too large to say are every reader's and owner's own.
for _, kind := range []string{"first-view-unanswered", "view-too-large", "view-unsaid", "family-not-heard"} {
if h := Homes[kind]; h.Observer != "" || h.Carried || h.Controller {
t.Errorf("%s's home is %+v", kind, h)
}
}
// Each module that writes a reader's unanswered first view (ADR 0615 §6), or that a delivery part's consumer stopped
// hearing, is read: the delivery parts and the tool runner by name (other repositories), and every catalogue module
// whose code writes a views finding, found in the catalogue's tree, so a new writer cannot go unobserved.
// And the node-engine, whose node writes seat-list-unread when it has read no seat list (ADR 0636 §1, issue 1966).
readers := []string{watchman, "mesh-delivery-teller", enrolment, builder, "mesh-delivery-checker", "mesh-delivery-view",
indexer, assignerModule, configModule, vaultModule, gitea, "node-tools", "mesh-host"}
readers = append(readers, viewFindingWriters(t, "../../..")...)
for _, reader := range readers {
if !slices.Contains(Observers, reader) {
t.Errorf("the reader %s's findings are not read by the keeper", reader)
}
}
// 19 of ADR 0564, the 63 of the table, plan-failed and stalled, mesh-config's four (design 32), the readers' unanswered
// first view, the owners' view too large, the builder's four, the indexer's four and the package seats' token refused.
// And the builder's three publishing kinds and the delivery parts' family not heard: 103.
// And the owners' changes not said, view-unsaid: 104. And the indexer's mesh-wide hold past its bound, manifest-held
// (ADR 0632 §2): 105. (The bus module's own conditions, nats.status-unwritten among them, are said to the keeper
// directly, never through findings, so they have no home here.)
if len(Homes) != 105 || carried != 37 {
t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried)
}
}
// viewFindingWriters are the catalogue modules (under modules) whose non-test Go code writes a views finding into its own
// findings: views.UnansweredFinding, views.TooLargeFinding, views.UnsaidFinding or views.OwnFindings.
func viewFindingWriters(t *testing.T, modules string) []string {
t.Helper()
writes := regexp.MustCompile(`(views\.(UnansweredFinding|TooLargeFinding|UnsaidFinding)|meshconditionsclient\.FindingWriter)\b`)
var out []string
entries, err := os.ReadDir(modules)
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
if !e.IsDir() || e.Name() == Module {
continue
}
found := false
_ = filepath.WalkDir(filepath.Join(modules, e.Name()), func(path string, d fs.DirEntry, err error) error {
if err != nil || found || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return nil
}
if src, err := os.ReadFile(path); err == nil && writes.Match(src) {
found = true
}
return nil
})
if found {
out = append(out, e.Name())
}
}
if len(out) < 5 {
t.Fatalf("found %d catalogue modules writing views findings in %s: the tree was not read (%v)", len(out), modules, out)
}
return out
}
// ADR 0634 §1, §4 (issue 1966): the keeper reads a condition written before the seat named its layout (the controller's
// body: `silenced` null, `actions` empty, the times with an offset) and writes it back in the seat's layout.
func TestTheKeeperReadsTheOldLayoutAndWritesTheSeatsLayout(t *testing.T) {
r := newRig()
key, raw := finding(t, watchman, cc.ScopeMachine, "g14", "silent", "S1", cc.Warning, r.now)
old := `{"key":"` + key + `","kind":"silent","subject":{"scope":"machine","id":"g14"},"severity":"warning",` +
`"summary":"old","headline":"","explanation":"","resolved":"","needs":"","actions":[],"evidence":[],"source":"S1",` +
`"raised":"2026-10-29T13:00:00+01:00","last-observed":"2026-10-29T13:00:00+01:00","observations":1,"count":1,` +
`"resolver":"self","silenced":null,"epoch":0}`
if _, err := r.st.Create(ConditionsState, key, json.RawMessage(old)); err != nil {
t.Fatal(err)
}
if err := r.k.Finding(watchman, key, raw, false); err != nil {
t.Fatal(err)
}
c, ok := r.open(key)
if !ok || c.Summary != "silent of g14" || !c.Raised.Equal(r.now) || c.Count != 1 {
t.Fatalf("the old condition, settled: %+v (open %v)", c, ok)
}
if _, err := r.k.Silence(key, time.Hour, "tester", "known"); err != nil {
t.Fatal(err)
}
if c, _ := r.open(key); c.Silenced == nil || !c.Silenced.Until.Equal(r.now.Add(time.Hour)) {
t.Errorf("the silence is not kept in the seat's layout: %+v", c.Silenced)
}
last := r.said[len(r.said)-1].Said()
if last.Change != cc.ChangeSilenced || last.Silenced == nil || last.Subject == nil || last.Subject.ID != "g14" || last.Show == "" {
t.Errorf("the event the seat hears: %+v", last)
}
}