ace's node-engine wrote seat-list-unread in its state findings, and no condition rose: the keeper's observers did not name the node-engine. It now reads mesh-host.findings, so ADR 0636 §1's condition is raised and clears at the first full read (issue 1966). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
421 lines
17 KiB
Go
421 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
|
seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
|
meshhealers "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhealers"
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
)
|
|
|
|
// fakeStore is the bus's states in memory, with revisions, as the runtime answers them.
|
|
type fakeStore struct {
|
|
mu sync.Mutex
|
|
vals map[string][]byte
|
|
revs map[string]uint64
|
|
next uint64
|
|
// ttls are the times to live the keys were created with.
|
|
ttls map[string]time.Duration
|
|
}
|
|
|
|
func newFakeStore() *fakeStore {
|
|
return &fakeStore{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
|
}
|
|
|
|
func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
v, ok := f.vals[state+"/"+key]
|
|
return v, f.revs[state+"/"+key], ok, nil
|
|
}
|
|
|
|
func (f *fakeStore) put(state, key string, value any) uint64 {
|
|
raw, _ := json.Marshal(value)
|
|
f.next++
|
|
f.vals[state+"/"+key], f.revs[state+"/"+key] = raw, f.next
|
|
return f.next
|
|
}
|
|
|
|
func (f *fakeStore) Create(state, key string, value any) (uint64, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if _, ok := f.vals[state+"/"+key]; ok {
|
|
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
|
|
}
|
|
return f.put(state, key, value), nil
|
|
}
|
|
|
|
func (f *fakeStore) Update(state, key string, value any, rev uint64) (uint64, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.revs[state+"/"+key] != rev {
|
|
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
|
|
}
|
|
return f.put(state, key, value), nil
|
|
}
|
|
|
|
func (f *fakeStore) Delete(state, key string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
delete(f.vals, state+"/"+key)
|
|
delete(f.revs, state+"/"+key)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeStore) Keys(state string) ([]string, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
var out []string
|
|
for k := range f.vals {
|
|
if s, key, _ := strings.Cut(k, "/"); s == state {
|
|
out = append(out, key)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
|
|
f.mu.Lock()
|
|
if f.ttls == nil {
|
|
f.ttls = map[string]time.Duration{}
|
|
}
|
|
f.ttls[state+"/"+key] = ttl
|
|
f.mu.Unlock()
|
|
_, err := f.Create(state, key, value)
|
|
return err
|
|
}
|
|
|
|
// rig is a keeper over a fake store, its clock and what it said.
|
|
type rig struct {
|
|
k *Keeper
|
|
st *fakeStore
|
|
now time.Time
|
|
said []Event
|
|
}
|
|
|
|
func newRig() *rig {
|
|
r := &rig{st: newFakeStore(), now: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)}
|
|
r.k = &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e Event) error {
|
|
r.said = append(r.said, e)
|
|
return nil
|
|
}}
|
|
return r
|
|
}
|
|
|
|
func (r *rig) changes() string {
|
|
var out []string
|
|
for _, e := range r.said {
|
|
out = append(out, e.Change+" "+e.Key)
|
|
}
|
|
r.said = nil
|
|
return strings.Join(out, ", ")
|
|
}
|
|
|
|
func (r *rig) open(key string) (Condition, bool) {
|
|
raw, _, ok, _ := r.st.Get(ConditionsState, key)
|
|
var c Condition
|
|
if ok {
|
|
// The seat's layout, read strictly: a field the seat does not name is a fault of the keeper's write.
|
|
v, err := seat.DecodeConditionsByKeyValue(raw)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("the condition %s is not the seat's layout: %v: %s", key, err, raw))
|
|
}
|
|
c = conditionOf(v)
|
|
}
|
|
return c, ok
|
|
}
|
|
|
|
// encodeFinding writes a finding as an observer does: the seat's layout, checked.
|
|
func encodeFinding(f seat.FindingsByKeyValue) ([]byte, error) {
|
|
f.V = cc.FindingVersion
|
|
if f.Evidence == nil {
|
|
f.Evidence = []seat.Evidence{}
|
|
}
|
|
if err := cc.CheckFinding(f); err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(f)
|
|
}
|
|
|
|
// finding is an observer's finding as mesh-sdk's layout writes it (encodeFinding).
|
|
func finding(t *testing.T, observer, scope, id, kind, source string, sev string, at time.Time) (string, []byte) {
|
|
t.Helper()
|
|
f := seat.FindingsByKeyValue{Key: cc.Key(scope, id, kind), Kind: kind, Subject: seat.Subject{Scope: scope, ID: id},
|
|
Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source,
|
|
Observer: observer, At: at, Evidence: []seat.Evidence{{At: at, Said: kind}}}
|
|
raw, err := encodeFinding(f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return f.Key, raw
|
|
}
|
|
|
|
// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer
|
|
// clears nothing; the healers' attempts reach the condition; a reopening within ten
|
|
// minutes is the same fault; a silence is a transition in history.
|
|
func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) {
|
|
r := newRig()
|
|
key, raw := finding(t, watchman, cc.ScopeMachine, "g14", "silent", "S1", cc.Warning, r.now)
|
|
steps := []struct {
|
|
name string
|
|
do func() error
|
|
said string
|
|
check func(c Condition, open bool) string
|
|
}{
|
|
{name: "a finding written raises", do: func() error { return r.k.Finding(watchman, key, raw, false) },
|
|
said: "raised machine.g14.silent", check: func(c Condition, open bool) string {
|
|
if !open || c.Severity != cc.Warning || c.Resolver != cc.ResolverSelf || c.Epoch != 0 || c.Count != 1 {
|
|
return fmt.Sprintf("%+v", c)
|
|
}
|
|
return ""
|
|
}},
|
|
{name: "the same finding again says nothing", do: func() error { return r.k.Finding(watchman, key, raw, false) }},
|
|
{name: "an observer gone blind clears nothing", do: func() error { r.k.Blind(watchman, "refused"); _, err := r.k.Resync(); return err },
|
|
check: func(c Condition, open bool) string {
|
|
if !open {
|
|
return "cleared by blindness"
|
|
}
|
|
return ""
|
|
}},
|
|
{name: "a healer's attempt reaches tried and the resolver", do: func() error {
|
|
a, _ := json.Marshal(meshhealers.AttemptsByAttemptValue{Healer: "H3", Condition: key, Act: "asserted", Outcome: meshhealers.AttemptsByAttemptValueOutcomeActed, Begun: r.now})
|
|
return r.k.Attempt("h3-1", a, false)
|
|
}, said: "resolver machine.g14.silent", check: func(c Condition, _ bool) string {
|
|
if c.Resolver != "healer:H3" || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" {
|
|
return fmt.Sprintf("%+v", c)
|
|
}
|
|
return ""
|
|
}},
|
|
{name: "an escalation makes it urgent, the operator's", do: func() error {
|
|
a, _ := json.Marshal(meshhealers.AttemptsByAttemptValue{Healer: "H3", Condition: key, Act: "handed over", Outcome: meshhealers.AttemptsByAttemptValueOutcomeEscalated,
|
|
Begun: r.now.Add(time.Minute)})
|
|
return r.k.Attempt("h3-2", a, false)
|
|
}, said: "severity machine.g14.silent, resolver machine.g14.silent", check: func(c Condition, _ bool) string {
|
|
if c.Severity != cc.Urgent || c.Resolver != cc.ResolverOperator || len(c.Tried) != 2 {
|
|
return fmt.Sprintf("%+v", c)
|
|
}
|
|
return ""
|
|
}},
|
|
{name: "a silence is a transition with who and why", do: func() error {
|
|
_, err := r.k.Silence(key, time.Hour, "jochen", "known, a cable")
|
|
return err
|
|
}, said: "silenced machine.g14.silent"},
|
|
{name: "the finding deleted clears", do: func() error { return r.k.Finding(watchman, key, nil, true) },
|
|
said: "cleared machine.g14.silent", check: func(_ Condition, open bool) string {
|
|
if open {
|
|
return "still open"
|
|
}
|
|
return ""
|
|
}},
|
|
{name: "raised again within ten minutes is a reopening", do: func() error {
|
|
r.now = r.now.Add(5 * time.Minute)
|
|
return r.k.Finding(watchman, key, raw, false)
|
|
}, said: "reopened machine.g14.silent", check: func(c Condition, _ bool) string {
|
|
if c.Count != 2 || len(c.Gaps) != 1 {
|
|
return fmt.Sprintf("%+v", c)
|
|
}
|
|
return ""
|
|
}},
|
|
}
|
|
for _, s := range steps {
|
|
if err := s.do(); err != nil {
|
|
t.Fatalf("%s: %v", s.name, err)
|
|
}
|
|
if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" {
|
|
t.Errorf("%s: said %q, want %q", s.name, got, s.said)
|
|
}
|
|
if s.check != nil {
|
|
c, open := r.open(key)
|
|
if why := s.check(c, open); why != "" {
|
|
t.Errorf("%s: %s", s.name, why)
|
|
}
|
|
}
|
|
}
|
|
// Every transition is kept in history, and each event carries the keeper's show.
|
|
keys, _ := r.st.Keys(HistoryState)
|
|
if len(keys) < 7 {
|
|
t.Errorf("history keeps %d transitions", len(keys))
|
|
}
|
|
raw0, _, _, _ := r.st.Get(HistoryState, keys[0])
|
|
if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) {
|
|
t.Errorf("the first transition is %s", raw0)
|
|
}
|
|
}
|
|
|
|
// 1126/A5/AC2: the keeper carries no controller condition: the controller's bucket, as Controller hands it, opens no
|
|
// condition and changes none (no escalation, no silence). What it carried while the controller ran, still open, is cleared
|
|
// once by EndCarried, said as cleared; a condition an observer's finding holds stays; a second EndCarried clears nothing.
|
|
func TestTheKeeperCarriesNoControllerConditionAndClearsWhatItCarriedOnce(t *testing.T) {
|
|
r := newRig()
|
|
key, raw := finding(t, watchman, cc.ScopeMachine, "g14", "silent", "S1", cc.Warning, r.now)
|
|
if err := r.k.Finding(watchman, key, raw, false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.said = nil
|
|
// Carried while the controller ran: two of kinds that ended with it, still open in conditions.
|
|
for _, c := range []Condition{
|
|
{Key: "core.controller.deaf", Kind: "controller-deaf", Severity: cc.Urgent, Resolver: "self", Count: 1},
|
|
{Key: "seat.vault.ace.holder-silent", Kind: "holder-silent", Severity: cc.Warning, Resolver: "self", Count: 1},
|
|
} {
|
|
if _, err := r.st.Create(ConditionsState, c.Key, c.Stored()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
n, err := r.k.EndCarried()
|
|
if err != nil || n != 2 || r.changes() != "cleared core.controller.deaf, cleared seat.vault.ace.holder-silent" {
|
|
t.Fatalf("EndCarried cleared %d (%v)", n, err)
|
|
}
|
|
if _, open := r.open(key); !open {
|
|
t.Error("a condition a finding holds was cleared")
|
|
}
|
|
if n, err := r.k.EndCarried(); err != nil || n != 0 || r.changes() != "" {
|
|
t.Errorf("a second EndCarried cleared %d (%v)", n, err)
|
|
}
|
|
}
|
|
|
|
// ADR 0564 (issue 1952 C5; the lead 2026-10-29): every kind the controller raised has its home in the table, and the
|
|
// kinds that ended with it are marked carried, which EndCarried clears once (issue 1126). compare, take-over and
|
|
// hand-back went with the controller (ADR 0590): there is no other side to compare and nothing to hand back to.
|
|
func TestEveryControllerKindHasItsHome(t *testing.T) {
|
|
// The lead's table of 2026-10-29: each kind has its home, lease-split and image-missing do not end, and the 22
|
|
// that end are carried; with the 11 kinds of the probes the operator dropped on 2026-10-30, 33.
|
|
carried := 0
|
|
for _, h := range Homes {
|
|
if h.Carried {
|
|
carried++
|
|
}
|
|
}
|
|
for kind, observer := range map[string]string{"needs-operator": watchman, "machine-network": watchman, "lease-split": selfCheck,
|
|
"data-shrank": selfCheck, "retire-waiting": provisioner, "asks-undelivered": messenger,
|
|
"image-missing": artifactStore, "build-failed": builder, "floor-silent": builder, "old-form-held": builder,
|
|
"tag-removal-not-done": builder, "seat-definition-unparsed": indexer, "generated-types-out-of-step": indexer,
|
|
"seat-defined-twice": indexer, "accepted-unsaid": indexer, "manifest-held": indexer, "packages-token-refused": gitea,
|
|
"publish-refused": builder, "publish-unanswered": builder, "seat-outcome-unread": builder} {
|
|
if h, ok := Homes[kind]; !ok || h.Observer != observer || h.Carried {
|
|
t.Errorf("%s's home is %+v", kind, h)
|
|
}
|
|
// The keeper reads every observer's findings that a kind's home names (the registry milestone's join).
|
|
if !slices.Contains(Observers, observer) {
|
|
t.Errorf("%s's observer %s is not read by the keeper", kind, observer)
|
|
}
|
|
}
|
|
for _, kind := range []string{"resolver-wrong", "holder-silent", "archives-missing", "archives-unheld", "binding-kept",
|
|
"binding-moved", "binding-moving", "grant-composes-nothing", "root-not-free", "cleanup-waiting", "data-held-twice"} {
|
|
if h := Homes[kind]; !h.Carried || h.Observer != "" || h.Why != Dropped {
|
|
t.Errorf("the dropped probe's kind %s is %+v", kind, h)
|
|
}
|
|
}
|
|
// And the controller's own bus upgrade, carried (the operator, 2026-10-31): 34; and the bus-consumer probe's three
|
|
// kinds, ended with the controller (issue 1126): 37.
|
|
for _, kind := range []string{"consumer-wrong", "consumer-lost", "consumer-behind"} {
|
|
if h := Homes[kind]; !h.Carried || h.Observer != "" || h.Why != EndedD6 {
|
|
t.Errorf("the ended probe's kind %s is %+v", kind, h)
|
|
}
|
|
}
|
|
// The readers' unanswered first view and the owners' views too large to say are every reader's and owner's own.
|
|
for _, kind := range []string{"first-view-unanswered", "view-too-large", "view-unsaid", "family-not-heard"} {
|
|
if h := Homes[kind]; h.Observer != "" || h.Carried || h.Controller {
|
|
t.Errorf("%s's home is %+v", kind, h)
|
|
}
|
|
}
|
|
// Each module that writes a reader's unanswered first view (ADR 0615 §6), or that a delivery part's consumer stopped
|
|
// hearing, is read: the delivery parts and the tool runner by name (other repositories), and every catalogue module
|
|
// whose code writes a views finding, found in the catalogue's tree, so a new writer cannot go unobserved.
|
|
// And the node-engine, whose node writes seat-list-unread when it has read no seat list (ADR 0636 §1, issue 1966).
|
|
readers := []string{watchman, "mesh-delivery-teller", enrolment, builder, "mesh-delivery-checker", "mesh-delivery-view",
|
|
indexer, assignerModule, configModule, vaultModule, gitea, "node-tools", "mesh-host"}
|
|
readers = append(readers, viewFindingWriters(t, "../../..")...)
|
|
for _, reader := range readers {
|
|
if !slices.Contains(Observers, reader) {
|
|
t.Errorf("the reader %s's findings are not read by the keeper", reader)
|
|
}
|
|
}
|
|
// 19 of ADR 0564, the 63 of the table, plan-failed and stalled, mesh-config's four (design 32), the readers' unanswered
|
|
// first view, the owners' view too large, the builder's four, the indexer's four and the package seats' token refused.
|
|
// And the builder's three publishing kinds and the delivery parts' family not heard: 103.
|
|
// And the owners' changes not said, view-unsaid: 104. And the indexer's mesh-wide hold past its bound, manifest-held
|
|
// (ADR 0632 §2): 105. (The bus module's own conditions, nats.status-unwritten among them, are said to the keeper
|
|
// directly, never through findings, so they have no home here.)
|
|
if len(Homes) != 105 || carried != 37 {
|
|
t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried)
|
|
}
|
|
}
|
|
|
|
// viewFindingWriters are the catalogue modules (under modules) whose non-test Go code writes a views finding into its own
|
|
// findings: views.UnansweredFinding, views.TooLargeFinding, views.UnsaidFinding or views.OwnFindings.
|
|
func viewFindingWriters(t *testing.T, modules string) []string {
|
|
t.Helper()
|
|
writes := regexp.MustCompile(`(views\.(UnansweredFinding|TooLargeFinding|UnsaidFinding)|meshconditionsclient\.FindingWriter)\b`)
|
|
var out []string
|
|
entries, err := os.ReadDir(modules)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, e := range entries {
|
|
if !e.IsDir() || e.Name() == Module {
|
|
continue
|
|
}
|
|
found := false
|
|
_ = filepath.WalkDir(filepath.Join(modules, e.Name()), func(path string, d fs.DirEntry, err error) error {
|
|
if err != nil || found || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
|
return nil
|
|
}
|
|
if src, err := os.ReadFile(path); err == nil && writes.Match(src) {
|
|
found = true
|
|
}
|
|
return nil
|
|
})
|
|
if found {
|
|
out = append(out, e.Name())
|
|
}
|
|
}
|
|
if len(out) < 5 {
|
|
t.Fatalf("found %d catalogue modules writing views findings in %s: the tree was not read (%v)", len(out), modules, out)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ADR 0634 §1, §4 (issue 1966): the keeper reads a condition written before the seat named its layout (the controller's
|
|
// body: `silenced` null, `actions` empty, the times with an offset) and writes it back in the seat's layout.
|
|
func TestTheKeeperReadsTheOldLayoutAndWritesTheSeatsLayout(t *testing.T) {
|
|
r := newRig()
|
|
key, raw := finding(t, watchman, cc.ScopeMachine, "g14", "silent", "S1", cc.Warning, r.now)
|
|
old := `{"key":"` + key + `","kind":"silent","subject":{"scope":"machine","id":"g14"},"severity":"warning",` +
|
|
`"summary":"old","headline":"","explanation":"","resolved":"","needs":"","actions":[],"evidence":[],"source":"S1",` +
|
|
`"raised":"2026-10-29T13:00:00+01:00","last-observed":"2026-10-29T13:00:00+01:00","observations":1,"count":1,` +
|
|
`"resolver":"self","silenced":null,"epoch":0}`
|
|
if _, err := r.st.Create(ConditionsState, key, json.RawMessage(old)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.k.Finding(watchman, key, raw, false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c, ok := r.open(key)
|
|
if !ok || c.Summary != "silent of g14" || !c.Raised.Equal(r.now) || c.Count != 1 {
|
|
t.Fatalf("the old condition, settled: %+v (open %v)", c, ok)
|
|
}
|
|
if _, err := r.k.Silence(key, time.Hour, "tester", "known"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if c, _ := r.open(key); c.Silenced == nil || !c.Silenced.Until.Equal(r.now.Add(time.Hour)) {
|
|
t.Errorf("the silence is not kept in the seat's layout: %+v", c.Silenced)
|
|
}
|
|
last := r.said[len(r.said)-1].Said()
|
|
if last.Change != cc.ChangeSilenced || last.Silenced == nil || last.Subject == nil || last.Subject.ID != "g14" || last.Show == "" {
|
|
t.Errorf("the event the seat hears: %+v", last)
|
|
}
|
|
}
|