342 lines
13 KiB
Go
342 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
)
|
|
|
|
// fakeStore is the bus's states in memory, with revisions, as the runtime answers them.
|
|
type fakeStore struct {
|
|
mu sync.Mutex
|
|
vals map[string][]byte
|
|
revs map[string]uint64
|
|
next uint64
|
|
// ttls are the times to live the keys were created with.
|
|
ttls map[string]time.Duration
|
|
}
|
|
|
|
func newFakeStore() *fakeStore {
|
|
return &fakeStore{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
|
}
|
|
|
|
func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
v, ok := f.vals[state+"/"+key]
|
|
return v, f.revs[state+"/"+key], ok, nil
|
|
}
|
|
|
|
func (f *fakeStore) put(state, key string, value any) uint64 {
|
|
raw, _ := json.Marshal(value)
|
|
f.next++
|
|
f.vals[state+"/"+key], f.revs[state+"/"+key] = raw, f.next
|
|
return f.next
|
|
}
|
|
|
|
func (f *fakeStore) Create(state, key string, value any) (uint64, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if _, ok := f.vals[state+"/"+key]; ok {
|
|
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
|
|
}
|
|
return f.put(state, key, value), nil
|
|
}
|
|
|
|
func (f *fakeStore) Update(state, key string, value any, rev uint64) (uint64, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.revs[state+"/"+key] != rev {
|
|
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
|
|
}
|
|
return f.put(state, key, value), nil
|
|
}
|
|
|
|
func (f *fakeStore) Delete(state, key string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
delete(f.vals, state+"/"+key)
|
|
delete(f.revs, state+"/"+key)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeStore) Keys(state string) ([]string, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
var out []string
|
|
for k := range f.vals {
|
|
if s, key, _ := strings.Cut(k, "/"); s == state {
|
|
out = append(out, key)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
|
|
f.mu.Lock()
|
|
if f.ttls == nil {
|
|
f.ttls = map[string]time.Duration{}
|
|
}
|
|
f.ttls[state+"/"+key] = ttl
|
|
f.mu.Unlock()
|
|
_, err := f.Create(state, key, value)
|
|
return err
|
|
}
|
|
|
|
// rig is a keeper over a fake store, its clock and what it said.
|
|
type rig struct {
|
|
k *Keeper
|
|
st *fakeStore
|
|
now time.Time
|
|
said []health.Event
|
|
}
|
|
|
|
func newRig() *rig {
|
|
r := &rig{st: newFakeStore(), now: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)}
|
|
r.k = &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e health.Event) error {
|
|
r.said = append(r.said, e)
|
|
return nil
|
|
}}
|
|
return r
|
|
}
|
|
|
|
func (r *rig) changes() string {
|
|
var out []string
|
|
for _, e := range r.said {
|
|
out = append(out, e.Change+" "+e.Key)
|
|
}
|
|
r.said = nil
|
|
return strings.Join(out, ", ")
|
|
}
|
|
|
|
func (r *rig) open(key string) (health.Condition, bool) {
|
|
raw, _, ok, _ := r.st.Get(ConditionsState, key)
|
|
var c health.Condition
|
|
if ok {
|
|
_ = json.Unmarshal(raw, &c)
|
|
}
|
|
return c, ok
|
|
}
|
|
|
|
// finding is an observer's finding as mesh-sdk's layout writes it (health.EncodeFinding).
|
|
func finding(t *testing.T, observer, scope, id, kind, source string, sev health.Severity, at time.Time) (string, []byte) {
|
|
t.Helper()
|
|
f := health.Finding{Key: health.Key(scope, id, kind), Kind: kind, Subject: health.Subject{Scope: scope, ID: id},
|
|
Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source,
|
|
Observer: observer, At: at, Evidence: []health.Evidence{{At: at, Said: kind}}}
|
|
raw, err := health.EncodeFinding(f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return f.Key, raw
|
|
}
|
|
|
|
// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer
|
|
// clears nothing; the healers' attempts and the controller's escalation reach the condition; a reopening within ten
|
|
// minutes is the same fault; a silence is a transition in history.
|
|
func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) {
|
|
r := newRig()
|
|
key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now)
|
|
steps := []struct {
|
|
name string
|
|
do func() error
|
|
said string
|
|
check func(c health.Condition, open bool) string
|
|
}{
|
|
{name: "a finding written raises", do: func() error { return r.k.Finding(watchman, key, raw, false) },
|
|
said: "raised machine.g14.silent", check: func(c health.Condition, open bool) string {
|
|
if !open || c.Severity != health.Warning || c.Resolver != health.ResolverSelf || c.Epoch != 0 || c.Count != 1 {
|
|
return fmt.Sprintf("%+v", c)
|
|
}
|
|
return ""
|
|
}},
|
|
{name: "the same finding again says nothing", do: func() error { return r.k.Finding(watchman, key, raw, false) }},
|
|
{name: "an observer gone blind clears nothing", do: func() error { r.k.Blind(watchman, "refused"); _, err := r.k.Resync(); return err },
|
|
check: func(c health.Condition, open bool) string {
|
|
if !open {
|
|
return "cleared by blindness"
|
|
}
|
|
return ""
|
|
}},
|
|
{name: "a healer's attempt reaches tried and the resolver", do: func() error {
|
|
a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "asserted", Outcome: health.OutcomeActed, Begun: r.now})
|
|
return r.k.Attempt("h3-1", a, false)
|
|
}, said: "resolver machine.g14.silent", check: func(c health.Condition, _ bool) string {
|
|
if c.Resolver != "healer:H3" || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" {
|
|
return fmt.Sprintf("%+v", c)
|
|
}
|
|
return ""
|
|
}},
|
|
{name: "an escalation makes it urgent, the operator's", do: func() error {
|
|
a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "handed over", Outcome: health.OutcomeEscalated,
|
|
Begun: r.now.Add(time.Minute)})
|
|
return r.k.Attempt("h3-2", a, false)
|
|
}, said: "severity machine.g14.silent, resolver machine.g14.silent", check: func(c health.Condition, _ bool) string {
|
|
if c.Severity != health.Urgent || c.Resolver != health.ResolverOperator || len(c.Tried) != 2 {
|
|
return fmt.Sprintf("%+v", c)
|
|
}
|
|
return ""
|
|
}},
|
|
{name: "a silence is a transition with who and why", do: func() error {
|
|
_, err := r.k.Silence(key, time.Hour, "jochen", "known, a cable")
|
|
return err
|
|
}, said: "silenced machine.g14.silent"},
|
|
{name: "the finding deleted clears", do: func() error { return r.k.Finding(watchman, key, nil, true) },
|
|
said: "cleared machine.g14.silent", check: func(_ health.Condition, open bool) string {
|
|
if open {
|
|
return "still open"
|
|
}
|
|
return ""
|
|
}},
|
|
{name: "raised again within ten minutes is a reopening", do: func() error {
|
|
r.now = r.now.Add(5 * time.Minute)
|
|
return r.k.Finding(watchman, key, raw, false)
|
|
}, said: "reopened machine.g14.silent", check: func(c health.Condition, _ bool) string {
|
|
if c.Count != 2 || len(c.Gaps) != 1 {
|
|
return fmt.Sprintf("%+v", c)
|
|
}
|
|
return ""
|
|
}},
|
|
// Until the lease write the controller's escalation for the same key reaches the condition (clause 8).
|
|
{name: "the controller's escalation is carried", do: func() error {
|
|
ctrl, _ := json.Marshal(health.Condition{Key: key, Kind: "silent", Severity: health.Urgent, Resolver: health.ResolverOperator,
|
|
Tried: []health.Tried{{At: r.now, What: "x", Outcome: "escalated", By: "healer H1"}}})
|
|
_ = r.k.Attempt("h3-1", nil, true)
|
|
_ = r.k.Attempt("h3-2", nil, true)
|
|
return r.k.Controller(key, ctrl, false)
|
|
}, check: func(c health.Condition, _ bool) string {
|
|
if c.Resolver != health.ResolverOperator || c.Severity != health.Urgent {
|
|
return fmt.Sprintf("%+v", c)
|
|
}
|
|
return ""
|
|
}},
|
|
}
|
|
for _, s := range steps {
|
|
if err := s.do(); err != nil {
|
|
t.Fatalf("%s: %v", s.name, err)
|
|
}
|
|
if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" && s.name != "the controller's escalation is carried" {
|
|
t.Errorf("%s: said %q, want %q", s.name, got, s.said)
|
|
}
|
|
if s.check != nil {
|
|
c, open := r.open(key)
|
|
if why := s.check(c, open); why != "" {
|
|
t.Errorf("%s: %s", s.name, why)
|
|
}
|
|
}
|
|
}
|
|
// Every transition is kept in history, and each event carries the keeper's show.
|
|
keys, _ := r.st.Keys(HistoryState)
|
|
if len(keys) < 7 {
|
|
t.Errorf("history keeps %d transitions", len(keys))
|
|
}
|
|
raw0, _, _, _ := r.st.Get(HistoryState, keys[0])
|
|
if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) {
|
|
t.Errorf("the first transition is %s", raw0)
|
|
}
|
|
// A carried kind (it ends with the controller) is raised and cleared as the controller's bucket says, until the
|
|
// lease is handed away, with the controller's show (clause 9, Limits 9).
|
|
deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4",
|
|
Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self", Epoch: 9})
|
|
if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if c, open := r.open("core.controller.deaf"); !open || c.Epoch != 0 || r.said[0].Show != "mesh-controller.conditions key=core.controller.deaf" {
|
|
t.Errorf("a carried kind is %+v, said %+v", c, r.said)
|
|
}
|
|
r.said = nil
|
|
if err := r.k.Lease(true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, open := r.open("core.controller.deaf"); open || r.changes() != "cleared core.controller.deaf" {
|
|
t.Error("a carried kind outlived the lease write")
|
|
}
|
|
}
|
|
|
|
// ADR 0564 clause 9 (issue 1952 C5, C8; the lead 2026-10-29): compare counts only what both sides can raise. A key held
|
|
// while its module is delivered, a kind that ends with the controller and a kind the controller never raises are listed
|
|
// apart; a kind with no home decided is a difference naming the gap; take-over is refused while compare differs, and
|
|
// hand-back after the lease write.
|
|
func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) {
|
|
r := newRig()
|
|
put := func(key, kind string) []byte {
|
|
raw, _ := json.Marshal(health.Condition{Key: key, Kind: kind, Severity: health.Warning, Source: "x"})
|
|
return raw
|
|
}
|
|
key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now)
|
|
_ = r.k.Finding(watchman, key, raw, false)
|
|
_ = r.k.Controller(key, put(key, "silent"), false)
|
|
only, onlyRaw := finding(t, delivery.AssignerModule, health.ScopeSeat, "mesh-placement.ace", "states-unbound", "say", health.Warning, r.now)
|
|
_ = r.k.Finding(delivery.AssignerModule, only, onlyRaw, false)
|
|
_ = r.k.Controller("core.controller.deaf", put("core.controller.deaf", "controller-deaf"), false)
|
|
_ = r.k.Controller("delivery.x.mystery", put("delivery.x.mystery", "mystery"), false)
|
|
_ = r.k.Controller("bus.x.consumer-wrong", put("bus.x.consumer-wrong", "consumer-wrong"), false)
|
|
_ = r.k.Controller("seat.vault.ace.holder-silent", put("seat.vault.ace.holder-silent", "holder-silent"), false)
|
|
cmp, err := r.k.Compare()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reasons := map[string]string{}
|
|
for _, d := range append(append([]Difference(nil), cmp.Differences...), cmp.Apart...) {
|
|
reasons[d.Key] = d.Reason
|
|
}
|
|
for key, says := range map[string]string{
|
|
only: "keeper only",
|
|
"core.controller.deaf": "carried",
|
|
"delivery.x.mystery": "no home is decided for the kind mystery",
|
|
"bus.x.consumer-wrong": "mesh-self-check has not written its finding",
|
|
// A kind of a probe the operator dropped (2026-10-30) is carried, never missing.
|
|
"seat.vault.ace.holder-silent": "carried from its bucket until the lease write (dropped 2026-10-30 by the operator",
|
|
} {
|
|
if !strings.Contains(reasons[key], says) {
|
|
t.Errorf("%s: %q, want it to say %q", key, reasons[key], says)
|
|
}
|
|
}
|
|
if cmp.Same != 1 || len(cmp.Differences) != 2 {
|
|
t.Errorf("compare: %+v", cmp)
|
|
}
|
|
// The lead's table of 2026-10-29: each kind has its home, lease-split and image-missing do not end, and the 22
|
|
// that end are carried; with the 11 kinds of the probes the operator dropped on 2026-10-30, 33.
|
|
carried := 0
|
|
for _, h := range Homes {
|
|
if h.Carried {
|
|
carried++
|
|
}
|
|
}
|
|
for kind, observer := range map[string]string{"needs-operator": watchman, "machine-network": watchman, "lease-split": selfCheck,
|
|
"data-shrank": selfCheck, "bus-upgrade-failed": assigner, "retire-waiting": provisioner, "asks-undelivered": messenger,
|
|
"image-missing": artifactStore} {
|
|
if h, ok := Homes[kind]; !ok || h.Observer != observer || h.Carried {
|
|
t.Errorf("%s's home is %+v", kind, h)
|
|
}
|
|
}
|
|
for _, kind := range []string{"resolver-wrong", "holder-silent", "archives-missing", "archives-unheld", "binding-kept",
|
|
"binding-moved", "binding-moving", "grant-composes-nothing", "root-not-free", "cleanup-waiting", "data-held-twice"} {
|
|
if h := Homes[kind]; !h.Carried || h.Observer != "" || h.Why != Dropped {
|
|
t.Errorf("the dropped probe's kind %s is %+v", kind, h)
|
|
}
|
|
}
|
|
if len(Homes) != 84 || carried != 33 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled
|
|
t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried)
|
|
}
|
|
if _, err := r.k.TakeOver(nil, "jochen", "now"); err == nil || !strings.Contains(err.Error(), "2 difference") {
|
|
t.Errorf("take-over while compare differs: %v", err)
|
|
}
|
|
if _, err := r.k.HandBack("jochen", "back"); err != nil {
|
|
t.Errorf("hand-back before the lease write: %v", err)
|
|
}
|
|
_ = r.k.Lease(true)
|
|
if _, err := r.k.HandBack("jochen", "back"); err == nil {
|
|
t.Error("hand-back after the lease write was taken")
|
|
}
|
|
}
|