334 lines
14 KiB
Go
334 lines
14 KiB
Go
package view
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
)
|
|
|
|
var t0 = time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)
|
|
|
|
func have(t *testing.T, build string, phase delivery.Phase, health delivery.Health, detail string) []byte {
|
|
t.Helper()
|
|
b, err := delivery.EncodeHave(delivery.Have{Build: build, Phase: phase, Health: health, Detail: detail})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
func haveE(t *testing.T, node, module string, v []byte, at time.Time) Entry {
|
|
return Entry{Bucket: delivery.HaveBucket, Key: node + "." + module, Value: v, At: at}
|
|
}
|
|
|
|
// place is the seat's placement of node, as the assigner keeps it (ADR 0567): the modules placed there.
|
|
func place(t *testing.T, node string, modules ...string) Entry {
|
|
t.Helper()
|
|
p := delivery.Placement{Node: node}
|
|
for _, m := range modules {
|
|
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
|
}
|
|
key, err := delivery.PlacementKey(node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v, err := delivery.EncodePlacement(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return Entry{Bucket: delivery.PlacementBucket, Key: key, Value: v, At: t0}
|
|
}
|
|
|
|
func find(t *testing.T, s Snapshot, node, module string) Row {
|
|
t.Helper()
|
|
for _, r := range s.Rows {
|
|
if r.Node == node && r.Module == module {
|
|
return r
|
|
}
|
|
}
|
|
t.Fatalf("no row for %s.%s in %+v", node, module, s.Rows)
|
|
return Row{}
|
|
}
|
|
|
|
func fill(m *Model, es ...Entry) {
|
|
for _, e := range es {
|
|
m.Apply(e)
|
|
}
|
|
}
|
|
|
|
// base: two nodes (a, b), three modules (x, y, z).
|
|
func base(t *testing.T) []Entry {
|
|
es := []Entry{
|
|
place(t, "a", "x", "y"), place(t, "b", "x", "z"),
|
|
haveE(t, "a", "x", have(t, "b1", delivery.PhaseRunning, delivery.HealthOK, ""), t0.Add(1*time.Minute)),
|
|
haveE(t, "b", "x", have(t, "b1", delivery.PhaseChecking, delivery.HealthWaits, "starting"), t0.Add(2*time.Minute)),
|
|
haveE(t, "b", "z", have(t, "b3", delivery.PhaseRunning, delivery.HealthOK, ""), t0.Add(3*time.Minute)),
|
|
}
|
|
return es
|
|
}
|
|
|
|
func TestRowsPerAssignedModuleAndNode(t *testing.T) { // 1134/AC1
|
|
m := New()
|
|
fill(m, base(t)...)
|
|
s := m.Snapshot()
|
|
if len(s.Rows) != 4 || s.Counted() != 4 {
|
|
t.Fatalf("want 4 rows, got %d counted %d: %+v", len(s.Rows), s.Counted(), s.Rows)
|
|
}
|
|
r := find(t, s, "b", "x")
|
|
want := Row{Node: "b", Module: "x", Status: StatusAssigned, Have: "b1", Phase: "checking", Health: "waits",
|
|
Since: t0.Add(2 * time.Minute), Detail: "starting"}
|
|
if !reflect.DeepEqual(r, want) {
|
|
t.Fatalf("row = %+v, want %+v", r, want)
|
|
}
|
|
y := find(t, s, "a", "y")
|
|
if y.Waits != "no have yet" || y.Have != "" {
|
|
t.Fatalf("a.y = %+v, want 'no have yet'", y)
|
|
}
|
|
}
|
|
|
|
func TestLeftBehindIsWhatThePlacementDoesNotPut(t *testing.T) { // 1134/AC3, issue 774 (the seat's placement)
|
|
m := New()
|
|
fill(m, base(t)...)
|
|
fill(m, place(t, "b", "x")) // z taken off b
|
|
s := m.Snapshot()
|
|
if r := find(t, s, "b", "z"); r.Status != StatusLeftBehind || s.Counted() != 3 {
|
|
t.Errorf("status = %q counted %d, want %q and 3", r.Status, s.Counted(), StatusLeftBehind)
|
|
}
|
|
m = New()
|
|
fill(m, base(t)...)
|
|
fill(m, haveE(t, "a", "w", have(t, "b9", delivery.PhaseRunning, delivery.HealthOK, ""), t0)) // placed nowhere
|
|
s = m.Snapshot()
|
|
if find(t, s, "a", "w").Status != StatusLeftBehind || s.Counted() != 4 {
|
|
t.Fatalf("a have with no placement: %+v counted %d", find(t, s, "a", "w"), s.Counted())
|
|
}
|
|
fill(m, Entry{Bucket: delivery.PlacementBucket, Key: "placement.b", Deleted: true})
|
|
if s := m.Snapshot(); s.Counted() != 2 || find(t, s, "b", "x").Status != StatusLeftBehind {
|
|
t.Fatalf("b's placement deleted: counted %d, %+v", s.Counted(), s.Rows)
|
|
}
|
|
}
|
|
|
|
func TestUnknownVersionSaysSo(t *testing.T) { // 1134/AC4
|
|
m := New()
|
|
fill(m, base(t)...)
|
|
before := m.Snapshot()
|
|
fill(m, haveE(t, "a", "x", []byte(`{"v":9,"phase":"running","health":"ok"}`), t0))
|
|
s := m.Snapshot()
|
|
r := find(t, s, "a", "x")
|
|
_, sdkErr := delivery.DecodeHave([]byte(`{"v":9,"phase":"running","health":"ok"}`))
|
|
if !errors.Is(sdkErr, delivery.ErrUnknownVersion) {
|
|
t.Fatal("test premise: the SDK must refuse version 9")
|
|
}
|
|
if r.Error != sdkErr.Error() || !strings.Contains(r.Error, "version 9") {
|
|
t.Fatalf("error = %q, want the SDK's %q", r.Error, sdkErr)
|
|
}
|
|
if r.Have != "" || r.Phase != "" || r.Health != "" {
|
|
t.Fatalf("an unreadable have must not be guessed: %+v", r)
|
|
}
|
|
for _, n := range [][2]string{{"a", "y"}, {"b", "x"}, {"b", "z"}} {
|
|
if !reflect.DeepEqual(find(t, s, n[0], n[1]), find(t, before, n[0], n[1])) {
|
|
t.Errorf("row %s.%s changed", n[0], n[1])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOnlyTheAffectedRowChanges(t *testing.T) { // 1134/AC5
|
|
m := New()
|
|
fill(m, base(t)...)
|
|
before := m.Snapshot()
|
|
fill(m, haveE(t, "b", "x", have(t, "b1", delivery.PhaseRunning, delivery.HealthOK, ""), t0.Add(9*time.Minute)))
|
|
after := m.Snapshot()
|
|
for i := range before.Rows {
|
|
changed := !reflect.DeepEqual(before.Rows[i], after.Rows[i])
|
|
if changed != (before.Rows[i].Node == "b" && before.Rows[i].Module == "x") {
|
|
t.Errorf("row %s.%s changed = %v", before.Rows[i].Node, before.Rows[i].Module, changed)
|
|
}
|
|
}
|
|
if r := find(t, after, "b", "x"); r.Phase != "running" || r.Health != "ok" || !r.Since.Equal(t0.Add(9*time.Minute)) {
|
|
t.Fatalf("changed row = %+v", r)
|
|
}
|
|
// a want's change touches only its row
|
|
fill(m, wantE(t, "a", "y", delivery.Want{FP: fpNew}))
|
|
if r := find(t, m.Snapshot(), "a", "y"); r.Assigned != fpNew {
|
|
t.Fatalf("want change not seen: %+v", r)
|
|
}
|
|
if !reflect.DeepEqual(find(t, m.Snapshot(), "b", "z"), find(t, after, "b", "z")) {
|
|
t.Fatal("a want of another row changed b.z")
|
|
}
|
|
// a deleted key (a bucket's expiry) removes its value
|
|
m.Apply(Entry{Bucket: delivery.HaveBucket, Key: "a.x", Deleted: true})
|
|
if !strings.HasPrefix(find(t, m.Snapshot(), "a", "x").Waits, "no have yet") {
|
|
t.Fatal("a deleted have must read as no have")
|
|
}
|
|
// a restart rebuilds the same rows from last values
|
|
a, b := New(), New()
|
|
es := base(t)
|
|
fill(a, es...)
|
|
fill(b, es...)
|
|
if !reflect.DeepEqual(a.Snapshot(), b.Snapshot()) {
|
|
t.Fatal("the same last values must give the same rows")
|
|
}
|
|
}
|
|
|
|
func TestNodeRow(t *testing.T) { // 1134/AC1 (the node's own key gives a node row)
|
|
m := New()
|
|
v, _ := delivery.EncodeHaveNode(delivery.HaveNode{Windows: []byte(`[1,2]`), Busy: []byte(`{"a":1}`), Strays: []byte(`[]`)})
|
|
fill(m, Entry{Bucket: delivery.HaveBucket, Key: "a.node", Value: v, At: t0})
|
|
n := m.Snapshot().Nodes
|
|
if len(n) != 1 || n[0].Node != "a" || n[0].Windows != 2 || n[0].Busy != 1 || n[0].Strays != 0 {
|
|
t.Fatalf("nodes = %+v", n)
|
|
}
|
|
}
|
|
|
|
func presenceE(t *testing.T, key string, online bool, at time.Time) Entry {
|
|
t.Helper()
|
|
b, err := delivery.EncodePresence(delivery.Presence{Online: online})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return Entry{Bucket: delivery.PresenceBucket, Key: key, Value: b, At: at}
|
|
}
|
|
|
|
func stuckE(t *testing.T, bucket, key, waited string, since time.Time, moved bool) Entry {
|
|
t.Helper()
|
|
b, err := delivery.EncodeStuck(delivery.Stuck{Bucket: bucket, Key: key, Waited: waited, Since: since, Moved: moved})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return Entry{Bucket: delivery.StuckBucket, Key: bucket + "." + key, Value: b, At: since}
|
|
}
|
|
|
|
func TestTheWatchmansKeysAndStatus(t *testing.T) {
|
|
t.Run("offline and stuck are listed", offlineAndStuckAreListed)
|
|
t.Run("an unknown watchman version is unreadable, not dropped", unknownWatchmanVersionIsUnreadableNotDropped)
|
|
t.Run("1952/V1: status derives each part from its bucket, and says a part it cannot read", statusDerivesEachPartFromItsBucket)
|
|
}
|
|
|
|
func offlineAndStuckAreListed(t *testing.T) { // 1168/AC2
|
|
m := New()
|
|
fill(m, base(t)...)
|
|
fill(m,
|
|
presenceE(t, "a", false, t0),
|
|
presenceE(t, "b", true, t0),
|
|
presenceE(t, "part.mesh-delivery-view.i1", false, t0.Add(time.Minute)),
|
|
stuckE(t, "node-engine_have", "a.x", "a report from a", t0.Add(2*time.Minute), false),
|
|
stuckE(t, "node-engine_have", "b.x", "a report from b", t0, true),
|
|
)
|
|
s := m.Snapshot()
|
|
want := []OfflineRow{{Kind: "node", Node: "a", Since: t0}, {Kind: "part", Part: "mesh-delivery-view", Instance: "i1", Since: t0.Add(time.Minute)}}
|
|
if !reflect.DeepEqual(s.Offline, want) {
|
|
t.Fatalf("offline = %+v, want %+v", s.Offline, want)
|
|
}
|
|
wantStuck := []StuckRow{{Bucket: "node-engine_have", Key: "a.x", Waited: "a report from a", Since: t0.Add(2 * time.Minute)}}
|
|
if !reflect.DeepEqual(s.Stuck, wantStuck) {
|
|
t.Fatalf("stuck = %+v, want %+v", s.Stuck, wantStuck)
|
|
}
|
|
// back online and moved leave the lists
|
|
fill(m, presenceE(t, "a", true, t0.Add(5*time.Minute)), presenceE(t, "part.mesh-delivery-view.i1", true, t0.Add(5*time.Minute)),
|
|
stuckE(t, "node-engine_have", "a.x", "a report from a", t0.Add(2*time.Minute), true))
|
|
s = m.Snapshot()
|
|
if len(s.Offline) != 0 || len(s.Stuck) != 0 {
|
|
t.Fatalf("offline %+v stuck %+v, want both empty", s.Offline, s.Stuck)
|
|
}
|
|
}
|
|
|
|
func unknownWatchmanVersionIsUnreadableNotDropped(t *testing.T) { // 1168/AC3
|
|
m := New()
|
|
fill(m,
|
|
Entry{Bucket: delivery.PresenceBucket, Key: "c", Value: []byte(`{"v":9,"online":false}`), At: t0},
|
|
Entry{Bucket: delivery.StuckBucket, Key: "node-engine_have.c.x", Value: []byte(`{"v":9}`), At: t0},
|
|
presenceE(t, "a", false, t0),
|
|
)
|
|
s := m.Snapshot()
|
|
if len(s.Unreadable) != 2 || len(s.Offline) != 1 {
|
|
t.Fatalf("unreadable %+v offline %+v", s.Unreadable, s.Offline)
|
|
}
|
|
_, sdkErr := delivery.DecodePresence([]byte(`{"v":9,"online":false}`))
|
|
var got *UnreadableRow
|
|
for i := range s.Unreadable {
|
|
if s.Unreadable[i].Key == "c" {
|
|
got = &s.Unreadable[i]
|
|
}
|
|
}
|
|
if got == nil || got.Bucket != delivery.PresenceBucket || got.Error != sdkErr.Error() {
|
|
t.Fatalf("unreadable = %+v, want key c with the SDK's error %q", s.Unreadable, sdkErr)
|
|
}
|
|
// a good value later replaces the unreadable one
|
|
fill(m, presenceE(t, "c", true, t0))
|
|
if len(m.Snapshot().Unreadable) != 1 {
|
|
t.Fatal("a readable value must clear the unreadable mark")
|
|
}
|
|
}
|
|
|
|
// 1952/V1 (ADR 0564 §10): status reads the keeper's open conditions, the watchman's presence and stuck keys, the wants against
|
|
// the haves and the healers' attempts of the week; nothing writes a summary, and a part it cannot read is said, never well.
|
|
func statusDerivesEachPartFromItsBucket(t *testing.T) {
|
|
now := t0.Add(time.Hour)
|
|
composedC := "sha256:" + strings.Repeat("c", 64)
|
|
fp1, fp2, fp3 := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64), delivery.SourcePrefix2+strings.Repeat("3", 64)
|
|
enc := func(v any) string { b, _ := json.Marshal(v); return string(b) }
|
|
want := func(fp string) string {
|
|
b, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: composedC})
|
|
return string(b)
|
|
}
|
|
have := func(fp string, phase delivery.Phase, health delivery.Health, detail string) string {
|
|
b, err := delivery.EncodeHave(delivery.Have{FP: fp, Composed: composedC, Phase: phase, Health: health, Detail: detail})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(b)
|
|
}
|
|
presence := func(on bool) string { b, _ := delivery.EncodePresence(delivery.Presence{Online: on}); return string(b) }
|
|
stuckOf := func(bucket, key, waited string, moved bool) string {
|
|
b, _ := delivery.EncodeStuck(delivery.Stuck{Bucket: bucket, Key: key, Waited: waited, Since: t0, Moved: moved})
|
|
return string(b)
|
|
}
|
|
stuckWant, _ := delivery.StuckKey(delivery.AssignerWantsBucket, "want.b.web")
|
|
fin := t0.Add(time.Minute)
|
|
r := StatusRead{
|
|
Conditions: states(map[string]string{
|
|
"machine.b.silent": enc(health.Condition{Key: "machine.b.silent", Severity: health.Warning, Summary: "b silent", Raised: t0}),
|
|
"core.doctor.self-check": enc(health.Condition{Key: "core.doctor.self-check", Severity: health.Urgent, Summary: "no run", Raised: t0.Add(time.Minute)}),
|
|
}),
|
|
Presence: states(map[string]string{"a": presence(true), "b": presence(false)}),
|
|
Stuck: states(map[string]string{stuckWant: stuckOf(delivery.AssignerWantsBucket, "want.b.web", "b to run web", false),
|
|
"x.moved": stuckOf("x", "moved", "y", true)}),
|
|
Wants: states(map[string]string{"want.a.web": want(fp2), "want.a.mail": want(fp1), "want.b.web": want(fp2), "want.a.db": want(fp3)}),
|
|
Haves: states(map[string]string{"a.web": have(fp1, delivery.PhaseRunning, delivery.HealthOK, ""), "a.mail": have(fp1, delivery.PhaseRunning, delivery.HealthOK, ""),
|
|
"b.web": have(fp1, delivery.PhaseRunning, delivery.HealthOK, ""), "a.db": have(fp3, delivery.PhaseRunning, delivery.HealthFailed, "the unit failed")}),
|
|
Attempts: states(map[string]string{"1": enc(health.Attempt{Outcome: health.OutcomeActed, Begun: t0, Finished: &fin}),
|
|
"2": enc(health.Attempt{Outcome: health.OutcomeEscalated, Begun: t0}), "old": enc(health.Attempt{Outcome: health.OutcomeActed, Begun: now.Add(-8 * 24 * time.Hour)})}),
|
|
}
|
|
st := DeriveStatus(r, now)
|
|
if len(st.Conditions) != 2 || st.Conditions[0].Key != "core.doctor.self-check" || st.Conditions[1].Key != "machine.b.silent" {
|
|
t.Errorf("conditions = %+v, want the urgent one first", st.Conditions)
|
|
}
|
|
if len(st.Quiet) != 2 || st.Quiet[0].Key != "b" || st.Quiet[1].Key != stuckWant {
|
|
t.Errorf("quiet = %+v", st.Quiet)
|
|
}
|
|
wantMoving := []StatusMoving{
|
|
{Node: "a", Module: "db", State: delivery.MotionStuck, Outcome: delivery.OutcomeFailed, Words: "the unit failed", FromFP: fp3, ToFP: fp3},
|
|
{Node: "a", Module: "web", State: delivery.MotionMoving, FromFP: fp1, ToFP: fp2},
|
|
{Node: "b", Module: "web", State: delivery.MotionStuck, Outcome: delivery.OutcomeStopped, Words: "b to run web", FromFP: fp1, ToFP: fp2},
|
|
}
|
|
if !reflect.DeepEqual(st.Moving, wantMoving) {
|
|
t.Errorf("moving = %+v\nwant %+v", st.Moving, wantMoving)
|
|
}
|
|
if !reflect.DeepEqual(st.Heals, map[string]int{health.OutcomeActed: 1, health.OutcomeEscalated: 1}) || st.Well {
|
|
t.Errorf("heals = %v, well %v", st.Heals, st.Well)
|
|
}
|
|
// A part it cannot read is said, and the mesh is not well; with nothing wrong and every part read it is.
|
|
if st := DeriveStatus(StatusRead{}, now); st.ConditionsUnread == "" || st.QuietUnread == "" || st.MovingUnread == "" || st.HealsUnread == "" || st.Well {
|
|
t.Errorf("nothing read = %+v", st)
|
|
}
|
|
empty := states(map[string]string{})
|
|
if st := DeriveStatus(StatusRead{Conditions: empty, Presence: empty, Stuck: empty, Wants: empty, Haves: empty, Attempts: empty}, now); !st.Well {
|
|
t.Errorf("an empty mesh = %+v, want well", st)
|
|
}
|
|
}
|