package view import ( "encoding/json" "errors" "strings" "testing" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate" "git.novox.be/novox/mesh-sdk/go/keys" ) const ( fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111" fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222" ) func wantE(t *testing.T, node, module string, w meshwantsclient.Want) Entry { t.Helper() key, err := meshwantsclient.WantKey(node, module) if err != nil { t.Fatal(err) } if !w.Removed { w.Composed = "sha256:" + strings.Repeat("c", 64) } v, err := meshwantsclient.EncodeWant(w) if err != nil { t.Fatal(err) } return Entry{Bucket: AssignerWantsBucket, Key: key, Value: v, At: t0} } func haveFP(t *testing.T, node, module, fp string, health string) Entry { t.Helper() b, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: health}) if err != nil { t.Fatal(err) } key, err := nodeengineclient.HaveKey(node, module) if err != nil { t.Fatal(err) } return Entry{Bucket: HaveBucket, Key: key, Value: b, At: t0} } func moduleE(t *testing.T, module, fp string) Entry { t.Helper() v, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/" + module, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) if err != nil { t.Fatal(err) } return Entry{Bucket: ModulesBucket, Key: module, Value: v, At: t0, Branch: testBranch} } // testBranch is the watched branch of these tests: no branch is named in the view's code (design 47 G §532, A-173). const testBranch = "release" func sourceE(t *testing.T, revision uint64) Entry { return sourceOn(t, testBranch, revision) } // sourceOn is one watched branch's source as the gate's verb source answers it (A-173). func sourceOn(t *testing.T, branch string, revision uint64) Entry { t.Helper() v, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{Group: "g", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) if err != nil { t.Fatal(err) } return Entry{Bucket: SourceBucket, Key: keys.EscapeName(branch), Value: v, At: t0, Revision: revision} } // sourceRowOf is the snapshot's source row of one branch, nil when it has none. func sourceRowOf(m *Model, branch string) *SourceRow { for _, r := range m.Snapshot().Sources { if r.Branch == branch { return &r } } return nil } // 1906/D7 AC7: every want is a row beside the have of its node, with where it stands. func TestEveryWantIsARowAgainstItsHave(t *testing.T) { m := New(viewstate.ViewModule) fill(m, wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK), wantE(t, "ace", "mail", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "ace", "mail", fpOld, nodeengineclient.HealthOK), wantE(t, "g14", "web", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "g14", "web", fpNew, nodeengineclient.HealthFailed), wantE(t, "g14", "mail", meshwantsclient.Want{Fp: fpOld, Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace does not run it healthy yet"}), wantE(t, "g14", "old", meshwantsclient.Want{Removed: true}), wantE(t, "novox", "web", meshwantsclient.Want{Fp: fpNew}), ) got := map[string]WantRow{} for _, r := range m.Snapshot().Wants { got[r.Node+"."+r.Module] = r } for key, state := range map[string]string{ "ace.web": WantApplied, "ace.mail": WantPending, "g14.web": WantFailed, "g14.mail": WantHeldBack, "g14.old": WantRemoved, "novox.web": WantPending, } { if got[key].State != state { t.Errorf("%s = %+v, want state %s", key, got[key], state) } } if r := got["g14.mail"]; r.Rule != meshwantsclient.RuleCanary || r.Newest != fpNew || r.Words == "" { t.Errorf("the held-back row keeps its rule and words: %+v", r) } if r := got["ace.mail"]; r.Want != fpNew || r.Have != fpOld { t.Errorf("a pending row shows both: %+v", r) } // a want taken away leaves the list fill(m, Entry{Bucket: AssignerWantsBucket, Key: "want.novox.web", Deleted: true}) if n := len(m.Snapshot().Wants); n != 5 { t.Errorf("wants = %d after a delete, want 5", n) } } // 1906/D7 AC7: a source revision is delivered when every want equals its module's fingerprint and every have equals its // want, healthy; one failed have shows it not delivered and names the module. func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) { m := New(viewstate.ViewModule) var said []DeliveredEvent m.SetDeliveredEvent(func(e DeliveredEvent) error { if e.Repository == "" { said = append(said, e) } return nil }) fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), moduleE(t, "mail", fpNew), place(t, "ace", "web"), place(t, "g14", "web"), wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK), wantE(t, "g14", "web", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "g14", "web", fpNew, nodeengineclient.HealthOK), wantE(t, "ace", "mail", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "ace", "mail", fpNew, nodeengineclient.HealthOK)) s := sourceRowOf(m, testBranch) if s == nil || s.Revision != 14355 || s.State != SourceDelivered || len(s.Waiting) != 0 { t.Fatalf("source = %+v, want revision 14355 delivered", s) } // design 47 G A-173: each watched branch's source has its own row and its own delivered, over the modules whose branch it // is: a module of another branch not yet run holds that branch's revision alone. cms := moduleE(t, "cms", fpNew) cms.Branch = "beta" fill(m, sourceOn(t, "beta", 14360), cms, wantE(t, "ace", "cms", meshwantsclient.Want{Fp: fpNew})) if b := sourceRowOf(m, "beta"); b == nil || b.Revision != 14360 || b.State != SourceNotDelivered || b.Wants != 1 { t.Errorf("the branch beta's source = %+v, want revision 14360 not delivered over its one want", b) } if s := sourceRowOf(m, testBranch); s == nil || s.State != SourceDelivered || s.Wants != 3 { t.Errorf("the branch %s's source = %+v, want delivered over its own three wants", testBranch, s) } fill(m, haveFP(t, "ace", "cms", fpNew, nodeengineclient.HealthOK)) if len(said) != 2 || said[0].Branch != testBranch || said[0].Revision != 14355 || said[1].Branch != "beta" || said[1].Revision != 14360 { t.Errorf("delivered said = %+v, want %s at 14355, then beta at 14360", said, testBranch) } fill(m, haveFP(t, "g14", "web", fpNew, nodeengineclient.HealthFailed)) s = sourceRowOf(m, testBranch) if s.State != SourceNotDelivered || len(s.Waiting) != 1 || !strings.Contains(s.Waiting[0], "web") || !strings.Contains(s.Waiting[0], "g14") { t.Errorf("a failed have: %+v", s) } fill(m, haveFP(t, "g14", "web", fpNew, nodeengineclient.HealthOK), wantE(t, "ace", "mail", meshwantsclient.Want{Fp: fpOld})) s = sourceRowOf(m, testBranch) if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "mail") { t.Errorf("a want that is not the module's fingerprint: %+v", s) } fill(m, wantE(t, "ace", "mail", meshwantsclient.Want{Fp: fpNew}), place(t, "novox", "web")) s = sourceRowOf(m, testBranch) if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "novox") { t.Errorf("a node assigned the module with no want: %+v", s) } // ADR 0603 §7: a node away holds nothing back and is named; it counts once it returns and runs its wants. fill(m, presenceE(t, "novox", false, t0)) s = sourceRowOf(m, testBranch) if s.State != SourceDelivered || len(s.Away) != 1 || s.Away[0] != "novox" { t.Errorf("a node away: %+v", s) } fill(m, presenceE(t, "novox", true, t0)) if s = sourceRowOf(m, testBranch); s.State != SourceNotDelivered || len(s.Away) != 0 { t.Errorf("the node returned and runs nothing yet: %+v", s) } } func TestNoSourceRevisionHeldMeansNoStateIsDerived(t *testing.T) { if s := New(viewstate.ViewModule).Snapshot().Sources; len(s) != 0 { t.Errorf("source = %+v", s) } } func TestTheWantsPageSaysWantsAgainstHaves(t *testing.T) { m := New(viewstate.ViewModule) fill(m, wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "ace", "web", fpOld, nodeengineclient.HealthOK)) text := wantText(m.Snapshot()) if len(text) != 1 || text[0][0] != "ace" || text[0][1] != "web" || text[0][2] != fpNew || text[0][3] != fpOld || text[0][4] != WantPending || text[0][5] != "none" { t.Errorf("text = %v", text) } } // 1906/D7 (ADR 0539 §14): the controller's judgement of the move a want names is shown beside it, with its words. func TestTheJudgementOfTheMoveIsShownBesideTheWant(t *testing.T) { // 1955/L7 (ADR 0571 §6): the judgement is the node agent's, in its have waiting for a yes; the controller's composed copy // is not read. m := New(viewstate.ViewModule) h, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK, Judgement: nodeengineclient.JudgedCannot, Waits: "sha256:" + strings.Repeat("c", 64), Detail: "the composition it runs is gone"}) if err != nil { t.Fatal(err) } fill(m, wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew}), Entry{Bucket: HaveBucket, Key: "ace.web", Value: h, At: t0}) rows := m.Snapshot().Wants if len(rows) != 1 || rows[0].Judgement != string(nodeengineclient.JudgedCannot) || !strings.Contains(rows[0].Judged, "gone") { t.Fatalf("rows = %+v", rows) } text := wantText(m.Snapshot()) if len(text[0]) != 7 || text[0][5] != "cannot-judge" { t.Errorf("text = %v", text) } } // 1906/D9 (asked by the lead for issue 1946): the view says, once for each source revision, that it is delivered: the event the // issue tracker derives "fix delivered" from, in place of the gate's states that go in release 6 (ADR 0539 §17). func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) { m := New(viewstate.ViewModule) var said []DeliveredEvent refuse := true // the bus refuses the first try to say it (issue 1126): it is said at the next change, not lost m.SetDeliveredEvent(func(e DeliveredEvent) error { if refuse && e.Repository == "" { refuse = false return errors.New("nats: no responders available for request") } said = append(said, e) return nil }) fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), place(t, "ace", "web"), wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew})) if len(said) != 0 { t.Fatalf("said %v before any have", said) } fill(m, haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthFailed)) if len(said) != 0 { t.Fatalf("said %v for a failed have", said) } fill(m, haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK)) if len(said) != 0 || refuse { t.Fatalf("said = %+v though the bus refused it", said) } fill(m, wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew})) if len(said) != 1 || said[0].Revision != 14355 || said[0].Group != "g" || said[0].Heads["novox/web"] == "" { t.Fatalf("said = %+v, want revision 14355 once with its group and heads", said) } fill(m, haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK), wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew})) if len(said) != 1 { t.Errorf("said again for the same revision: %v", said) } // a newer revision is delivered when its wants and haves are; a revision that goes bad and good again is not said twice fill(m, sourceE(t, 14356)) if len(said) != 2 || said[1].Revision != 14356 { t.Errorf("the next revision: %v", said) } fill(m, haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthFailed), haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK)) if len(said) != 2 { t.Errorf("said again after a have failed and recovered: %v", said) } } // Review round 1 (view notes): the controller no longer writes the build of an assignment (C6), so a row reads the want and the // have: the row shows the fingerprint wanted and waits while the node runs another. func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) { m := New(viewstate.ViewModule) fill(m, place(t, "ace", "web"), wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "ace", "web", fpOld, nodeengineclient.HealthOK)) r := find(t, m.Snapshot(), "ace", "web") if r.Assigned != fpNew || r.Have != fpOld || !strings.Contains(r.Waits, "has not applied") { t.Fatalf("row = %+v, want it to wait for the want's fingerprint", r) } fill(m, haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK)) if r := find(t, m.Snapshot(), "ace", "web"); r.Waits != "" || r.Have != fpNew { t.Errorf("row = %+v, want no wait once the have names the want's fingerprint", r) } } // Review round 1 (view notes, issue 1150): a module whose name holds a dot has its have under the escaped key; the row finds it, // and a have nothing is assigned to keeps the module's name. func TestADottedModuleNameIsFoundAndKept(t *testing.T) { m := New(viewstate.ViewModule) // The seat's assignments refuses a dotted module name (mesh-sdk meshmoduleassigner.Assignments), so such a module is never placed: // its have is a row left behind under its own name. fill(m, place(t, "ace"), wantE(t, "ace", "novox.be", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "ace", "novox.be", fpNew, nodeengineclient.HealthOK), haveFP(t, "ace", "old.name", fpNew, nodeengineclient.HealthOK)) s := m.Snapshot() if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Status != StatusLeftBehind { t.Errorf("the dotted module's row = %+v", r) } if r := find(t, s, "ace", "old.name"); r.Status != StatusLeftBehind { t.Errorf("the left-behind dotted module = %+v", r) } if rows := s.Wants; len(rows) != 1 || rows[0].Module != "novox.be" || rows[0].State != WantApplied { t.Errorf("wants = %+v", rows) } } // 1906/D10 (asked by the lead for the messenger, ADR 0308): with no plans to say that a module is in motion, the view says it: `moving` // when a module's want on a node differs from its have, `settled` when the have reaches the want running and healthy, `stuck` when it // is held or failed or the watchman says it stopped moving; each once for a node, a module and a composition. func motionLog(m *Model) *[]MotionEvent { var said []MotionEvent m.SetMotionEvent(func(e MotionEvent) error { said = append(said, e); return nil }) for _, b := range []string{AssignerWantsBucket, HaveBucket, StuckBucket} { m.Initial(b) } return &said } func types(said []MotionEvent) string { var out []string for _, e := range said { out = append(out, e.Type) } return strings.Join(out, ",") } func TestAModuleInMotionIsSaidMovingThenSettledOnceEach(t *testing.T) { m := New(viewstate.ViewModule) said := motionLog(m) fill(m, haveFP(t, "ace", "web", fpOld, nodeengineclient.HealthOK)) if len(*said) != 0 { t.Fatalf("said %v for a have with no want", types(*said)) } fill(m, wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew})) if types(*said) != "moving" { t.Fatalf("said %q, want moving", types(*said)) } mv := (*said)[0] if mv.Body.Node != "ace" || mv.Body.Module != "web" || mv.Body.FromFP != fpOld || mv.Body.ToFP != fpNew || mv.Body.Composed != "sha256:"+strings.Repeat("c", 64) || mv.Body.Since.IsZero() || mv.Body.At.IsZero() || mv.Body.BoundSeconds != 1800 || mv.ID != "moving.ace.web."+strings.Repeat("c", 64) { t.Errorf("moving = %+v", mv) } fill(m, haveFP(t, "ace", "web", fpOld, nodeengineclient.HealthOK), wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew})) if types(*said) != "moving" { t.Errorf("said %q after the same state again", types(*said)) } // 1126: the bus refuses the settled once; it is said at the next change of the have, not lost. refused := false m.SetMotionEvent(func(e MotionEvent) error { if !refused { refused = true return errors.New("nats: no responders available for request") } *said = append(*said, e) return nil }) fill(m, haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK)) if types(*said) != "moving" || !refused { t.Errorf("said %q though the bus refused it", types(*said)) } fill(m, haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK)) if types(*said) != "moving,settled" || (*said)[1].Body.Outcome != "running" { t.Errorf("said %q, want settled with its outcome", types(*said)) } fill(m, haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK)) if types(*said) != "moving,settled" { t.Errorf("settled said twice: %q", types(*said)) } } func TestAModuleHeldOrFailedIsSaidStuckWithItsWords(t *testing.T) { m := New(viewstate.ViewModule) said := motionLog(m) fill(m, wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew})) b, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"}) fill(m, Entry{Bucket: HaveBucket, Key: "ace.web", Value: b, At: t0}) if types(*said) != "moving,stuck" || (*said)[1].Body.Outcome != "stopped" || (*said)[1].Body.Words != "no space left" { t.Fatalf("said %v: %+v", types(*said), *said) } fill(m, Entry{Bucket: HaveBucket, Key: "ace.web", Value: b, At: t0}) if types(*said) != "moving,stuck" { t.Errorf("stuck said twice: %q", types(*said)) } // failed on the wanted fingerprint m2 := New(viewstate.ViewModule) said2 := motionLog(m2) fill(m2, wantE(t, "g14", "web", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "g14", "web", fpNew, nodeengineclient.HealthFailed)) if types(*said2) != "moving,stuck" || (*said2)[1].Body.Outcome != "failed" { t.Errorf("failed: said %q", types(*said2)) } } func TestAWantTheWatchmanSaysStoppedMovingIsSaidStuckPastItsBound(t *testing.T) { m := New(viewstate.ViewModule) said := motionLog(m) fill(m, wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew}), haveFP(t, "ace", "web", fpOld, nodeengineclient.HealthOK)) sk, _ := meshwatchdogsclient.StuckKey(AssignerWantsBucket, "want.ace.web") v, err := meshwatchdogsclient.EncodeStuck(meshwatchdogsclient.Stuck{Bucket: AssignerWantsBucket, Key: "want.ace.web", Waited: "the node agent of ace to run web", Since: t0}) if err != nil { t.Fatal(err) } fill(m, Entry{Bucket: StuckBucket, Key: sk, Value: v, At: t0}) if types(*said) != "moving,stuck" || (*said)[1].Body.Outcome != "stopped" { t.Errorf("said %q: %+v", types(*said), *said) } } func TestANodeThatSettlesWithoutBeingSeenMovingSaysNothing(t *testing.T) { m := New(viewstate.ViewModule) said := motionLog(m) fill(m, haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK), wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew})) if len(*said) != 0 { t.Errorf("said %v for a module that never moved", types(*said)) } } // Nothing is said of a module in motion before the buckets have been read: a want read ahead of its have is not a node behind. What is // in motion when they have been read is said then, and a module that settles after is said settled. func TestMotionIsSaidOnlyOnceTheBucketsHaveBeenRead(t *testing.T) { m := New(viewstate.ViewModule) var said []MotionEvent m.SetMotionEvent(func(e MotionEvent) error { said = append(said, e); return nil }) fill(m, wantE(t, "ace", "web", meshwantsclient.Want{Fp: fpNew}), wantE(t, "ace", "mail", meshwantsclient.Want{Fp: fpNew})) m.Initial(AssignerWantsBucket) fill(m, haveFP(t, "ace", "web", fpNew, nodeengineclient.HealthOK), haveFP(t, "ace", "mail", fpOld, nodeengineclient.HealthOK)) m.Initial(HaveBucket) if len(said) != 0 { t.Fatalf("said %v before the stuck keys were read", types(said)) } m.Initial(StuckBucket) if len(said) != 1 || said[0].Type != MotionMoving || said[0].Body.Module != "mail" { t.Fatalf("said %v: %+v, want only mail moving (web is already there)", types(said), said) } fill(m, haveFP(t, "ace", "mail", fpNew, nodeengineclient.HealthOK)) if types(said) != "moving,settled" { t.Errorf("said %q", types(said)) } } // assigner_wants lists a node's own want (want..node, ADR 0539 clause 6) among the wants, and never as unreadable: // on 2026-10-25 every node's own want was listed unreadable, though its value was what the assigner writes (issue 1945). func TestTheWantsListANodesOwnWant(t *testing.T) { own, err := meshwantsclient.WantNodeKey("ace") if err != nil { t.Fatal(err) } value, err := meshwantsclient.EncodeNodeWant(meshwantsclient.NodeWant{Composed: "sha256:" + strings.Repeat("d", 64)}) if err != nil { t.Fatal(err) } moduleKey, _ := meshwantsclient.WantKey("ace", "web") moduleValue, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpNew, Composed: "sha256:" + strings.Repeat("c", 64)}) state := map[string][]byte{own: value, moduleKey: moduleValue} r := Reader{WantsState: StateRead{ Keys: func() ([]string, error) { return []string{own, moduleKey}, nil }, Get: func(k string) (json.RawMessage, error) { return state[k], nil }, }} got, err := r.Wants("", "") if err != nil { t.Fatal(err) } if len(got.Unreadable) != 0 || got.Wants[own] == nil || got.Wants[moduleKey] == nil { t.Fatalf("the wants: %v, unreadable %v", got.Wants, got.Unreadable) } if got, _ := r.Wants("g14", ""); len(got.Wants) != 0 { t.Errorf("another node's wants: %v", got.Wants) } if got, _ := r.Wants("ace", "web"); got.Wants[own] != nil { t.Errorf("the node's own want was listed under the module web") } }