mesh-tools
97a2d841c7
The controller plans config, templates and moved providers' provisions, hands bus accounts to the vault and judges the new manifest words, on the feature's SDK and node-engine validator (feature 57, config and provisioning).
243 lines
11 KiB
Go
243 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send,
|
|
// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading.
|
|
func TestAWalkEndsOnItsMachinesReports(t *testing.T) {
|
|
now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
|
sent := now.Add(-time.Minute)
|
|
long := now.Add(-inventory.ApplySilentAfter - time.Minute)
|
|
at := now.Add(-10 * time.Second)
|
|
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
|
|
"app": {Rest: map[string]inventory.SentDeclaration{
|
|
"ace": {Digest: "d-ace", At: sent},
|
|
"novox": {Digest: "d-novox", At: sent},
|
|
"shanks": {Digest: "d-shanks", At: sent},
|
|
"laptop": {Digest: "d-laptop", At: long},
|
|
}},
|
|
"proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}},
|
|
}}
|
|
reports := []inventory.Reported{
|
|
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
|
|
{Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"},
|
|
{Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"},
|
|
}
|
|
waiting, failed, silent := walkReports(p, reports, now, nil)
|
|
if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) ||
|
|
!slices.Equal(silent, []string{"laptop"}) {
|
|
t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent)
|
|
}
|
|
reports[2].Declared = "d-shanks"
|
|
if waiting, _, _ := walkReports(p, reports, now, nil); len(waiting) != 0 {
|
|
t.Fatalf("the walk still waits once every machine reported: %v", waiting)
|
|
}
|
|
}
|
|
|
|
// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before
|
|
// novox/hq ADR 0360 started.
|
|
func withTheGate(t *testing.T) {
|
|
t.Helper()
|
|
was := gateFirstMachine
|
|
gateFirstMachine = true
|
|
t.Cleanup(func() { gateFirstMachine = was })
|
|
}
|
|
|
|
// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7);
|
|
// before, the first machine was judged before the rest were sent (ADR 0353 §1).
|
|
func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklogOnThree(t)
|
|
ctx := t.Context()
|
|
said := events(t)
|
|
for i := 0; i < 12; i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
p := b.release(t)
|
|
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
for _, n := range []string{"desk", "laptop", "anchor"} {
|
|
if !slices.Contains(*said, "send "+n) {
|
|
t.Fatalf("%s was not sent: %v", n, *said)
|
|
}
|
|
}
|
|
if slices.Contains(*said, "judge") {
|
|
t.Fatalf("a machine was judged by its health: %v", *said)
|
|
}
|
|
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
|
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
|
}
|
|
}
|
|
|
|
// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other
|
|
// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3).
|
|
func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklogOnThree(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
b.broken["desk"] = true // desk fails what it is sent
|
|
advancePlans(ctx, b.open)
|
|
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
p := b.release(t)
|
|
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") {
|
|
t.Fatalf("the others have no verdict of their own: %+v", p.Release)
|
|
}
|
|
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
|
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
|
}
|
|
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" {
|
|
t.Fatalf("desk was put back: %v", sent)
|
|
}
|
|
}
|
|
|
|
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
|
// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7).
|
|
func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
p, err := releaseBacklog(ctx, b.open, "")
|
|
if err != nil || p == nil {
|
|
t.Fatalf("no release opened: %v", err)
|
|
}
|
|
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending {
|
|
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
|
}
|
|
if err := inv.SavePlan(ctx, p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The send is made, and the controller is replaced before it keeps it.
|
|
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advancePlans(ctx, b.open)
|
|
if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 {
|
|
t.Fatalf("anchor was sent again: %v", b.sent)
|
|
}
|
|
g := b.release(t).Release.Gates["anchor"]
|
|
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
|
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
|
}
|
|
}
|
|
|
|
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
|
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7).
|
|
func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
reporting := sendRollout
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
|
b.sent = append(b.sent, names)
|
|
current, _ := open.inventory.CurrentBuilds(ctx)
|
|
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
|
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
|
}
|
|
return reporting(ctx, open, names)
|
|
}
|
|
t.Cleanup(func() { sendRollout = reporting })
|
|
for i := 0; i < 8; i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
p := b.release(t)
|
|
g := p.Release.Gates["laptop"]
|
|
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
|
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
|
}
|
|
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
|
g.Since = &since
|
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advancePlans(ctx, b.open)
|
|
p = b.release(t)
|
|
// anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7).
|
|
if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
|
t.Fatal("a silent machine's build was marked failed")
|
|
}
|
|
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
|
t.Fatalf("laptop was put back: %v", sent)
|
|
}
|
|
obs, err := silentObservation(ctx, inv)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
|
t.Fatalf("the silence is not said: %+v", obs)
|
|
}
|
|
}
|
|
|
|
// Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once
|
|
// the engine and the machine's provisioner client both said so.
|
|
func TestAWalkEndWaitsForTheProvisioner(t *testing.T) {
|
|
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
|
at := now.Add(-10 * time.Second)
|
|
plan := func(sentAt time.Time) inventory.Plan {
|
|
return inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{
|
|
"ace": {Digest: "d-ace", At: sentAt, Provisioned: map[string]string{"app": "t1"}},
|
|
"g14": {Digest: "d-g14", At: sentAt, Provisioned: map[string]string{"app": "t1"}},
|
|
}}}}
|
|
}
|
|
reports := []inventory.Reported{
|
|
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
|
|
{Node: "g14", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-g14"},
|
|
}
|
|
says := map[string]inventory.ProvReport{}
|
|
prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
|
|
applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: at}
|
|
|
|
// (a) the engine applied, the client has not said: the end waits on both.
|
|
waiting, failed, silent := walkReports(plan(now.Add(-time.Minute)), reports, now, prov)
|
|
if !slices.Equal(waiting, []string{"ace", "g14"}) || len(failed)+len(silent) != 0 {
|
|
t.Fatalf("a: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
|
}
|
|
// ace's client applied, g14's failed on that build (b): ace done, g14 a failed apply naming the reason.
|
|
says["ace/app"] = applied
|
|
says["g14/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: at}
|
|
waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov)
|
|
if len(waiting) != 0 || len(silent) != 0 || len(failed) != 1 || !strings.HasPrefix(failed[0], "g14 (") ||
|
|
!strings.Contains(failed[0], "no value") {
|
|
t.Fatalf("b: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
|
}
|
|
// (c) no client report past ApplySilentAfter: named silent, never applied.
|
|
delete(says, "g14/app")
|
|
waiting, failed, silent = walkReports(plan(now.Add(-inventory.ApplySilentAfter-time.Minute)), reports, now, prov)
|
|
if len(waiting) != 0 || len(failed) != 0 || !slices.Equal(silent, []string{"g14"}) {
|
|
t.Fatalf("c: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
|
}
|
|
// both said it: nothing waits.
|
|
says["g14/app"] = applied
|
|
if waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
|
|
t.Fatalf("both said: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
|
}
|
|
// (d) a send handing nothing to the provisioner is the engine's report alone, as before.
|
|
bare := plan(now.Add(-time.Minute))
|
|
bare.Modules["app"].Rest = map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: now.Add(-time.Minute)}}
|
|
if waiting, failed, silent = walkReports(bare, reports, now, func(string, string) (inventory.ProvReport, bool) {
|
|
return inventory.ProvReport{}, false
|
|
}); len(waiting)+len(failed)+len(silent) != 0 {
|
|
t.Fatalf("d: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
|
}
|
|
}
|