Give every module of a failed send a verdict, and excuse only the wait a build's own send brought (hq issue 318 review)

2026-10-08 15:25:58 +02:00
parent a759ac65a5
commit 363898ec8a
10 changed files with 599 additions and 87 deletions
@@ -0,0 +1,279 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a
+// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own
+// wait, and a provider waiting for a login says who waits on it.
+
+// withGateBounds sets the gate's bounds for one test.
+func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
+	t.Helper()
+	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	gateSettle, gateEvery, gateBound = settle, every, bound
+	t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+}
+
+// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
+// of its modules' resources.
+func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts {
+	return func() gateFacts {
+		now := time.Now()
+		at := now
+		return gateFacts{now: now,
+			reports:    map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}},
+			engines:    map[string]string{},
+			rolledBack: map[string][]lease.Rollback{},
+			served:     map[string]served{machine: {runtime: true, tools: map[string]bool{}}},
+			health:     map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}},
+		}
+	}
+}
+
+func healthyContainer(module string) inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
+		State: link.StateHealthy}
+}
+
+// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node
+// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
+// counted over the gate's spacing — before it says its own. The broken one alone is put back.
+func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
+	since := time.Now().Add(-time.Second)
+	base := factsOn(t, "laptop", since, healthyContainer("app"))
+	wasGather := gatherGateFacts
+	t.Cleanup(func() { gatherGateFacts = wasGather })
+	gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) {
+		f := base()
+		f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
+			At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}}
+		return f, nil
+	}
+	g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
+	pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}}
+	judgings := 0
+	for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); {
+		if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil {
+			t.Fatal(err)
+		}
+		judgings++
+		time.Sleep(30 * time.Millisecond)
+	}
+	if g.Verdict != inventory.GateFailed || judgings < gatePasses {
+		t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict,
+			judgings, g)
+	}
+	if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) {
+		t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing)
+	}
+	if !strings.Contains(g.Why, "witness") {
+		t.Errorf("the verdict does not say what broke: %q", g.Why)
+	}
+}
+
+// broker_runtime is the node tools' module name, a core component a witness judges.
+const broker_runtime = "node-tools"
+
+// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
+// while a module beside it is not yet healthy counts the healthy one once.
+func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	withGateBounds(t, 0, time.Hour, time.Hour)
+	since := time.Now().Add(-time.Minute)
+	base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
+		Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
+	wasGather := gatherGateFacts
+	t.Cleanup(func() { gatherGateFacts = wasGather })
+	gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil }
+	g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
+	pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}}
+	now := time.Now()
+	for i := 0; i < 3; i++ {
+		if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 {
+		t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy)
+	}
+}
+
+// **A carried build's pass carries its own wait, never another's** (review (e)).
+func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	since := time.Now().Add(-time.Minute)
+	g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed,
+		Why:   "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}),
+		Waits: map[string]string{"late": "relogin needed on laptop: late waits"},
+		Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"},
+			{Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}}
+	passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "")
+	app, _, _ := inv.GateOf(ctx, "build-app-c2")
+	late, _, _ := inv.GateOf(ctx, "build-late-c2")
+	if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed {
+		t.Errorf("app's pass: %+v; want it without late's wait", app)
+	}
+	if !strings.Contains(late.Why, "relogin needed on laptop") {
+		t.Errorf("late's pass: %+v; want its own wait", late)
+	}
+}
+
+// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send
+// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
+// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
+func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
+	g := aGateMesh(t)
+	ctx := t.Context()
+	inv := g.open.inventory
+	// `late` waits on anchor for a gate: c1 sent, c2 registered and built.
+	for _, b := range []inventory.Build{
+		{ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late",
+			Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
+		{ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late",
+			Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
+	} {
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit})
+		b.Manifest = manifest
+		b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}}
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{
+			Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit,
+			Asked: b.Asked}); err != nil {
+			t.Fatal(err)
+		}
+		if b.Commit == "l1" {
+			if _, err := inv.Assign(ctx, "anchor", "late"); err != nil {
+				t.Fatal(err)
+			}
+			if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1",
+				"late": "l1"}); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	wasMoves := machineMoves
+	t.Cleanup(func() { machineMoves = wasMoves })
+	machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
+		modules, err := open.inventory.Assigned(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		sent, known, err := open.inventory.SentBuilds(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		return f.moves(node, modules, sent, known, all), nil
+	}
+	gather := gatherGateFacts
+	gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+		f, err := gather(ctx, open, component)
+		f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
+			healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late",
+				State: link.StateUnhealthy, Reason: "down"}}}}
+		return f, err
+	}
+	gateEvery, gateBound = 0, 400*time.Millisecond
+	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
+		advancePlans(ctx, g.open)
+		if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone {
+			break
+		}
+		time.Sleep(30 * time.Millisecond)
+	}
+	p := g.plan(t)
+	if p.State != inventory.PlanFailed {
+		t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note)
+	}
+	if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
+		!strings.Contains(v.Why, "on its own") {
+		t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
+	}
+	if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed {
+		t.Fatal("late's build is not marked failed at its gate")
+	}
+	if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" {
+		t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit,
+			current["late"].Commit)
+	}
+}
+
+// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
+// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
+func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	k := conditionsFrom
+	register(t, open, catalogue.Manifest{Module: "db", Version: "1",
+		Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
+	register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
+	for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
+		if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
+		Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
+		t.Fatal(err)
+	}
+	at := h0
+	say := func(machine string, rs ...link.ResourceHealth) {
+		t.Helper()
+		at = at.Add(time.Second)
+		for i := range rs {
+			rs[i].Since = at
+		}
+		if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+	account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator",
+		Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"}
+	unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service",
+		Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
+	shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop",
+		State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
+	for look := 0; look < 2; look++ {
+		say("anchor", account, unit)
+		say("laptop", shop)
+	}
+	list, err := k.Open(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var keys []string
+	var c conditions.Condition
+	for _, x := range list {
+		keys = append(keys, x.Key)
+		if x.Key == "module.db.anchor.relogin-needed" {
+			c = x
+		}
+	}
+	if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) {
+		t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys)
+	}
+	if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") {
+		t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
+	}
+}