Put a broken module back at once, and excuse a wait only for a move that added an account group (hq issue 318 review)

cmd/mesh-controller/gate_followup_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -277,3 +278,146 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
 		t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
 	}
 }
+
+// **A broken module is put back at once** (issue 318 review): the laptop's witness put the node tools back
+// within a minute of a send that also moved `app`, and `app` reads not yet healthy because of it. The node
+// tools are marked and put back in the very judging that found them broken — their registered build is the
+// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
+func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	withConditionsInMemory(t)
+	was := doctorFrom
+	doctorFrom = nil
+	t.Cleanup(func() { doctorFrom = was })
+	old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
+	build := func(module, commit string, asked time.Time) {
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
+		if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
+			Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
+			Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + commit}}}); err != nil {
+			t.Fatal(err)
+		}
+		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
+			Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
+			Asked: asked}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, m := range []string{"app", broker_runtime} {
+		build(m, "c1", old)
+		if _, err := inv.Assign(ctx, "laptop", m); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop", map[string]string{"app": "c1", broker_runtime: "c1"}); err != nil {
+		t.Fatal(err)
+	}
+	build("app", "c2", recent)
+	build(broker_runtime, "c2", recent)
+
+	wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
+	t.Cleanup(func() {
+		machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
+	})
+	machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
+		modules, _ := open.inventory.Assigned(ctx, node)
+		sent, known, err := open.inventory.SentBuilds(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		return f.moves(node, modules, sent, known, all), nil
+	}
+	releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil }
+	var sends []string
+	n := 0
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		current, err := open.inventory.CurrentBuilds(ctx)
+		if err != nil {
+			return nil, err
+		}
+		for _, node := range names {
+			n++
+			carried := map[string]string{"app": current["app"].Commit, broker_runtime: current[broker_runtime].Commit}
+			sends = append(sends, node+":"+carried[broker_runtime])
+			digest := "d-" + node + "-" + time.Now().Format("150405.000000") + string(rune('a'+n))
+			if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
+				return nil, err
+			}
+			if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
+				Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
+				return nil, err
+			}
+		}
+		return names, nil
+	}
+	var seen []string // the node tools' registered build at each judging
+	gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+		now := time.Now()
+		f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
+			rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
+		reports, _ := open.inventory.LastReports(ctx)
+		for _, r := range reports {
+			f.reports[r.Node] = r
+		}
+		current, _ := open.inventory.CurrentBuilds(ctx)
+		seen = append(seen, current[broker_runtime].Commit)
+		app := healthyContainer("app")
+		if current[broker_runtime].Commit == "c2" {
+			// The witness reverted the node tools; app cannot reach them yet.
+			f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
+				From: "c2", To: "c1", At: now, Why: "the node tools did not answer"}}
+			app.State, app.Reason = link.StateUnhealthy, "down"
+		}
+		f.served["laptop"] = served{runtime: current[broker_runtime].Commit == "c1", tools: map[string]bool{}}
+		f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
+		return f, nil
+	}
+	withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
+
+	release := func() inventory.Plan {
+		t.Helper()
+		plans, _ := inv.RecentPlans(ctx, 10)
+		for _, p := range plans {
+			if p.Release != nil {
+				return p
+			}
+		}
+		t.Fatal("no walk")
+		return inventory.Plan{}
+	}
+	// Judged until the first judging has found the node tools broken, and one more.
+	for i := 0; i < 20 && len(seen) < 2; i++ {
+		advancePlans(ctx, open)
+	}
+	if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" {
+		t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at the very next judging", seen)
+	}
+	if failed, _ := inv.GateFailed(ctx, "build-"+broker_runtime+"-c2"); !failed {
+		t.Fatal("the node tools' build is not marked failed at once")
+	}
+	if p := release(); p.State != inventory.PlanRolling || p.Release.Gate == nil || p.Release.Gate.Verdict != "" {
+		t.Fatalf("the walk is %s with gate %+v; want it still judging app", p.State, p.Release.Gate)
+	}
+	if !slices.Contains(sends, "laptop:c1") {
+		t.Fatalf("sends %v; want the laptop sent the node tools' earlier build at once", sends)
+	}
+	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
+		advancePlans(ctx, open)
+		if release().State != inventory.PlanRolling {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	p := release()
+	if p.State != inventory.PlanFailed {
+		t.Fatalf("the walk is %s: %s; want failed, for the node tools", p.State, p.Note)
+	}
+	if v, found, _ := inv.GateOf(ctx, "build-app-c2"); !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") {
+		t.Fatalf("app's verdict: %+v; want its own pass", v)
+	}
+	if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
+		t.Fatalf("app is registered at %s; want it kept at c2", current["app"].Commit)
+	}
+}