Reconcile a module from the machine's facts at engine start and at each edge of a declared-running resource going down (issue 1957, ADR 0568)

A reboot left the have running, so a unit that failed to start at boot was never started again and nothing said so. Baseline now asks the machine; the judge's look calls Agent.Reconcile once per down edge, leaving held and unwanted modules alone.

internal/agent/agent_test.go

@@ -342,6 +342,7 @@ func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) {
 
 // What the node-engine itself last applied is what runs after a restart, not the kept declaration.
 func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
+	t.Run("1957: an engine start reads the machine's facts", testAnEngineStartReconcilesAModuleFromTheMachinesFacts)
 	t.Run("says the have again", testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines)
 	// An apply that failed is not what runs, though its have names the want's build and digest: the agent starting on
 	// it applies again (review of the switch, round 1).
@@ -431,6 +432,7 @@ func testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines(t *testing
 // A module whose kept resources differ from the composition is applied, and a failed apply is retried only by a change
 // of either key.
 func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
+	t.Run("1957: the judge's edge reconciles, and a failed re-apply is failed", testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone)
 	// A real failure is not retried by the clock, however short the backoff: only by a change of either key.
 	t.Run("a real failure", func(t *testing.T) {
 		b := newBed(t)
@@ -1233,3 +1235,122 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 			r.Phase == string(delivery.PhaseRunning) && r.FP == fpB && r.Composed == rdig
 	})
 }
+
+// 1957/AC1 and AC4 (unit; ADR 0568 §1, §4): an engine start takes a module from what the machine shows, not only from the
+// have or the kept declaration it kept. Every resource up applies nothing; one declared running and not active applies the
+// module once, from a have or from the kept declaration; a module with no want is not looked at.
+func testAnEngineStartReconcilesAModuleFromTheMachinesFacts(t *testing.T) {
+	for _, c := range []struct {
+		name     string
+		have     bool // the engine's own have says running at the want
+		held     bool // else its have says held, from a failed backup
+		kept     bool // else the kept declaration says it runs
+		inactive bool
+		unwanted bool
+		applied  int
+		asked    bool // the machine was looked at
+	}{
+		{"all up: nothing is applied", true, false, false, false, false, 0, true},
+		{"a unit not active: applied once", true, false, false, true, false, 1, true},
+		{"kept declaration, a unit not active: applied once", false, false, true, true, false, 1, true},
+		{"kept declaration, all up: nothing is applied", false, false, true, false, false, 0, true},
+		{"a held module applies nothing at start", false, true, false, true, false, 0, false},
+		{"no want: nothing is looked at or applied", true, false, false, true, true, 0, false},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			b := newBed(t)
+			fp := fpOf("a")
+			rs := []json.RawMessage{file("web", "x", "1")}
+			ck, craw, dig := composition(t, "web", fp, false, rs...)
+			wk, wraw := wantFor(t, "web", fp, dig, "")
+			b.comps.before(ck, craw)
+			if !c.unwanted {
+				b.wants.before(wk, wraw)
+			}
+			if c.have {
+				b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: "running"}}
+			}
+			if c.held {
+				b.haves.stored = map[string]Have{"web": {Module: "web", FP: fpOf("0"), Composed: "sha256:" + strings.Repeat("0", 64),
+					Phase: string(delivery.PhaseHeld), Detail: "restic: the repository is locked"}}
+			}
+			if c.kept {
+				b.kept = map[string][]json.RawMessage{"web": rs}
+			}
+			var mu sync.Mutex
+			asked := false
+			b.env.Inactive = func(context.Context, delivery.Composed) bool {
+				mu.Lock()
+				defer mu.Unlock()
+				asked = true
+				return c.inactive
+			}
+			stop := b.run()
+			defer stop()
+			if !c.unwanted && !c.held {
+				eventually(t, "the module was not settled", func() bool { return b.haves.last("web").Composed == dig })
+			}
+			time.Sleep(50 * time.Millisecond)
+			if got := b.appliedList(); len(got) != c.applied {
+				t.Fatalf("applied %v, want %d", got, c.applied)
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			if asked != c.asked {
+				t.Fatalf("the machine was asked: %v, want %v", asked, c.asked)
+			}
+		})
+	}
+}
+
+// 1957/AC2, AC3 and AC4 (unit; ADR 0568 §2, §3, §4): the judge's edge, Reconcile, applies a running module again; one held
+// for a failed backup and one with no want are left alone; and an apply that fails writes the have `failed` with the
+// apply's own words.
+func testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone(t *testing.T) {
+	b := newBed(t)
+	fp := fpOf("a")
+	ckW, crawW, digW := composition(t, "web", fp, false, file("web", "x", "1"))
+	wkW, wrawW := wantFor(t, "web", fp, digW, "")
+	ckD, crawD, digD := composition(t, "db", fp, true, file("db", "x", "1"))
+	wkD, wrawD := wantFor(t, "db", fp, digD, "")
+	b.comps.before(ckW, crawW)
+	b.wants.before(wkW, wrawW)
+	b.comps.before(ckD, crawD)
+	b.wants.before(wkD, wrawD)
+	b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: digW, Phase: "running"},
+		"db": {Module: "db", FP: fpOf("0"), Composed: "sha256:" + strings.Repeat("0", 64), Phase: "running"}}
+	b.backup = func(string) Backup { return Backup{Words: "restic: the repository is locked"} }
+	stop := b.run()
+	defer stop()
+	eventually(t, "web was not settled", func() bool { return b.haves.last("web").Composed == digW })
+	eventually(t, "db was not held", func() bool { return b.haves.last("db").Phase == string(delivery.PhaseHeld) })
+	if got := b.appliedList(); len(got) != 0 {
+		t.Fatalf("applied at the start: %v", got)
+	}
+
+	b.ag.Reconcile("db", "ghost")
+	time.Sleep(50 * time.Millisecond)
+	if got := b.appliedList(); len(got) != 0 {
+		t.Fatalf("a held module and an unwanted one were applied: %v", got)
+	}
+	if h := b.haves.last("db"); h.Phase != string(delivery.PhaseHeld) {
+		t.Fatalf("the held module's have changed: %+v", h)
+	}
+
+	b.ag.Reconcile("web")
+	eventually(t, "web was not applied again", func() bool { return len(b.appliedList()) == 1 })
+	eventually(t, "web is not running again", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
+
+	b.mu.Lock()
+	b.failWith = "systemctl start web.service: exit status 1"
+	b.mu.Unlock()
+	b.ag.Reconcile("web")
+	eventually(t, "the failed re-apply did not write the have failed with its words", func() bool {
+		h := b.haves.last("web")
+		return h.Phase == string(delivery.PhaseFailed) && h.Detail == "systemctl start web.service: exit status 1"
+	})
+	time.Sleep(50 * time.Millisecond)
+	if got := b.appliedList(); len(got) != 2 {
+		t.Fatalf("a failed apply was tried again with no edge: %v", got)
+	}
+}