Reconcile a module from the machine's facts at engine start and at each edge of a declared-running resource going down (issue 1957, ADR 0568)

A reboot left the have running, so a unit that failed to start at boot was never started again and nothing said so. Baseline now asks the machine; the judge's look calls Agent.Reconcile once per down edge, leaving held and unwanted modules alone.

cmd/mesh-host/nodeagent_integration_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/novox/mesh-host/internal/agent"
 	"github.com/novox/mesh-host/internal/link"
 	"github.com/novox/mesh-host/internal/selfcompose"
+	"github.com/novox/mesh-host/internal/store"
 )
 
 // The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition
@@ -239,6 +240,7 @@ func within(t *testing.T, d time.Duration, what string, ok func() bool) {
 // applied within one second and its have names the fingerprint and the digest; a composition whose digest no want names
 // is not applied.
 func TestNatsAWantAndItsCompositionAreAppliedWithinASecond(t *testing.T) {
+	t.Run("1957: an engine start starts a declared-running unit", testNatsAnEngineStartStartsADeclaredRunningUnitTheHaveSaysRuns)
 	b := newItBed(t)
 	stop := b.start(false)
 	defer stop()
@@ -708,3 +710,71 @@ func h8rest(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, hkey str
 		t.Errorf("a candidate moved what runs: %+v, %q", h, b.file("web"))
 	}
 }
+
+// 1957/AC1 (integration; ADR 0568 §1): a node agent starting on a bus that holds a `running` have for a module at its want,
+// with a service manager (a systemctl on the PATH) that shows the module's unit inactive, starts the unit once and writes
+// the have running; the same start with the unit active applies nothing.
+func testNatsAnEngineStartStartsADeclaredRunningUnitTheHaveSaysRuns(t *testing.T) {
+	for _, active := range []bool{false, true} {
+		b := newItBed(t)
+		bin := t.TempDir()
+		stateFile, calls := filepath.Join(bin, "state"), filepath.Join(bin, "calls")
+		if active {
+			os.WriteFile(stateFile, []byte("active"), 0o644)
+		} else {
+			os.WriteFile(stateFile, []byte("inactive"), 0o644)
+		}
+		script := "#!/bin/sh\necho \"$@\" >> " + calls + "\ncase \"$1\" in\n" +
+			"show) printf 'LoadState=loaded\\nActiveState=%s\\nType=simple\\n' \"$(cat " + stateFile + ")\" ;;\n" +
+			"start) printf active > " + stateFile + " ;;\nis-enabled) echo enabled ;;\nesac\nexit 0\n"
+		if err := os.WriteFile(filepath.Join(bin, "systemctl"), []byte(script), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
+
+		// The engine's own record of the unit, as a previous run left it, and the have it wrote then.
+		res, _ := json.Marshal(map[string]any{"id": "web.unit", "type": "service", "unit": "web-it.service", "state": "running", "module": "web"})
+		var known store.State
+		known.Record(store.Applied{ID: "web.unit", Type: "service", Target: "web-it.service"})
+		if err := store.Save(b.opts.state, known); err != nil {
+			t.Fatal(err)
+		}
+		fp := fpIt("a")
+		c := delivery.Composed{Node: itNode, Module: "web", FP: fp, Resources: []json.RawMessage{res}}
+		signed, _ := delivery.SignComposed(c, b.controll)
+		raw, _ := delivery.EncodeComposed(signed)
+		dig, _ := delivery.ComposedDigestOf(signed)
+		key, _ := delivery.ComposedKey(itNode, "web", fp)
+		if _, err := b.composed.Put(key, raw); err != nil {
+			t.Fatal(err)
+		}
+		b.want("web", fp, dig, "")
+		hkey, _ := delivery.HaveKey(itNode, "web")
+		have, err := delivery.EncodeHave(delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := b.haves.Put(hkey, have); err != nil {
+			t.Fatal(err)
+		}
+
+		stop := b.start(false)
+		within(t, 5*time.Second, "the unit was not started and the have not running", func() bool {
+			got, _ := os.ReadFile(stateFile)
+			h := b.have("web")
+			return string(got) == "active" && h.Phase == delivery.PhaseRunning && h.Composed == dig
+		})
+		time.Sleep(300 * time.Millisecond)
+		stop()
+		starts := 0
+		log, _ := os.ReadFile(calls)
+		for _, l := range strings.Split(string(log), "\n") {
+			if strings.HasPrefix(l, "start") {
+				starts++
+			}
+		}
+		if want := map[bool]int{false: 1, true: 0}[active]; starts != want {
+			t.Fatalf("unit active=%v: started %d times, want %d", active, starts, want)
+		}
+	}
+}