Say a module assigned and left out of its machine's composition as a condition (issue 380)

A push leaves out a module whose settings do not compose and sends the rest,
which is right, but only plan said so: nfs-server ran nowhere for days while
the operator believed it ran. The self-check now raises needs-operator for a
setting nobody gave, naming the setting and the command, and left-out for any
other cause; both warnings, cleared once the module composes or is unassigned.
status and node show list the same modules as assigned, not applied.

cmd/mesh-controller/left_out_test.go

@@ -0,0 +1,203 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// novox/hq issue 380: nfs-server was assigned to the home server for days and every push left it out — "nfs-server
+// has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The
+// manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says
+// ${setting:shares}; the reason below is the one the live push printed.
+
+// leftOutNFS is the resolution of a machine assigned that nfs-server, with the settings given.
+func leftOutNFS(t *testing.T) catalogue.Resolution {
+	t.Helper()
+	raw, err := os.ReadFile("testdata/left-out/nfs-server.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	m, err := catalogue.ParseManifest(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return catalogue.Resolution{Modules: []catalogue.Manifest{m}}
+}
+
+// sharesGiven is the operator's setting for it on the machine.
+func sharesGiven(value string) catalogue.SettingsBy {
+	return catalogue.SettingsBy{"nfs-server": {{From: "anchor", Values: map[string]any{"shares": value}}}}
+}
+
+func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndTheCommand(t *testing.T) {
+	left := leftOutOf(leftOutNFS(t), nil, false)
+	if len(left) != 1 || left[0].Module != "nfs-server" || left[0].Setting != "shares" {
+		t.Fatalf("left out: %+v", left)
+	}
+	if !strings.Contains(left[0].Why, `nothing sets "shares" for it`) {
+		t.Fatalf("the reason is not the push's own: %s", left[0].Why)
+	}
+	obs := leftOutObservations("anchor", left)
+	if len(obs) != 1 {
+		t.Fatalf("raised %+v", obs)
+	}
+	o := obs[0]
+	if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindNeedsOperator || o.Machine != "anchor" ||
+		o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator {
+		t.Fatalf("raised %s as %s, %s, by %s, on %q", o.Key(), o.Kind, o.Severity, o.Resolver, o.Machine)
+	}
+	for _, want := range []string{"nfs-server", "anchor", `"shares"`, "`settings set nfs-server '{\"shares\": …}' --node anchor`"} {
+		if !strings.Contains(o.Summary, want) {
+			t.Errorf("the summary does not name %s: %s", want, o.Summary)
+		}
+	}
+	if o.Said != left[0].Why {
+		t.Errorf("the evidence is not the composition's reason: %s", o.Said)
+	}
+	plainExample(t, o, "nfs-server is not applied on anchor",
+		"Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+
+			"anchor, and every update of anchor leaves it out because nothing sets its setting shares. Nothing of it "+
+			"runs there until it is set; the rest of anchor is updated as usual.")
+}
+
+func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) {
+	// A setting stored that its definition can no longer compose: one with a line break (issue 339).
+	left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false)
+	if len(left) != 1 || left[0].Setting != "" {
+		t.Fatalf("left out: %+v", left)
+	}
+	obs := leftOutObservations("anchor", left)
+	if len(obs) != 1 {
+		t.Fatalf("raised %+v", obs)
+	}
+	o := obs[0]
+	if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindLeftOut || o.Severity != conditions.Warning {
+		t.Fatalf("raised %s as %s, %s", o.Key(), o.Kind, o.Severity)
+	}
+	if !strings.Contains(o.Said, "holds a line break") || strings.Contains(o.Summary, "/srv/") {
+		t.Fatalf("the reason is not the evidence, or the summary carries it to the channel:\n%s\n%s", o.Summary, o.Said)
+	}
+	plainExample(t, o, "nfs-server is not applied on anchor",
+		"Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+
+			"anchor, and every update of anchor leaves it out because what is set for it does not fit its definition. "+
+			"Nothing of it changes there; the rest of anchor is updated as usual.")
+}
+
+// The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes
+// again or is no longer assigned; a machine's health statement neither clears nor raises it.
+func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) {
+	plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil)
+	withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1,
+		run: func(context.Context, *doctor) ([]conditions.Observation, error) {
+			return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil
+		}})
+	store := conditions.NewInMemory()
+	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
+	defer k.Close(context.Background())
+	d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"}
+	key := "module.nfs-server.anchor.left-out"
+	openOnes := func() map[string]conditions.Condition {
+		t.Helper()
+		open, err := k.Open(t.Context())
+		if err != nil {
+			t.Fatal(err)
+		}
+		out := map[string]conditions.Condition{}
+		for _, c := range open {
+			out[c.Key] = c
+		}
+		return out
+	}
+
+	d.runOnce(t.Context(), "a test")
+	c, raised := openOnes()[key]
+	if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning {
+		t.Fatalf("a module left out raised %+v", openOnes())
+	}
+	// A statement from the machine that says nothing of it — the module runs nothing there — leaves it open.
+	if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil,
+		time.Now().Add(72*time.Hour)); err != nil {
+		t.Fatal(err)
+	}
+	if _, still := openOnes()[key]; !still {
+		t.Fatal("a health statement that says nothing of the module cleared its left-out condition")
+	}
+	d.runOnce(t.Context(), "a test")
+	if c := openOnes()[key]; c.Severity != conditions.Warning {
+		t.Fatalf("after a health statement and days, it is %+v", openOnes())
+	}
+
+	// The setting given: it composes, and the condition clears.
+	settings = sharesGiven("library=/srv/library")
+	d.runOnce(t.Context(), "a test")
+	if _, still := openOnes()[key]; still {
+		t.Fatalf("composed again, still open: %+v", openOnes())
+	}
+
+	// Left out again, then unassigned: no longer in the machine's set, and it clears.
+	settings = nil
+	d.runOnce(t.Context(), "a test")
+	if _, raised := openOnes()[key]; !raised {
+		t.Fatal("left out again and not raised")
+	}
+	plan = catalogue.Resolution{}
+	d.runOnce(t.Context(), "a test")
+	if _, still := openOnes()[key]; still {
+		t.Fatalf("unassigned, still open: %+v", openOnes())
+	}
+}
+
+func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) {
+	for _, p := range probeRegistry {
+		if p.ID == probeLeftOutID {
+			if p.run == nil || p.Kind != kindLeftOut {
+				t.Fatalf("%+v", p)
+			}
+			return
+		}
+	}
+	t.Fatalf("no probe %s: a module left out is said nowhere", probeLeftOutID)
+}
+
+// status and node show list it under "assigned, not applied" with the reason the condition says, and status is
+// not well while there is one.
+func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) {
+	left := map[string][]leftOutModule{"anchor": leftOutOf(leftOutNFS(t), nil, false)}
+	reason := leftOutObservations("anchor", left["anchor"])[0].Summary
+	asked := answers{leftOut: left}
+	if asked.well() {
+		t.Fatal("a mesh with a module assigned and not applied is called well")
+	}
+	shown := printed(t, func() error { return printStatus(asked) })
+	if !strings.Contains(shown, "1 module(s) assigned, not applied") || !strings.Contains(shown, "nfs-server") ||
+		!strings.Contains(shown, "`settings set nfs-server '{\"shares\": …}' --node anchor` sets it") {
+		t.Fatalf("status says:\n%s", shown)
+	}
+	if !strings.Contains(reason, left["anchor"][0].reason("anchor")) {
+		t.Fatalf("status and the condition say different reasons:\n%s\n%s", shown, reason)
+	}
+	body, err := statusAsJSON(asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var doc struct {
+		NotApplied []machineNotApplied `json:"not-applied"`
+	}
+	if err := json.Unmarshal(body, &doc); err != nil || len(doc.NotApplied) != 1 ||
+		doc.NotApplied[0].Setting != "shares" || doc.NotApplied[0].Node != "anchor" {
+		t.Fatalf("status --json: %v %s", err, body)
+	}
+	lines := strings.Join(notAppliedLines("anchor", left["anchor"]), "\n")
+	if !strings.Contains(lines, "assigned, not applied:") || !strings.Contains(lines, "nfs-server") ||
+		!strings.Contains(lines, `nothing sets its setting "shares"`) {
+		t.Fatalf("node show says:\n%s", lines)
+	}
+}