Keep what each machine did, so status can say what is wrong

A node reports back after applying a declaration: it worked, some of it failed, or the whole thing was refused. A refusal or a failure moved last_seen and the reason went to a log line — so "which machine is not doing what it was told" had no answer the next morning, which is the question a mesh exists to answer. Refused and failed are kept as different things, because they are different situations with different remedies: refused means the machine is exactly as it was and what is wrong is in what was sent; failed means it is in a state nobody declared and what is wrong is on the machine. One word for both would make the record say less than the node did. One row per node, replaced. The question is the machine's current state — "this failed an hour ago and then succeeded" is not a machine anybody needs to look at, and a table of every report would bury the ones that matter under the ones that do not. `status` now answers three questions in the order somebody asks them: is anything broken, is anything not answering, is anything out of date. The first has consequences now, the third is a plan for later, and a status leading with the third would bury the first. A machine that has never spoken is reported as quiet rather than as broken — new, switched off and unreachable are not the same as tried and could not. The mapping from a report to an outcome had no test at all, which the injection caught: it is the code deciding which of those situations a machine is in. It has four now, including that a partial report never becomes the account of what the machine holds — the fault that destroyed a substrate once.
2026-08-30 18:08:59 +02:00
parent 3195634441
commit 9681b288aa
6 changed files with 502 additions and 23 deletions
@@ -0,0 +1,157 @@
+package inventory
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+// What each machine did with what it was last sent.
+//
+// Until this, a refusal or a failure moved last_seen and the reason went to a log line — so
+// "which machine is not doing what it was told" had no answer the next morning, which is the
+// question a mesh exists to answer.
+
+func nodeNamed(t *testing.T, inv *Inventory, name string) string {
+	t.Helper()
+	n, err := inv.AddNode(context.Background(), name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return n.ID
+}
+
+func TestRefusedAndFailedAreDifferentSituations(t *testing.T) {
+	// Refused means the machine is exactly as it was, and what is wrong is in what was sent.
+	// Failed means it is in a state nobody declared, and what is wrong is on the machine. One
+	// word for both would make the report say less than the node did.
+	inv := fresh(t)
+	ctx := context.Background()
+	refuser := nodeNamed(t, inv, "refuser")
+	failer := nodeNamed(t, inv, "failer")
+
+	if err := inv.RecordDoing(ctx, refuser, Doing{
+		Outcome: OutcomeRefused, Refused: "resource \"x\": a file needs a path",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordDoing(ctx, failer, Doing{
+		Outcome: OutcomeFailed,
+		Failed:  []FailedResource{{ID: "svc", Error: "unit not found"}},
+		Applied: 4,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	wrong, err := inv.NotDoingWhatTheyWereTold(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(wrong) != 2 {
+		t.Fatalf("got %d", len(wrong))
+	}
+	by := map[string]Doing{}
+	for _, d := range wrong {
+		by[d.Node] = d
+	}
+	if by["refuser"].Outcome != OutcomeRefused || !strings.Contains(by["refuser"].Refused, "needs a path") {
+		t.Fatalf("got %+v", by["refuser"])
+	}
+	if by["failer"].Outcome != OutcomeFailed || len(by["failer"].Failed) != 1 {
+		t.Fatalf("got %+v", by["failer"])
+	}
+	// And how much DID work, because "four of five" and "none of five" are different machines.
+	if by["failer"].Applied != 4 {
+		t.Fatalf("what did apply was not kept: %+v", by["failer"])
+	}
+}
+
+func TestAMachineDoingWhatItWasToldIsNotOnTheList(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	id := nodeNamed(t, inv, "fine")
+	if err := inv.RecordDoing(ctx, id, Doing{Outcome: OutcomeApplied, Applied: 6}); err != nil {
+		t.Fatal(err)
+	}
+	wrong, err := inv.NotDoingWhatTheyWereTold(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(wrong) != 0 {
+		t.Fatalf("a machine that did as it was told is listed as wrong: %+v", wrong)
+	}
+	// It is still answerable about, which is a different question.
+	d, said, err := inv.DoingOf(ctx, "fine")
+	if err != nil || !said {
+		t.Fatalf("said=%v err=%v", said, err)
+	}
+	if d.Wrong() || d.Applied != 6 {
+		t.Fatalf("got %+v", d)
+	}
+}
+
+func TestTheLastReportReplacesTheOneBefore(t *testing.T) {
+	// The question is the machine's CURRENT state. "This failed an hour ago and then succeeded"
+	// is not a machine anybody needs to look at, and a table of every report would bury the ones
+	// that matter under the ones that do not.
+	inv := fresh(t)
+	ctx := context.Background()
+	id := nodeNamed(t, inv, "recovered")
+	if err := inv.RecordDoing(ctx, id, Doing{
+		Outcome: OutcomeFailed, Failed: []FailedResource{{ID: "a", Error: "no"}},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordDoing(ctx, id, Doing{Outcome: OutcomeApplied, Applied: 3}); err != nil {
+		t.Fatal(err)
+	}
+	wrong, err := inv.NotDoingWhatTheyWereTold(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(wrong) != 0 {
+		t.Fatalf("a machine that recovered is still listed as wrong: %+v", wrong)
+	}
+	d, _, _ := inv.DoingOf(ctx, "recovered")
+	if len(d.Failed) != 0 {
+		t.Fatalf("the old failure survived: %+v", d)
+	}
+}
+
+func TestAMachineThatHasSaidNothingIsNotWrong(t *testing.T) {
+	// It may be new, switched off, or unreachable. None of those is a machine that tried and
+	// could not, and treating silence as failure would have somebody debug a machine that has
+	// simply never been sent anything.
+	inv := fresh(t)
+	ctx := context.Background()
+	nodeNamed(t, inv, "silent")
+	wrong, err := inv.NotDoingWhatTheyWereTold(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(wrong) != 0 {
+		t.Fatalf("silence was read as failure: %+v", wrong)
+	}
+	if _, said, err := inv.DoingOf(ctx, "silent"); err != nil || said {
+		t.Fatalf("a machine that never reported was reported about: said=%v err=%v", said, err)
+	}
+}
+
+func TestWhatANodeSaidGoesWhenTheNodeDoes(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	id := nodeNamed(t, inv, "leaving")
+	if err := inv.RecordDoing(ctx, id, Doing{Outcome: OutcomeFailed}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'leaving'`); err != nil {
+		t.Fatal(err)
+	}
+	var left int
+	if err := inv.store.Pool().QueryRow(ctx, `select count(*) from node_report`).Scan(&left); err != nil {
+		t.Fatal(err)
+	}
+	if left != 0 {
+		t.Fatalf("%d report(s) outlived the machine", left)
+	}
+}