Merge remote-tracking branch 'origin/main' into feat/459-a-stop-re-walks-the-others

# Conflicts:
#	testdata/store-tests

internal/inventory/sends.go

@@ -100,6 +100,23 @@ type Send struct {
 	CounterRaisedTo int64
 	// Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown.
 	Recorded bool
+	// Answer is what the machine last reported of this send; filled only where a last send is read back
+	// (LastSend, LastSends), the zero answer elsewhere.
+	Answer SendAnswer
+}
+
+// SendAnswer is what a machine reported of one send, read from its last report when that report is about the
+// send's declaration (novox/hq issue 485): whether it applied it, failed part of it or refused it, and why.
+// The zero answer is a machine that has not reported on this send — yet, or since a later report replaced it.
+type SendAnswer struct {
+	// Outcome is applied, failed or refused, as the machine reported it; empty when it has not reported on it.
+	Outcome string
+	// Refused is the machine's own words when it refused the declaration whole.
+	Refused string
+	// Failed is how many of its resources failed, when some did.
+	Failed int
+	// At is when the machine reported it; nil when it has not.
+	At *time.Time
 }
 
 // unknownSender is the sender of a refused sequence the mesh has no record of sending.
@@ -209,20 +226,56 @@ func scanSends(rows pgx.Rows) ([]Send, error) {
 	return out, rows.Err()
 }
 
-// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. A refusal
-// of a sequence the mesh has no send for is not a send, and is not it.
+// LastSend is the last declaration a machine was sent, by its name, with what the machine answered of it; false
+// when none was recorded. A refusal of a sequence the mesh has no send for is not a send, and is not it.
 func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) {
-	rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+`
-		  from declaration_send s join node n on n.id = s.node
-		 where n.name = $1 and s.recorded order by s.id desc limit 1`, name)
+	sends, err := i.lastSends(ctx, `n.name = $1`, name)
 	if err != nil {
 		return Send{}, false, err
 	}
-	sends, err := scanSends(rows)
-	if err != nil || len(sends) == 0 {
-		return Send{}, false, err
+	s, found := sends[name]
+	return s, found, nil
+}
+
+// LastSends is every machine's last send, by machine name, each with what the machine answered of it: what
+// `status` says per machine (novox/hq issue 485). A machine never sent anything has no entry.
+func (i *Inventory) LastSends(ctx context.Context) (map[string]Send, error) {
+	return i.lastSends(ctx, `true`)
+}
+
+// lastSends reads the newest recorded send of each machine matching where, beside the machine's last report
+// when that report is about the send (novox/hq issue 485). It names the send's declaration (its digest), and:
+//   - from an ordered node-engine, one that reports the epoch and sequence it acted on, those are the send's
+//     own. A periodic report about an identical earlier send names the same digest and may arrive after a later
+//     send the machine never received; only the sequence tells them apart. It also holds for a fast machine
+//     whose report lands before the send's row is stamped.
+//   - from an engine that reports no order, the report came after the send — both times the store's own — so
+//     a machine sent again a declaration it once applied does not read as having applied the new send.
+func (i *Inventory) lastSends(ctx context.Context, where string, args ...any) (map[string]Send, error) {
+	rows, err := i.store.Pool().Query(ctx, `select distinct on (n.name) `+sendColumns+`,
+		       coalesce(r.outcome, ''), coalesce(r.refused, ''), case when jsonb_typeof(r.failed) = 'array' then jsonb_array_length(r.failed) else 0 end, r.at
+		  from declaration_send s join node n on n.id = s.node
+		  left join node_report r on r.node = s.node and r.declared <> '' and r.declared = s.digest
+		       and case when r.reported_sequence is not null
+		                 then r.reported_sequence = s.sequence and coalesce(r.reported_epoch, 0) = coalesce(s.epoch, 0)
+		                 else r.at >= s.sent_at end
+		 where s.recorded and `+where+`
+		 order by n.name, s.id desc`, args...)
+	if err != nil {
+		return nil, err
 	}
-	return sends[0], true, nil
+	defer rows.Close()
+	out := map[string]Send{}
+	for rows.Next() {
+		var s Send
+		if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest,
+			&s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded,
+			&s.Answer.Outcome, &s.Answer.Refused, &s.Answer.Failed, &s.Answer.At); err != nil {
+			return nil, err
+		}
+		out[s.NodeName] = s
+	}
+	return out, rows.Err()
 }
 
 // RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and

internal/inventory/sends_test.go

@@ -205,3 +205,144 @@ func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
 		}
 	}
 }
+
+// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its
+// words, or not answered yet — read from the machine's report only when that report is about the send's
+// declaration and came after it, so a report about an earlier send is never this one's answer. And every
+// machine's last send is read at once, for status, by name; a machine never sent anything has none.
+func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) {
+	inv, node := aNodeWithModules(t)
+	ctx := t.Context()
+	record, err := inv.NodeByName(ctx, node)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.AddNode(ctx, "laptop"); err != nil {
+		t.Fatal(err)
+	}
+	send := func(sequence int64, digest string) {
+		t.Helper()
+		if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
+			Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest,
+			Modules: []string{"docker", "sudo"}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	report := func(d Doing) {
+		t.Helper()
+		if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil {
+			t.Fatal(err)
+		}
+	}
+	answer := func() SendAnswer {
+		t.Helper()
+		last, found, err := inv.LastSend(ctx, node)
+		if err != nil || !found {
+			t.Fatalf("the last send is not read back: %v, %v", found, err)
+		}
+		return last.Answer
+	}
+
+	send(11, "d11")
+	if a := answer(); a.Outcome != "" || a.At != nil {
+		t.Fatalf("a send not reported on reads as answered: %+v", a)
+	}
+	report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3})
+	if a := answer(); a.Outcome != OutcomeApplied || a.At == nil {
+		t.Fatalf("a send the machine reported applying reads %+v", a)
+	}
+
+	send(12, "d12")
+	if a := answer(); a.Outcome != "" {
+		t.Fatalf("the report about the send before is taken for this one's answer: %+v", a)
+	}
+	report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"})
+	if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" {
+		t.Fatalf("a send the machine refused reads %+v", a)
+	}
+
+	// Sent again a declaration it reported applying before the send: not answered until it reports again.
+	report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3})
+	send(13, "d12")
+	if a := answer(); a.Outcome != "" {
+		t.Fatalf("a report from before the send is taken for its answer: %+v", a)
+	}
+
+	all, err := inv.LastSends(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 ||
+		!slices.Equal(all[node].Modules, []string{"docker", "sudo"}) {
+		t.Fatalf("every machine's last send reads %+v", all)
+	}
+	if _, found := all["laptop"]; found {
+		t.Error("a machine never sent anything has a last send")
+	}
+}
+
+// **A report about another declaration, or another send of the same one, is not the last send's answer**
+// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives
+// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about
+// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only
+// the report's epoch and sequence say which send it is about.
+func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) {
+	inv, node := aNodeWithModules(t)
+	ctx := t.Context()
+	record, err := inv.NodeByName(ctx, node)
+	if err != nil {
+		t.Fatal(err)
+	}
+	send := func(sequence int64, digest string) {
+		t.Helper()
+		if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
+			Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	ordered := func(epoch, sequence int64, digest string) {
+		t.Helper()
+		if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1},
+			ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil {
+			t.Fatal(err)
+		}
+	}
+	answer := func() string {
+		t.Helper()
+		last, found, err := inv.LastSend(ctx, node)
+		if err != nil || !found {
+			t.Fatalf("the last send is not read back: %v, %v", found, err)
+		}
+		return last.Answer.Outcome
+	}
+
+	// An engine that reports no order: the digest decides.
+	send(11, "d11")
+	send(12, "d12")
+	if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil {
+		t.Fatal(err)
+	}
+	if a := answer(); a != "" {
+		t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a)
+	}
+
+	// An ordered engine: the epoch and sequence decide, whatever the digest and the time.
+	send(20, "d20")
+	ordered(57, 20, "d20")
+	if a := answer(); a != OutcomeApplied {
+		t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a)
+	}
+	send(21, "d20")
+	ordered(57, 20, "d20")
+	if a := answer(); a != "" {
+		t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a)
+	}
+	ordered(56, 21, "d20")
+	if a := answer(); a != "" {
+		t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a)
+	}
+	ordered(57, 21, "d20")
+	if a := answer(); a != OutcomeApplied {
+		t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a)
+	}
+}