Say a machine waiting for its push as waiting, not uncomposable (hq issue 275)

Between assign and push a module's own secrets are not made yet; D1 composed
without making them and raised an urgent 'nothing can be sent' that the next
push resolved silently. D1 now composes as the push would (Foreseeing): a
secret the push makes gets a stand-in and is named, one the push is refused on
is refused with the push's words. Waiting is said only past 30 minutes, as a
warning. D3 and D13 expect a holder only once its machine was sent it and
reported or had ten minutes to.

cmd/mesh-controller/probes.go

@@ -23,6 +23,7 @@ import (
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/lease"
 	"github.com/novox/mesh-controller/internal/link"
 	"github.com/novox/mesh-controller/internal/overlay"
@@ -32,8 +33,25 @@ import (
 // wrong now as observations, or an error when it could not tell — never "nothing" for "could not
 // look" (ADR 0227 rule 4).
 
+// awaitingPushBound is how long a machine may have something only its next push makes — an own
+// secret of a module assigned since its last push — before that is said (novox/hq issue 275).
+var awaitingPushBound = 30 * time.Minute
+
+// kindAwaitingPush is D1's kind for a machine waiting for a push past awaitingPushBound.
+const kindAwaitingPush = "awaiting-push"
+
 // probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
 // (mesh-host's `validate`, the package the host runs) takes it.
+//
+// **Composed as the next push would compose it, without making anything** (Foreseeing, novox/hq issue
+// 275). Between `assign` and `push` a module's own secrets are not made yet — the push makes them —
+// and a composition that only reads them failed, which raised an urgent "nothing can be sent" about a
+// machine the next push sent to without a word. So a secret the push WILL make is composed with a
+// stand-in and named; one the push would be refused on is refused here, with the push's words. A
+// machine whose declaration composes and validates with only such stand-ins is waiting for a push,
+// not broken: nothing is said until awaitingPushBound passes from when it began waiting, and then a
+// warning (`awaiting-push`) naming what the push will make — never urgent, because nothing is wrong
+// that a push does not fix.
 func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
 	open := d.open
 	nodes, err := open.inventory.Nodes(ctx)
@@ -59,10 +77,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
 		if err != nil && !unresolvable(err) {
 			return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
 		}
-		var problems []string
+		var problems, foreseen []string
 		if err == nil && gensErr == nil {
 			var declared sendable
-			if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
+			if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
+				foreseen = declared.foreseen
 				// With the order it was last sent, so the validator reads the envelope a machine is sent
 				// — its epoch included (novox/hq to-be 45 §6).
 				var serr error
@@ -94,11 +113,30 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
 				Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
 					n.Name, len(problems), problems[0]),
 				Said: strings.Join(problems, "; ")})
+		case len(foreseen) > 0:
+			since, err := open.inventory.AwaitingSince(ctx, n.Name)
+			if err != nil {
+				return nil, err // the store, not the machine: the probe could not run
+			}
+			if waited := time.Since(since); waited > awaitingPushBound {
+				out = append(out, awaitingPush(n.Name, foreseen, since, waited))
+			}
 		}
 	}
 	return out, nil
 }
 
+// awaitingPush is D1's warning for a machine whose declaration composes only once its next push has
+// made what it names, past awaitingPushBound (novox/hq issue 275).
+func awaitingPush(node string, foreseen []string, since time.Time, waited time.Duration) conditions.Observation {
+	made := strings.Join(foreseen, ", ")
+	return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: kindAwaitingPush,
+		Kind: kindAwaitingPush, Machine: node, Severity: conditions.Warning,
+		Summary: fmt.Sprintf("%s has waited %s for a push: its declaration composes once the next push makes %s — "+
+			"`push %s` sends it", node, waited.Round(time.Minute), made, node),
+		Said: fmt.Sprintf("waiting since %s; the next push makes %s", since.UTC().Format(time.RFC3339), made)}
+}
+
 // probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
 // address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
 // takes as final (issue 262).
@@ -221,6 +259,11 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
 		return nil, err
 	}
 	heard := heardMachines(d)
+	delivered, err := readDeliveries(ctx, d.open.inventory)
+	if err != nil {
+		return nil, err
+	}
+	now := time.Now()
 	expected := map[string]map[string]bool{} // seat → machine
 	add := func(seat, node string) {
 		if expected[seat] == nil {
@@ -248,7 +291,10 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
 					continue
 				}
 				for _, node := range e.On {
-					if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
+					// Assigned is not there yet: a holder is expected once its machine was sent it and
+					// had time to report, never between `assign` and `push` (novox/hq issue 275).
+					if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) &&
+						delivered[node].settled(e.Manifest.Module, now) {
 						add(s.Name, node)
 					}
 				}
@@ -277,6 +323,58 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
 	return sortedFound(out), nil
 }
 
+// reportGrace is how long a machine is given, after it was sent a declaration, to apply it and report
+// before what the declaration carries is expected to answer (novox/hq issue 275).
+var reportGrace = 10 * time.Minute
+
+// lastSend is what a machine was last sent, as far as the self-check needs it: which modules the send
+// carried, when, and whether the machine has reported acting on it.
+type lastSend struct {
+	// carried is the modules its last send carried; nil when that was not kept (a send from before
+	// it was, or one by hand), and then every module is taken as carried — as before this existed.
+	carried map[string]string
+	// sent is when its current declaration went to it; nil if nothing ever did.
+	sent *time.Time
+	// current says its last report names the declaration last sent.
+	current bool
+}
+
+// settled says whether a module's holder on this machine can be asked to answer now: the machine was
+// sent a declaration carrying it, and has reported acting on that declaration or had reportGrace to.
+// A machine never sent anything holds nothing the mesh put there.
+func (d lastSend) settled(module string, now time.Time) bool {
+	if d.sent == nil {
+		return false
+	}
+	if d.carried != nil {
+		if _, in := d.carried[module]; !in {
+			return false
+		}
+	}
+	return d.current || now.Sub(*d.sent) > reportGrace
+}
+
+// readDeliveries is every machine's last send, by name, from the records a send and a report keep.
+func readDeliveries(ctx context.Context, inv *inventory.Inventory) (map[string]lastSend, error) {
+	reports, err := inv.LastReports(ctx)
+	if err != nil {
+		return nil, err
+	}
+	out := map[string]lastSend{}
+	for _, r := range reports {
+		builds, known, err := inv.SentBuilds(ctx, r.Node)
+		if err != nil {
+			return nil, err
+		}
+		d := lastSend{sent: r.Sent, current: r.Current}
+		if known {
+			d.carried = builds
+		}
+		out[r.Node] = d
+	}
+	return out, nil
+}
+
 // heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
 func heardMachines(d *doctor) map[string]bool {
 	out := map[string]bool{}