Assigner: hold a want while a provision its module requires has no key yet, and say it once it stays without one past the delivery bound, so a module never starts without its database unseen (issue 1954 D1)

Pins mesh-sdk 66ff33d.

modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go

@@ -34,6 +34,8 @@ type passer struct {
 	splits   map[string]bool
 	// failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4).
 	failedProvisions map[string]bool
+	// missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote.
+	missing, missingSaid map[string]bool
 	// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
 	answeredAt map[string]uint64
 	asks       *composeAsks
@@ -138,6 +140,8 @@ func (p *passer) wants(ctx context.Context) {
 	wanted := p.w.wantsOf()
 	chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
 	provisionsOf := p.provisionsOf()
+	missing := map[string]bool{}
+	defer func() { p.provisionsMissing(ctx, missing) }()
 	defer p.policySplits(manifests, chosen)
 	frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers()))
 	placed := p.w.placedNodes()
@@ -207,6 +211,11 @@ func (p *passer) wants(ctx context.Context) {
 			last[node] = p.w.lastHealthy(node, name, fp)
 		}
 		out := assign.Wants(m, nodes, wanted[name], last)
+		for _, node := range m.Nodes {
+			for _, prov := range m.MissingProvisions(node) {
+				missing[node+"."+name+"."+prov] = true
+			}
+		}
 		p.yeses(ctx, name, m, wanted[name], &out)
 		gatherNeither(neither, name, out)
 		leftNeither(left, m, wanted[name], out)
@@ -1069,3 +1078,65 @@ func (p *passer) provisionFindings(failed map[string]provisions.Provision) {
 		delete(p.failedProvisions, key)
 	}
 }
+
+// MissingBound is how long a required provision may have no key before it is said: a module's delivery bound, the default
+// when its manifest names none (ADR 0564 clause 13 (d)).
+var MissingBound = delivery.DefaultDeliveryBound
+
+// missingPulse is the pulse key of a provision with no key, `provision-missing.<node>.<module>.<provision>`.
+func missingPulse(key string) string { return "provision-missing." + key }
+
+// provisionsMissing is the loud end of a required provision with no key (the lead, 2026-10-29, ADR 0570 §4): each one is
+// pulsed with the bound when it is first seen; once its pulse expired and it still has no key, the finding
+// `provider.<node>.<module>.<provision>.provision-missing` is written; one keyed again ends both. The pulse's expiry, heard on
+// the watch, is the pass that writes it: no timer.
+func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) {
+	pulses := p.kvs[delivery.PlacementPulsesBucket]
+	if p.missing == nil {
+		p.missing = map[string]bool{}
+	}
+	for key := range missing {
+		switch {
+		case !p.missing[key]:
+			p.missing[key] = true
+			if pulses != nil {
+				p.pulse(ctx, missingPulse(key), MissingBound)
+			}
+		case pulses == nil || !p.pulsing(missingPulse(key)):
+			if p.missingSaid[key] || p.findings.Put == nil {
+				continue
+			}
+			value, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock())
+			if err == nil {
+				err = p.findings.Put(assign.ProvisionMissingKey(key), value)
+			}
+			if err != nil {
+				p.logf("the missing provision %s is not said: %v", key, err)
+				continue
+			}
+			if p.missingSaid == nil {
+				p.missingSaid = map[string]bool{}
+			}
+			p.missingSaid[key] = true
+		}
+	}
+	for key := range p.missing {
+		if missing[key] {
+			continue
+		}
+		delete(p.missing, key)
+		if pulses != nil && p.pulsing(missingPulse(key)) {
+			if err := pulses.Delete(ctx, missingPulse(key)); err == nil {
+				p.w.put(delivery.PlacementPulsesBucket, missingPulse(key), nil)
+			}
+		}
+		if p.missingSaid[key] && p.findings.Delete != nil {
+			if err := p.findings.Delete(assign.ProvisionMissingKey(key)); err != nil {
+				p.logf("the missing provision %s is not cleared: %v", key, err)
+				p.missing[key] = true
+				continue
+			}
+			delete(p.missingSaid, key)
+		}
+	}
+}

modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go

@@ -32,6 +32,11 @@ func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error)
 	return uint64(len(m.values)), nil
 }
 
+func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
+	m.values[key] = value
+	return uint64(len(m.values)), nil
+}
+
 func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
 	delete(m.values, key)
 	return nil

modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"os"
 	"slices"
@@ -182,6 +183,27 @@ func TestTheSeatsVerbs(t *testing.T) {
 		if len(findings) != 0 {
 			t.Errorf("findings after provided = %v", findings)
 		}
+		// A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed.
+		ctx := context.Background()
+		p.kvs[delivery.PlacementPulsesBucket] = &memKV{values: map[string][]byte{}}
+		gone := map[string]bool{"ace.photos.s3": true}
+		p.provisionsMissing(ctx, gone)
+		if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 {
+			t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings)
+		}
+		p.provisionsMissing(ctx, gone)
+		if len(findings) != 0 {
+			t.Fatal("said before its bound")
+		}
+		p.w.put(delivery.PlacementPulsesBucket, "provision-missing.ace.photos.s3", nil) // the server expired it
+		p.provisionsMissing(ctx, gone)
+		if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok {
+			t.Fatalf("past its bound: findings %v", findings)
+		}
+		p.provisionsMissing(ctx, map[string]bool{})
+		if len(findings) != 0 {
+			t.Errorf("keyed again: findings %v", findings)
+		}
 	})
 }
 

modules/mesh-delivery-assigner/go.mod

@@ -3,7 +3,7 @@ module mesh-delivery-assigner
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-delivery-assigner/go.sum

@@ -1,5 +1,5 @@
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2 h1:tQZYFKHd378XTF6OPGncIJnapxQCtRJAtL2FvlIzNQ4=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

modules/mesh-delivery-assigner/internal/assign/policy.go

@@ -161,3 +161,28 @@ func also(provider, consumer string) []string {
 	}
 	return []string{provider}
 }
+
+// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound.
+const KindProvisionMissing = "provision-missing"
+
+// ProvisionMissingKey is its key, `provider.<node>.<module>.<provision>.provision-missing`.
+func ProvisionMissingKey(key string) string {
+	return health.FindingKey(health.ScopeProvider, key, KindProvisionMissing)
+}
+
+// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `<node>.<module>.<provision>`).
+func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]byte, error) {
+	parts := strings.SplitN(key, ".", 3)
+	if len(parts) != 3 {
+		return nil, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
+	}
+	node, module, provision := parts[0], parts[1], parts[2]
+	summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+
+		"no provider of it may be placed", module, node, provision, bound, module)
+	return health.EncodeFinding(health.Finding{Key: ProvisionMissingKey(key), Kind: KindProvisionMissing,
+		Subject:  health.Subject{Scope: health.ScopeProvider, ID: key, Machine: node},
+		Severity: health.Warning, Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
+		Explanation: asks.NothingToDo + " " + summary + ". It clears once the provision has its key.",
+		Resolved:    provision + " for " + module + " on " + node + " has its key", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
+		Source: "assigner." + KindProvisionMissing, Observer: delivery.AssignerModule, At: at.UTC()})
+}

modules/mesh-delivery-assigner/internal/assign/wants.go

@@ -130,30 +130,52 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
 	if _, ok := m.composition(node, m.FP); !ok {
 		return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
 	}
-	if key, p, waiting := m.provisionWaiting(node); waiting {
-		words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
-		if p.Why != "" {
-			words += ": " + p.Why
-		}
+	if words, waiting := m.provisionWaiting(node); waiting {
 		return false, delivery.RuleProvisions, words
 	}
 	return true, 0, ""
 }
 
-// provisionWaiting is the first provision of the module on node, by key, that is not provided (ADR 0570 §4). A provision
-// the holder has not composed a key for yet holds nothing: the holder composes each key ahead of the want, from placement
-// and the newest build.
-func (m Module) provisionWaiting(node string) (string, provisions.Provision, bool) {
+// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
+// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
+func (m Module) provisionWaiting(node string) (string, bool) {
 	if m.Provisions == nil {
-		return "", provisions.Provision{}, false
+		return "", false
+	}
+	if missing := m.MissingProvisions(node); len(missing) > 0 {
+		return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
 	}
 	keyed := m.Provisions(node)
 	for _, key := range slices.Sorted(maps.Keys(keyed)) {
 		if p := keyed[key]; p.State != provisions.Provided {
-			return key, p, true
+			words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
+			if p.Why != "" {
+				words += ": " + p.Why
+			}
+			return words, true
 		}
 	}
-	return "", provisions.Provision{}, false
+	return "", false
+}
+
+// MissingProvisions are the provisions the module requires (not optionally) that have no key on node yet, sorted; none
+// while the seat's state is not read.
+func (m Module) MissingProvisions(node string) []string {
+	if m.Provisions == nil {
+		return nil
+	}
+	have := map[string]bool{}
+	for _, p := range m.Provisions(node) {
+		have[p.Provision] = true
+	}
+	var out []string
+	for _, r := range m.Manifest.Requires {
+		if !have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(out, r) {
+			out = append(out, r)
+		}
+	}
+	slices.Sort(out)
+	return out
 }
 
 // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned

modules/mesh-delivery-assigner/internal/assign/wants_test.go

@@ -73,7 +73,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
 		// 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want.
 		delivery.RuleProvisions: func(m *Module, _ *Nodes) {
 			m.Provisions = func(node string) map[string]provisions.Provision {
-				return map[string]provisions.Provision{node + ".mail.db": {Provision: "db", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}}
+				return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}}
 			}
 		},
 	} {
@@ -89,14 +89,14 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
 		if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) {
 			t.Errorf("rule %d broken: g14's want is %+v", rule, w)
 		}
-		if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.db") || !strings.Contains(out.Rules["g14"].Words, "asked")) {
+		if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.database") || !strings.Contains(out.Rules["g14"].Words, "asked")) {
 			t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words)
 		}
 	}
 	// 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it.
 	for _, all := range []func(string) map[string]provisions.Provision{
 		func(node string) map[string]provisions.Provision {
-			return map[string]provisions.Provision{node + ".mail.db": {Provision: "db", State: provisions.Provided}}
+			return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Provided}}
 		}, nil} {
 		m, n := ready()
 		m.Provisions = all
@@ -104,6 +104,15 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
 			t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words)
 		}
 	}
+	// 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named.
+	{
+		m, n := ready()
+		m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} }
+		if out := Wants(m, n, wantedOld(), nil); out.Rules["g14"].Rule != delivery.RuleProvisions ||
+			!strings.Contains(out.Rules["g14"].Words, "provision database has no key yet") || len(m.MissingProvisions("g14")) != 1 {
+			t.Errorf("a provision with no key: %+v", out.Rules["g14"])
+		}
+	}
 	// 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the
 	// provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not
 	// healthy held photos, and novox offline left a module with no canary.