From 7b7796b6faf670a409fe12407db5d3d1c14ca2d9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:41:44 +0100 Subject: [PATCH] Assigner: hold a want while a provision its module requires has no key yet, and say it once it stays without one past the delivery bound, so a module never starts without its database unseen (issue 1954 D1) Pins mesh-sdk 66ff33d. --- .../cmd/mesh-delivery-assigner/pass.go | 71 +++++++++++++++++++ .../cmd/mesh-delivery-assigner/pass_test.go | 5 ++ .../mesh-delivery-assigner/seat_tools_test.go | 22 ++++++ modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 4 +- .../internal/assign/policy.go | 25 +++++++ .../internal/assign/wants.go | 46 ++++++++---- .../internal/assign/wants_test.go | 15 +++- 8 files changed, 172 insertions(+), 18 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 9a6c0fd..9c841f7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -34,6 +34,8 @@ type passer struct { splits map[string]bool // failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4). failedProvisions map[string]bool + // missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote. + missing, missingSaid map[string]bool // answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want. answeredAt map[string]uint64 asks *composeAsks @@ -138,6 +140,8 @@ func (p *passer) wants(ctx context.Context) { wanted := p.w.wantsOf() chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket)) provisionsOf := p.provisionsOf() + missing := map[string]bool{} + defer func() { p.provisionsMissing(ctx, missing) }() defer p.policySplits(manifests, chosen) frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers())) placed := p.w.placedNodes() @@ -207,6 +211,11 @@ func (p *passer) wants(ctx context.Context) { last[node] = p.w.lastHealthy(node, name, fp) } out := assign.Wants(m, nodes, wanted[name], last) + for _, node := range m.Nodes { + for _, prov := range m.MissingProvisions(node) { + missing[node+"."+name+"."+prov] = true + } + } p.yeses(ctx, name, m, wanted[name], &out) gatherNeither(neither, name, out) leftNeither(left, m, wanted[name], out) @@ -1069,3 +1078,65 @@ func (p *passer) provisionFindings(failed map[string]provisions.Provision) { delete(p.failedProvisions, key) } } + +// MissingBound is how long a required provision may have no key before it is said: a module's delivery bound, the default +// when its manifest names none (ADR 0564 clause 13 (d)). +var MissingBound = delivery.DefaultDeliveryBound + +// missingPulse is the pulse key of a provision with no key, `provision-missing...`. +func missingPulse(key string) string { return "provision-missing." + key } + +// provisionsMissing is the loud end of a required provision with no key (the lead, 2026-10-29, ADR 0570 §4): each one is +// pulsed with the bound when it is first seen; once its pulse expired and it still has no key, the finding +// `provider....provision-missing` is written; one keyed again ends both. The pulse's expiry, heard on +// the watch, is the pass that writes it: no timer. +func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) { + pulses := p.kvs[delivery.PlacementPulsesBucket] + if p.missing == nil { + p.missing = map[string]bool{} + } + for key := range missing { + switch { + case !p.missing[key]: + p.missing[key] = true + if pulses != nil { + p.pulse(ctx, missingPulse(key), MissingBound) + } + case pulses == nil || !p.pulsing(missingPulse(key)): + if p.missingSaid[key] || p.findings.Put == nil { + continue + } + value, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock()) + if err == nil { + err = p.findings.Put(assign.ProvisionMissingKey(key), value) + } + if err != nil { + p.logf("the missing provision %s is not said: %v", key, err) + continue + } + if p.missingSaid == nil { + p.missingSaid = map[string]bool{} + } + p.missingSaid[key] = true + } + } + for key := range p.missing { + if missing[key] { + continue + } + delete(p.missing, key) + if pulses != nil && p.pulsing(missingPulse(key)) { + if err := pulses.Delete(ctx, missingPulse(key)); err == nil { + p.w.put(delivery.PlacementPulsesBucket, missingPulse(key), nil) + } + } + if p.missingSaid[key] && p.findings.Delete != nil { + if err := p.findings.Delete(assign.ProvisionMissingKey(key)); err != nil { + p.logf("the missing provision %s is not cleared: %v", key, err) + p.missing[key] = true + continue + } + delete(p.missingSaid, key) + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 6b7f420..affbf2e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -32,6 +32,11 @@ func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) return uint64(len(m.values)), nil } +func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) { + m.values[key] = value + return uint64(len(m.values)), nil +} + func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error { delete(m.values, key) return nil diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index 9322fc4..af24d5f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "os" "slices" @@ -182,6 +183,27 @@ func TestTheSeatsVerbs(t *testing.T) { if len(findings) != 0 { t.Errorf("findings after provided = %v", findings) } + // A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed. + ctx := context.Background() + p.kvs[delivery.PlacementPulsesBucket] = &memKV{values: map[string][]byte{}} + gone := map[string]bool{"ace.photos.s3": true} + p.provisionsMissing(ctx, gone) + if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 { + t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings) + } + p.provisionsMissing(ctx, gone) + if len(findings) != 0 { + t.Fatal("said before its bound") + } + p.w.put(delivery.PlacementPulsesBucket, "provision-missing.ace.photos.s3", nil) // the server expired it + p.provisionsMissing(ctx, gone) + if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok { + t.Fatalf("past its bound: findings %v", findings) + } + p.provisionsMissing(ctx, map[string]bool{}) + if len(findings) != 0 { + t.Errorf("keyed again: findings %v", findings) + } }) } diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 788e658..a9b4fbe 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 37f9598..326dffd 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2 h1:tQZYFKHd378XTF6OPGncIJnapxQCtRJAtL2FvlIzNQ4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/policy.go b/modules/mesh-delivery-assigner/internal/assign/policy.go index 6b938a9..c2c9aa8 100644 --- a/modules/mesh-delivery-assigner/internal/assign/policy.go +++ b/modules/mesh-delivery-assigner/internal/assign/policy.go @@ -161,3 +161,28 @@ func also(provider, consumer string) []string { } return []string{provider} } + +// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound. +const KindProvisionMissing = "provision-missing" + +// ProvisionMissingKey is its key, `provider....provision-missing`. +func ProvisionMissingKey(key string) string { + return health.FindingKey(health.ScopeProvider, key, KindProvisionMissing) +} + +// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `..`). +func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]byte, error) { + parts := strings.SplitN(key, ".", 3) + if len(parts) != 3 { + return nil, fmt.Errorf("%q is not ..", key) + } + node, module, provision := parts[0], parts[1], parts[2] + summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+ + "no provider of it may be placed", module, node, provision, bound, module) + return health.EncodeFinding(health.Finding{Key: ProvisionMissingKey(key), Kind: KindProvisionMissing, + Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: node}, + Severity: health.Warning, Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision), + Explanation: asks.NothingToDo + " " + summary + ". It clears once the provision has its key.", + Resolved: provision + " for " + module + " on " + node + " has its key", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, + Source: "assigner." + KindProvisionMissing, Observer: delivery.AssignerModule, At: at.UTC()}) +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index b2ab36d..237d38a 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -130,30 +130,52 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { if _, ok := m.composition(node, m.FP); !ok { return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP)) } - if key, p, waiting := m.provisionWaiting(node); waiting { - words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State) - if p.Why != "" { - words += ": " + p.Why - } + if words, waiting := m.provisionWaiting(node); waiting { return false, delivery.RuleProvisions, words } return true, 0, "" } -// provisionWaiting is the first provision of the module on node, by key, that is not provided (ADR 0570 §4). A provision -// the holder has not composed a key for yet holds nothing: the holder composes each key ahead of the want, from placement -// and the newest build. -func (m Module) provisionWaiting(node string) (string, provisions.Provision, bool) { +// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has +// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none. +func (m Module) provisionWaiting(node string) (string, bool) { if m.Provisions == nil { - return "", provisions.Provision{}, false + return "", false + } + if missing := m.MissingProvisions(node); len(missing) > 0 { + return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true } keyed := m.Provisions(node) for _, key := range slices.Sorted(maps.Keys(keyed)) { if p := keyed[key]; p.State != provisions.Provided { - return key, p, true + words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State) + if p.Why != "" { + words += ": " + p.Why + } + return words, true } } - return "", provisions.Provision{}, false + return "", false +} + +// MissingProvisions are the provisions the module requires (not optionally) that have no key on node yet, sorted; none +// while the seat's state is not read. +func (m Module) MissingProvisions(node string) []string { + if m.Provisions == nil { + return nil + } + have := map[string]bool{} + for _, p := range m.Provisions(node) { + have[p.Provision] = true + } + var out []string + for _, r := range m.Manifest.Requires { + if !have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(out, r) { + out = append(out, r) + } + } + slices.Sort(out) + return out } // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index a6e0191..06886cb 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -73,7 +73,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { // 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want. delivery.RuleProvisions: func(m *Module, _ *Nodes) { m.Provisions = func(node string) map[string]provisions.Provision { - return map[string]provisions.Provision{node + ".mail.db": {Provision: "db", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}} + return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}} } }, } { @@ -89,14 +89,14 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) { t.Errorf("rule %d broken: g14's want is %+v", rule, w) } - if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.db") || !strings.Contains(out.Rules["g14"].Words, "asked")) { + if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.database") || !strings.Contains(out.Rules["g14"].Words, "asked")) { t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words) } } // 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it. for _, all := range []func(string) map[string]provisions.Provision{ func(node string) map[string]provisions.Provision { - return map[string]provisions.Provision{node + ".mail.db": {Provision: "db", State: provisions.Provided}} + return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Provided}} }, nil} { m, n := ready() m.Provisions = all @@ -104,6 +104,15 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words) } } + // 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named. + { + m, n := ready() + m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} } + if out := Wants(m, n, wantedOld(), nil); out.Rules["g14"].Rule != delivery.RuleProvisions || + !strings.Contains(out.Rules["g14"].Words, "provision database has no key yet") || len(m.MissingProvisions("g14")) != 1 { + t.Errorf("a provision with no key: %+v", out.Rules["g14"]) + } + } // 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the // provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not // healthy held photos, and novox offline left a module with no canary.