diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 934df8f..b7d75eb 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -30,6 +30,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" "mesh-delivery-assigner/internal/assign" ) @@ -71,6 +72,7 @@ var readBuckets = []string{ delivery.HaveBucket, delivery.StepsBucket(goBuildModule), takeoverBucket, + provisions.ProvisionsBucket, } // takeoverBucket is the condition keeper's take-over key's bucket (novox/hq ADR 0564 §9): whether the assigner's said @@ -157,6 +159,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) continue } + if name == provisions.ProvisionsBucket && err != nil { + // The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made; + // until then no want waits for a provision, as before. + logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) + continue + } if name == assign.PoliciesBucket && err != nil { // The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no // person's policy is read, so the controller's beside each composition holds, as before. @@ -313,6 +321,7 @@ var watched = []string{ delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket, assign.PoliciesBucket, + provisions.ProvisionsBucket, } // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after @@ -409,7 +418,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } started := 0 for _, name := range watched { - if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket || name == assign.PoliciesBucket) { + if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket || name == assign.PoliciesBucket || name == provisions.ProvisionsBucket) { continue // not on the bus at this start (open said so) } started++ @@ -439,7 +448,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } } // And the two that choose where grants are read from, with the same delivery as the grants part that writes one. - for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket} { + for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket} { if kvs[name] == nil && js != nil { name := name process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read", diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index facdff2..9a6c0fd 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" "git.novox.be/novox/mesh-sdk/go/sdkversion" "mesh-delivery-assigner/internal/assign" @@ -31,6 +32,8 @@ type passer struct { // modules whose policy-split finding stands, nil until the first pass, which deletes any a previous process left. findings assign.Findings splits map[string]bool + // failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4). + failedProvisions map[string]bool // answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want. answeredAt map[string]uint64 asks *composeAsks @@ -134,6 +137,7 @@ func (p *passer) wants(ctx context.Context) { } wanted := p.w.wantsOf() chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket)) + provisionsOf := p.provisionsOf() defer p.policySplits(manifests, chosen) frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers())) placed := p.w.placedNodes() @@ -160,7 +164,7 @@ func (p *passer) wants(ctx context.Context) { for name := range names { fp := trunk[name].FP2 name := name - m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], + m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], Self: func(node string) bool { return placed[node] }, Composed: func(node, fp string) (delivery.ComposedDigest, bool) { if placed[node] { @@ -994,3 +998,74 @@ func (p *passer) policySplits(manifests map[string]catalogue.Manifest, chosen ma } p.splits = split } + +// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat +// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed +// provision is also said as the assigner's finding `provider..provision-failed` (a waiting one is the watchman's +// needs-operator), deleted once it is no longer failed. +func (p *passer) provisionsOf() func(module string) func(node string) map[string]provisions.Provision { + if p.kvs[provisions.ProvisionsBucket] == nil { + return func(string) func(string) map[string]provisions.Provision { return nil } + } + by := map[string]map[string]map[string]provisions.Provision{} // module, node, key + failed := map[string]provisions.Provision{} + for key, raw := range p.w.read(provisions.ProvisionsBucket) { + pr, err := provisions.Decode(raw) + if err != nil { + continue + } + node, module := pr.Consumer.Node, pr.Consumer.Module + if by[module] == nil { + by[module] = map[string]map[string]provisions.Provision{} + } + if by[module][node] == nil { + by[module][node] = map[string]provisions.Provision{} + } + by[module][node][key] = pr + if pr.State == provisions.Failed { + failed[key] = pr + } + } + p.provisionFindings(failed) + return func(module string) func(string) map[string]provisions.Provision { + return func(node string) map[string]provisions.Provision { return by[module][node] } + } +} + +// provisionFindings writes the finding of each failed provision and deletes the ones no longer failed. +func (p *passer) provisionFindings(failed map[string]provisions.Provision) { + if p.findings.Put == nil { + return + } + now := time.Now + if p.now != nil { + now = p.now + } + for key, pr := range failed { + if p.failedProvisions[key] { + continue + } + value, err := assign.ProvisionFailedFinding(key, pr, now()) + if err == nil { + err = p.findings.Put(assign.ProvisionFailedKey(key), value) + } + if err != nil { + p.logf("the failed provision %s is not said: %v", key, err) + continue + } + if p.failedProvisions == nil { + p.failedProvisions = map[string]bool{} + } + p.failedProvisions[key] = true + } + for key := range p.failedProvisions { + if _, still := failed[key]; still { + continue + } + if err := p.findings.Delete(assign.ProvisionFailedKey(key)); err != nil { + p.logf("the failed provision %s is not cleared: %v", key, err) + continue + } + delete(p.failedProvisions, key) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index cba607c..9322fc4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -12,6 +12,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" "mesh-delivery-assigner/internal/assign" ) @@ -158,6 +159,30 @@ func TestTheSeatsVerbs(t *testing.T) { t.Errorf("the seat's verbs as served: %+v", tools) } t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs) + t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) { + findings := map[string][]byte{} + p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{ + Put: func(k string, v []byte) error { findings[k] = v; return nil }, + Delete: func(k string) error { delete(findings, k); return nil }}} + failed, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"}, + Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Failed, Why: "the role exists already"}) + p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", failed, 1, time.Now()) + p.kvs[provisions.ProvisionsBucket] = &memKV{} + of := p.provisionsOf() + if got := of("photos")("ace"); got["ace.photos.db"].State != provisions.Failed { + t.Fatalf("provisions = %+v", got) + } + if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 { + t.Fatalf("findings = %v", findings) + } + provided, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"}, + Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Provided}) + p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", provided, 2, time.Now()) + p.provisionsOf() + if len(findings) != 0 { + t.Errorf("findings after provided = %v", findings) + } + }) } // wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index e1d9aad..788e658 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 685613a..37f9598 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd h1:WhqMEXFZ9Zc+fEfLsnf1jhhGkc4IqMwUpXXwZq+V3+E= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212112-2d2aa7ba9bbd/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2 h1:tQZYFKHd378XTF6OPGncIJnapxQCtRJAtL2FvlIzNQ4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213741-6e39ed1852b2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/policy.go b/modules/mesh-delivery-assigner/internal/assign/policy.go index 287f8df..6b938a9 100644 --- a/modules/mesh-delivery-assigner/internal/assign/policy.go +++ b/modules/mesh-delivery-assigner/internal/assign/policy.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one @@ -133,3 +134,30 @@ func clipHeadline(s string) string { } return string([]rune(s)[:60]) } + +// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4). +const KindProvisionFailed = "provision-failed" + +// ProvisionFailedKey is a failed provision's finding's key, `provider...[.].provision-failed`. +func ProvisionFailedKey(key string) string { + return health.FindingKey(health.ScopeProvider, key, KindProvisionFailed) +} + +// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why. +func ProvisionFailedFinding(key string, p provisions.Provision, at time.Time) ([]byte, error) { + summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision, + p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module) + return health.EncodeFinding(health.Finding{Key: ProvisionFailedKey(key), Kind: KindProvisionFailed, + Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node)}, + Severity: health.Warning, Summary: summary, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"), + Explanation: asks.NothingToDo + " " + summary + ". It clears once the provider provides it.", + Resolved: p.Provision + " for " + p.Consumer.Module + " is provided", Evidence: []health.Evidence{{At: at.UTC(), Said: p.Why}}, + Source: "assigner." + KindProvisionFailed, Observer: delivery.AssignerModule, At: at.UTC()}) +} + +func also(provider, consumer string) []string { + if provider == "" || provider == consumer { + return nil + } + return []string{provider} +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index d839b93..b2ab36d 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the @@ -29,6 +30,9 @@ type Module struct { // 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone. GrantedOn func(node string) bool Manifest catalogue.Manifest + // Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want + // waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it. + Provisions func(node string) map[string]provisions.Provision // Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none. Chosen *catalogue.PersonPolicy Nodes []string @@ -126,9 +130,32 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { if _, ok := m.composition(node, m.FP); !ok { return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP)) } + if key, p, waiting := m.provisionWaiting(node); waiting { + words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State) + if p.Why != "" { + words += ": " + p.Why + } + return false, delivery.RuleProvisions, words + } return true, 0, "" } +// provisionWaiting is the first provision of the module on node, by key, that is not provided (ADR 0570 §4). A provision +// the holder has not composed a key for yet holds nothing: the holder composes each key ahead of the want, from placement +// and the newest build. +func (m Module) provisionWaiting(node string) (string, provisions.Provision, bool) { + if m.Provisions == nil { + return "", provisions.Provision{}, false + } + keyed := m.Provisions(node) + for _, key := range slices.Sorted(maps.Keys(keyed)) { + if p := keyed[key]; p.State != provisions.Provided { + return key, p, true + } + } + return "", provisions.Provision{}, false +} + // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned // elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node // offers it) and holds. diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index 3a6bb0c..a6e0191 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -8,6 +8,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" ) func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) } @@ -69,6 +70,12 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { delivery.RuleComposed: func(m *Module, _ *Nodes) { m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false } }, + // 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want. + delivery.RuleProvisions: func(m *Module, _ *Nodes) { + m.Provisions = func(node string) map[string]provisions.Provision { + return map[string]provisions.Provision{node + ".mail.db": {Provision: "db", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}} + } + }, } { m, n := ready() breakIt(&m, &n) @@ -82,6 +89,20 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) { t.Errorf("rule %d broken: g14's want is %+v", rule, w) } + if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.db") || !strings.Contains(out.Rules["g14"].Words, "asked")) { + t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words) + } + } + // 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it. + for _, all := range []func(string) map[string]provisions.Provision{ + func(node string) map[string]provisions.Provision { + return map[string]provisions.Provision{node + ".mail.db": {Provision: "db", State: provisions.Provided}} + }, nil} { + m, n := ready() + m.Provisions = all + if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].FP != fNew { + t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words) + } } // 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the // provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 6f0c3ca..0003bae 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -35,7 +35,7 @@ func TestManifest(t *testing.T) { slices.Sort(reads) want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "node-engine.have"} + "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"} if !reflect.DeepEqual(reads, want) { t.Errorf("reads = %v, want exactly %v", reads, want) } diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 74542b6..ecbdc4b 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -41,6 +41,7 @@ "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", + "mesh-provisioner.provisions", "node-engine.have" ], "uses": [