From c19d4516ef18b84b134fe0cbbd26b5a129d0a12b Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 17:23:55 +0200 Subject: [PATCH] Keep the builds every want names and each composition's provisioner plans; find the replaced controller in the build records (issue 1945) With no plan left to name a build or a send to keep the plans, a want names what a collection must keep, and the composition is where the provisioner's plans are composed for the assignments mirror (ADR 0539 clauses 1 and 5); the assignments no longer name a node's build, so the controller replaced is the newest earlier build of it. --- cmd/mesh-controller/composed_live.go | 70 ++++++++++++++++++++ cmd/mesh-controller/composed_plans_test.go | 29 ++++++++ cmd/mesh-controller/composed_start.go | 51 +++++++++----- cmd/mesh-controller/composed_start_test.go | 21 ++++++ cmd/mesh-controller/new_modules.go | 2 + internal/inventory/collection.go | 46 ++++++++++++- internal/inventory/collection_wanted_test.go | 37 +++++++++++ internal/inventory/nodes.go | 3 + internal/inventory/provisioning.go | 12 ++++ 9 files changed, 251 insertions(+), 20 deletions(-) create mode 100644 cmd/mesh-controller/composed_plans_test.go create mode 100644 internal/inventory/collection_wanted_test.go diff --git a/cmd/mesh-controller/composed_live.go b/cmd/mesh-controller/composed_live.go index 5ed9c1ce..15cbc799 100644 --- a/cmd/mesh-controller/composed_live.go +++ b/cmd/mesh-controller/composed_live.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "sort" @@ -79,6 +80,10 @@ func composeModules(ctx context.Context, open *stores, nodes []string) (map[stri errs = append(errs, fmt.Errorf("%s: %w", node, err)) continue } + // The provisioner's plans of the modules the node's composition hands to it, for the assignments mirror. + if err := open.inventory.RecordComposedPlans(ctx, node, plansKept(declared)); err != nil { + errs = append(errs, fmt.Errorf("%s: its provisioner's plans: %w", node, err)) + } comps, moved, err := compositionsOf(node, body, fps, manifests) if err != nil { errs = append(errs, fmt.Errorf("%s: %w", node, err)) @@ -196,3 +201,68 @@ func haveOn(ctx context.Context, js jetstream.JetStream, node, module string) (s } return h, true, nil } + +// wantedOnTheBus is every fingerprint a want names, as ` `: what the keep set keeps the builds of +// (ADR 0539 clause 1). Read from the assigner's bucket on each look; no bucket yet names none. +func wantedOnTheBus(ctx context.Context) (map[string]bool, error) { + bus, err := aBus() + if err != nil { + return nil, err + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return nil, err + } + kv, err := js.KeyValue(ctx, sdkdelivery.AssignerWantsBucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return map[string]bool{}, nil + } + if err != nil { + return nil, err + } + lister, err := kv.ListKeys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return map[string]bool{}, nil + } + if err != nil { + return nil, err + } + out := map[string]bool{} + for key := range lister.Keys() { + _, module, err := sdkdelivery.ParseWantKey(key) + if err != nil { + continue + } + e, err := kv.Get(ctx, key) + if err != nil { + continue + } + w, err := sdkdelivery.DecodeWant(e.Value()) + if err != nil || w.FP == "" { + continue + } + out[module+" "+w.FP] = true + } + return out, nil +} + +// plansKept is a composition's provisioner's plans as the store keeps them: by module, each plan and what requires a +// module only pulled in by another (design 57 P6, issue 765); nil when it hands none. +func plansKept(d sendable) []byte { + if len(d.plans) == 0 { + return nil + } + kept := map[string]composedPlan{} + for module, p := range d.plans { + kept[module] = composedPlan{Plan: p, RequiredBy: d.requiredBy[module]} + } + raw, _ := json.Marshal(kept) // plain strings and slices: it marshals + return raw +} + +// composedPlan is one handed module's plan as the store keeps it, the shape inventory.SentProvisioningPlans reads. +type composedPlan struct { + catalogue.Plan + RequiredBy string `json:"required-by,omitempty"` +} diff --git a/cmd/mesh-controller/composed_plans_test.go b/cmd/mesh-controller/composed_plans_test.go new file mode 100644 index 00000000..375b0d0e --- /dev/null +++ b/cmd/mesh-controller/composed_plans_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "encoding/json" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// 1945 (dev-r6-controller's finding): the provisioner's plans a composition hands are kept in the shape the +// assignments mirror reads (inventory.SentProvisioningPlans: each module's plan and what requires it), and a +// composition that hands none keeps none. +func TestAComposedNodesProvisionerPlansAreKept(t *testing.T) { + if plansKept(sendable{}) != nil { + t.Errorf("a composition that hands nothing kept plans") + } + d := sendable{plans: map[string]catalogue.Plan{"umami": {Placement: map[string]string{"dir:state": "/var/lib/umami"}}}, + requiredBy: map[string]string{"umami": "analytics"}} + var kept map[string]struct { + Placement map[string]string `json:"placement"` + RequiredBy string `json:"required-by"` + } + if err := json.Unmarshal(plansKept(d), &kept); err != nil { + t.Fatal(err) + } + if kept["umami"].Placement["dir:state"] != "/var/lib/umami" || kept["umami"].RequiredBy != "analytics" { + t.Errorf("kept %+v", kept) + } +} diff --git a/cmd/mesh-controller/composed_start.go b/cmd/mesh-controller/composed_start.go index 4e4947e3..18838eb0 100644 --- a/cmd/mesh-controller/composed_start.go +++ b/cmd/mesh-controller/composed_start.go @@ -15,6 +15,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -109,15 +110,15 @@ func composeAtStart(ctx context.Context, open *stores) { } } -// replacedController is the controller a control-node ran before this one: the build the mesh last sent it and that -// build's fingerprint and manifest. +// replacedController is the controller a control-node ran before this one: its fingerprint and manifest. type replacedController struct { node, fp string manifest catalogue.Manifest } -// replacedControllers are, for each node the controller is assigned to, the controller build it was last sent when its -// fingerprint is of the new form and not the one the controller is registered at now. +// replacedControllers are, for each node the controller is assigned to, the controller it replaced: the newest build of +// the controller that worked, whose fingerprint is of the new form and not the one the controller is registered at now +// (the assignments name no build since ADR 0539 §12, so the build records say it). None when there is no such build. func replacedControllers(ctx context.Context, open *stores) ([]replacedController, error) { inv := open.inventory fps, err := currentFingerprints(ctx, inv) @@ -134,26 +135,40 @@ func replacedControllers(ctx context.Context, open *stores) ([]replacedControlle on = e.On } } + if len(on) == 0 { + return nil, nil + } + builds, err := inv.Builds(ctx, controllerModule, 20) + if err != nil { + return nil, err + } + before, found := replacedBuild(builds, fps[controllerModule]) + if !found { + return nil, nil + } + var m catalogue.Manifest + if err := json.Unmarshal(before.Manifest, &m); err != nil { + return nil, fmt.Errorf("the manifest of %s's build %s: %w", controllerModule, before.ID, err) + } var out []replacedController for _, node := range on { - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil || !known || sent[controllerModule] == "" { - continue - } - build, found, err := inv.BuildByID(ctx, sent[controllerModule]) - if err != nil || !found || build.SourceFingerprint == fps[controllerModule] || - !strings.HasPrefix(build.SourceFingerprint, sdkdelivery.SourcePrefix2) || len(build.Manifest) == 0 { - continue - } - var m catalogue.Manifest - if err := json.Unmarshal(build.Manifest, &m); err != nil { - return nil, fmt.Errorf("the manifest of %s's build %s: %w", controllerModule, build.ID, err) - } - out = append(out, replacedController{node: node, fp: build.SourceFingerprint, manifest: m}) + out = append(out, replacedController{node: node, fp: before.SourceFingerprint, manifest: m}) } return out, nil } +// replacedBuild is, of a module's builds newest first, the newest that worked, kept its manifest, and names a +// fingerprint of the new form other than current. +func replacedBuild(builds []inventory.Build, current string) (inventory.Build, bool) { + for _, b := range builds { + if b.Worked() && b.SourceFingerprint != current && strings.HasPrefix(b.SourceFingerprint, sdkdelivery.SourcePrefix2) && + len(b.Manifest) > 0 { + return b, true + } + } + return inventory.Build{}, false +} + // composeReplacedController writes, on each control-node, the controller's composition at the fingerprint of the one // it replaced: the node composed with that build's manifest in place of the controller's registered one, in the // reading mode (nothing made), and only the controller's composition of it kept. diff --git a/cmd/mesh-controller/composed_start_test.go b/cmd/mesh-controller/composed_start_test.go index 2e3393d6..ef36ea16 100644 --- a/cmd/mesh-controller/composed_start_test.go +++ b/cmd/mesh-controller/composed_start_test.go @@ -12,6 +12,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" ) // fakeBuckets is the bus's buckets by name, for deleting. @@ -70,3 +71,23 @@ func TestTheReplacedControllerIsComposedFromItsOwnManifest(t *testing.T) { t.Errorf("composing the replaced controller changed the Input's catalogue") } } + +// 1945/AC6 (unit half): the controller replaced is the newest earlier build of the controller that worked and names a +// fingerprint of the new form, not the one registered now. +func TestTheReplacedControllerIsItsNewestEarlierBuild(t *testing.T) { + now, before, older := sdkdelivery.SourcePrefix2+strings.Repeat("6", 64), sdkdelivery.SourcePrefix2+strings.Repeat("5", 64), + sdkdelivery.SourcePrefix2+strings.Repeat("4", 64) + m := []byte(`{"module":"mesh-controller","version":"5"}`) + builds := []inventory.Build{ + {ID: "c6", SourceFingerprint: now, Manifest: m}, + {ID: "c5-failed", SourceFingerprint: before, Failed: "no", Manifest: m}, + {ID: "c5", SourceFingerprint: before, Manifest: m}, + {ID: "c4", SourceFingerprint: older, Manifest: m}, + } + if b, ok := replacedBuild(builds, now); !ok || b.ID != "c5" { + t.Errorf("the replaced build is %+v (%v), want c5", b, ok) + } + if _, ok := replacedBuild(builds[:1], now); ok { + t.Errorf("a controller with no earlier build replaced one") + } +} diff --git a/cmd/mesh-controller/new_modules.go b/cmd/mesh-controller/new_modules.go index 0f66ef6a..0273ff3d 100644 --- a/cmd/mesh-controller/new_modules.go +++ b/cmd/mesh-controller/new_modules.go @@ -209,6 +209,8 @@ func watchNewModules(ctx context.Context, open *stores) { inBackground(func() { composeAtStart(ctx, open) }) watchDoneKeys(ctx, open) watchHavesForJudgement(ctx, open) + // And the keep set keeps the builds every want names (ADR 0539 clause 1), with no plan left to name them. + open.inventory.SetWanted(wantedOnTheBus) keys, closeKeys, err := openBuilderKeys() if err != nil { fmt.Printf("new modules: the builder's keys are not watched: %v\n", err) diff --git a/internal/inventory/collection.go b/internal/inventory/collection.go index d6272be0..cc6f1eb0 100644 --- a/internal/inventory/collection.go +++ b/internal/inventory/collection.go @@ -2,6 +2,7 @@ package inventory import ( "context" + "fmt" "slices" "sort" "strings" @@ -47,6 +48,9 @@ const ( KeptByRecentRecord = "recorded-lately" // KeptByOpenPlan: a plan that has not ended names the build that made it (novox/hq issue 1933). KeptByOpenPlan = "open-plan" + // KeptByWant: a want names the fingerprint of the build that made it, so a node may be told to run it (novox/hq + // ADR 0539 clause 1): with no plan, the want is what names a build a node runs or moves to. + KeptByWant = "wanted" ) // KeptLately is how long after a build is recorded its artifacts are kept whatever else the records say (novox/hq @@ -458,7 +462,8 @@ func keptLately(builds []Build, planned map[string]bool, now time.Time, within t return out } -// keptLatelyOrPlanned is keptLately of every build the records hold, (no plan names a build any more, ADR 0539 §17). +// keptLatelyOrPlanned is keptLately of every build the records hold, and the builds of every fingerprint a want names +// (no plan names a build any more, ADR 0539 §17; a want does, clause 1). func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]string, error) { c, err := i.theChain(ctx) if err != nil { @@ -468,7 +473,44 @@ func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]strin if err != nil { return nil, err } - return keptLately(all, nil, time.Now().UTC(), i.lately()), nil + out := keptLately(all, nil, time.Now().UTC(), i.lately()) + if i.wanted == nil { + return out, nil + } + wanted, err := i.wanted(ctx) + if err != nil { + return nil, fmt.Errorf("the fingerprints the wants name: %w", err) + } + for reference, whys := range keptByWants(all, wanted) { + for _, why := range whys { + if !slices.Contains(out[reference], why) { + out[reference] = append(out[reference], why) + } + } + } + return out, nil +} + +// keptByWants is, of these builds, the artifacts of each that worked and whose module and fingerprint a want names +// (` `), each kept as wanted. +func keptByWants(builds []Build, wanted map[string]bool) map[string][]string { + out := map[string][]string{} + for _, b := range builds { + if !b.Worked() || b.SourceFingerprint == "" || !wanted[b.Module+" "+b.SourceFingerprint] { + continue + } + for _, a := range b.Made { + if reference := asRecorded(a.Reference); reference != "" && !slices.Contains(out[reference], KeptByWant) { + out[reference] = append(out[reference], KeptByWant) + } + } + } + return out +} + +// SetWanted gives the keep set every fingerprint a want names, as ` ` (ADR 0539 clause 1). +func (i *Inventory) SetWanted(wanted func(ctx context.Context) (map[string]bool, error)) { + i.wanted = wanted } // digestIn is the `sha256:` a reference names, empty when it names none. diff --git a/internal/inventory/collection_wanted_test.go b/internal/inventory/collection_wanted_test.go new file mode 100644 index 00000000..05871bf0 --- /dev/null +++ b/internal/inventory/collection_wanted_test.go @@ -0,0 +1,37 @@ +package inventory + +import ( + "strings" + "testing" + "time" +) + +// 1945 (dev-r6-controller's finding, for ADR 0539 clause 1): with no plan left to name a build, the builds of every +// fingerprint a want names are kept: a collection never lets go of what a node is told to run. A build of a +// fingerprint no want names, one of another module at that fingerprint, and a failed one keep nothing this way. +func TestTheBuildsOfAWantedFingerprintAreKept(t *testing.T) { + fp := "src2:" + strings.Repeat("a", 64) + other := "src2:" + strings.Repeat("b", 64) + at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + made := func(id, module, fingerprint, reference string) Build { + return Build{ID: id, Module: module, SourceFingerprint: fingerprint, At: at, + Made: []Artifact{{Name: "files", Kind: "archive", Reference: reference}}} + } + const ( + wanted = "artifact-store://route-proxy/files/blobs/sha256:a1" + unwanted = "artifact-store://route-proxy/files/blobs/sha256:b2" + elsewise = "artifact-store://mesh-mail/files/blobs/sha256:c3" + ) + builds := []Build{made("key-a-1", "route-proxy", fp, wanted), made("key-b-1", "route-proxy", other, unwanted), + made("key-c-1", "mesh-mail", fp, elsewise), + {ID: "failed", Module: "route-proxy", SourceFingerprint: fp, Failed: "no", Made: []Artifact{{Reference: "artifact-store://f@sha256:ff"}}}} + kept := keptByWants(builds, map[string]bool{"route-proxy " + fp: true}) + if !has(kept[wanted], KeptByWant) { + t.Errorf("the archive of a wanted fingerprint is not kept: %v", kept) + } + for _, r := range []string{unwanted, elsewise, "artifact-store://f@sha256:ff"} { + if len(kept[r]) > 0 { + t.Errorf("%s is kept as wanted: %v", r, kept[r]) + } + } +} diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index ff2a4995..90ee3113 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -43,6 +43,9 @@ type Inventory struct { // keptLately is how long a build's artifacts are kept after it is recorded (KeptLately, novox/hq issue 1933); zero // is KeptLately, negative none (ForTest: the tests of the keep set's other rules record their builds a moment apart). keptLately time.Duration + // wanted is every fingerprint a want names, as ` ` (novox/hq ADR 0539 clause 1; SetWanted): the + // builds of each are kept, so a collection never lets go of what a node is told to run. Nil names none. + wanted func(ctx context.Context) (map[string]bool, error) } // lately is how long a build's artifacts are kept after it is recorded, whatever else the records say. diff --git a/internal/inventory/provisioning.go b/internal/inventory/provisioning.go index 09d58bfd..3fdeb641 100644 --- a/internal/inventory/provisioning.go +++ b/internal/inventory/provisioning.go @@ -408,6 +408,18 @@ func sortedPlanKeys(plans map[string]Provisioning) []string { // (migration 0100) by module, and carried by the assignments mirror in each handed module's assignment. So the // client is handed the plan of the build its machine was sent, and a controller started afresh mirrors the same. +// RecordComposedPlans keeps the provisioner's plan of each module a node's composition hands to it, as JSON by module +// (`: plan and required-by`), or none: what SentProvisioningPlans reads for the assignments mirror (design 57 P6). +// With no send left, the composition is where the plans are composed (novox/hq ADR 0539 clause 5). +func (i *Inventory) RecordComposedPlans(ctx context.Context, node string, plans []byte) error { + var kept any + if len(plans) > 0 { + kept = plans + } + _, err := i.store.Pool().Exec(ctx, `update node set sent_provisioning = $2 where name = $1`, node, kept) + return err +} + // SentProvisioningPlans is every plan the machines were last sent, keyed `.`: what // SetProvisioningPlans is given. A plan of a module only pulled in by another is kept with what required it, and is // mirrored in a value of its own until a send drops it (issue 765). A plan of a module assigned when it was sent and