From af487d38ede1fbe2016d6b3f21b8d58fdf0105b8 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 20:04:06 +0100 Subject: [PATCH] mesh-vault: adopt from the bus node's kept have when the placement names it A have of the bus module heard while the placement did not yet name its node as holding the bus adopted nothing, and nothing adopted it again until the have changed, so a vault could stay without the bus node's adoption and the loaded list's revision for good. The vault now keeps each node's latest have and, on the placement's next change, adopts from the bus node's kept have if it was not adopted: the placement's event, no timer. The rotation test now writes the have before the placement (its wait for the vault's picture is gone), so it takes this path every run. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- modules/mesh-vault/cmd/mesh-vault/adopt.go | 73 +++++++++++++++++++ modules/mesh-vault/cmd/mesh-vault/main.go | 7 +- .../cmd/mesh-vault/mint_integration_test.go | 40 ---------- .../cmd/mesh-vault/placementview.go | 8 +- .../cmd/mesh-vault/placementview_test.go | 2 +- .../mesh-vault/rotation_integration_test.go | 44 +++++------ .../mesh-vault/cmd/mesh-vault/tools_test.go | 22 ++++++ modules/mesh-vault/cmd/mesh-vault/vault.go | 3 + 8 files changed, 131 insertions(+), 68 deletions(-) diff --git a/modules/mesh-vault/cmd/mesh-vault/adopt.go b/modules/mesh-vault/cmd/mesh-vault/adopt.go index c274783d7..60129cf92 100644 --- a/modules/mesh-vault/cmd/mesh-vault/adopt.go +++ b/modules/mesh-vault/cmd/mesh-vault/adopt.go @@ -58,6 +58,79 @@ type adoptionKept struct { failed string // why the last look found nothing to adopt, or "" } +// heardHave is a node's latest have of the bus module, as the watch handed it, and whether it was adopted: a have heard +// while the placement does not yet name its node as holding the bus adopts nothing then, and the have may not change +// again for long, so it is adopted when the placement next changes (PlacementChanged), never on a timer. +type heardHave struct { + raw []byte + by string + adopted bool +} + +// HaveHeard keeps node's latest have of the bus module and adopts from it. +func (v *Vault) HaveHeard(node string, raw []byte, by string) (Adoption, error) { + v.mu.Lock() + if v.heard == nil { + v.heard = map[string]heardHave{} + } + v.heard[node] = heardHave{raw: raw, by: by} + v.mu.Unlock() + a, err := v.AdoptFrom(node, raw, by, false) + if err == nil { + v.markAdopted(node, raw) + } + return a, err +} + +// HaveGone forgets node's have of the bus module: it was deleted. +func (v *Vault) HaveGone(node string) { + v.mu.Lock() + delete(v.heard, node) + v.mu.Unlock() +} + +// PlacementChanged adopts from the kept have of the node the placement now names as holding the bus, when that have was +// not adopted yet: the placement's change is the event that makes it adoptable. +func (v *Vault) PlacementChanged() { + v.mu.Lock() + pending := 0 + for _, h := range v.heard { + if !h.adopted { + pending++ + } + } + v.mu.Unlock() + if pending == 0 { + return + } + bus, err := v.busNode() + if err != nil { + return + } + v.mu.Lock() + h, ok := v.heard[bus] + v.mu.Unlock() + if !ok || h.adopted { + return + } + by := h.by + ", taken again when the placement named " + bus + " as holding the bus" + if _, err := v.AdoptFrom(bus, h.raw, by, false); err != nil { + v.log("adopted nothing from %s.%s when the placement named it as holding the bus: %v", bus, BusModule, err) + return + } + v.markAdopted(bus, h.raw) +} + +// markAdopted marks node's kept have adopted, unless a later have replaced it meanwhile. +func (v *Vault) markAdopted(node string, raw []byte) { + v.mu.Lock() + defer v.mu.Unlock() + if h, ok := v.heard[node]; ok && string(h.raw) == string(raw) { + h.adopted = true + v.heard[node] = h + } +} + // AdoptNow is the tool account_adopt: the bus node's have read now and adopted, though it says the same as before. func (v *Vault) AdoptNow(caller string) (Adoption, error) { if v.Haves == nil { diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index 713b7956b..ebc3b88c5 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -122,7 +122,7 @@ func watchLeavers(env *Env) bool { if node, ok := leaverKey(key); ok { env.Vault.NodeLeft(node) } - }, func(key string, v json.RawMessage) error { + }, env.Vault.PlacementChanged, func(key string, v json.RawMessage) error { _, err := stdio.State(FindingsState).Put(key, v) return err }, @@ -174,11 +174,12 @@ func watchHandings(env *Env) bool { // it changes. var said string err := stdio.State(HaveRead).Watch("*."+BusModule, func(c stdio.StateChange) error { + node := strings.TrimSuffix(c.Key, "."+BusModule) if c.Op == "delete" || len(c.Value) == 0 { + env.Vault.HaveGone(node) return nil } - node := strings.TrimSuffix(c.Key, "."+BusModule) - _, err := env.Vault.AdoptFrom(node, c.Value, "the watch of "+HaveRead, false) + _, err := env.Vault.HaveHeard(node, c.Value, "the watch of "+HaveRead) if line := fmt.Sprint(err); err != nil && line != said { say("adopted nothing from %s: %v", c.Key, err) said = line diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go index a8295d826..8e42369c9 100644 --- a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go @@ -194,46 +194,6 @@ func (p *player) firstViewAnswered() { } } -// placementHeld waits until the vault's picture of the placement holds every key the placement's bucket holds now: -// the owner said each change, and the vault applied it (its first-views tool says the revision its picture holds). What -// the vault does on another state's change (a have) reads the placement from that picture, so a test writes that change -// only after this, never racing the owner's events. -func (p *player) placementHeld() { - p.t.Helper() - kv, err := p.js.KeyValue(p.ctx, "mesh-delivery-assigner_placement") - if err != nil { - p.t.Fatal(err) - } - keys, err := kv.Keys(p.ctx) - if err != nil { - p.t.Fatal(err) - } - var want uint64 - eventually(p.t, "the owner said every key of the placement", func() bool { - p.placeMu.Lock() - defer p.placeMu.Unlock() - for _, k := range keys { - if _, ok := p.placeValues[k]; !ok { - return false - } - } - want = p.placeRevision - return true - }) - eventually(p.t, "the vault's picture of the placement holds every key", func() bool { - raw, err := p.call(FirstViewsTool, "person.jochen", nil) - if err != nil { - return false - } - var a struct { - FirstViews []struct { - Revision uint64 `json:"revision"` - } `json:"first-views"` - } - return json.Unmarshal(raw, &a) == nil && len(a.FirstViews) == 1 && a.FirstViews[0].Revision >= want - }) -} - // bucketOf is the bucket a state names: the vault's own by its local name, another's as `.`. func bucketOf(state string) string { if owner, name, ok := strings.Cut(state, "."); ok { diff --git a/modules/mesh-vault/cmd/mesh-vault/placementview.go b/modules/mesh-vault/cmd/mesh-vault/placementview.go index d69e6bf1b..35b4effc1 100644 --- a/modules/mesh-vault/cmd/mesh-vault/placementview.go +++ b/modules/mesh-vault/cmd/mesh-vault/placementview.go @@ -34,8 +34,9 @@ const FindingsState = "findings" // placementView is the picture; nil until main begins it. var placementView *views.Follower -// newPlacementView makes the follower: ask calls the tool, gone is told of a deleted key. -func newPlacementView(ask func(key string, body any) (json.RawMessage, error), gone func(key string), +// newPlacementView makes the follower: ask calls the tool, gone is told of a deleted key, changed of every change after +// it is held (a first view's keys too). +func newPlacementView(ask func(key string, body any) (json.RawMessage, error), gone func(key string), changed func(), put func(key string, value json.RawMessage) error, drop func(key string) error) *views.Follower { return &views.Follower{Owner: delivery.AssignerModule, Tool: PlacementTool, State: "placement", Logf: say, Ask: func(_ context.Context, after string) (views.Answer, error) { @@ -69,6 +70,9 @@ func newPlacementView(ask func(key string, body any) (json.RawMessage, error), g if deleted { gone(key) } + if changed != nil { + changed() + } }} } diff --git a/modules/mesh-vault/cmd/mesh-vault/placementview_test.go b/modules/mesh-vault/cmd/mesh-vault/placementview_test.go index 99d26a8ca..d7132f10f 100644 --- a/modules/mesh-vault/cmd/mesh-vault/placementview_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/placementview_test.go @@ -30,7 +30,7 @@ func TestAnUnansweredFirstViewOfThePlacementIsAFindingTheAnswerClears(t *testing } put := func(k string, v json.RawMessage) error { mu.Lock(); held[k] = v; mu.Unlock(); return nil } drop := func(k string) error { mu.Lock(); delete(held, k); mu.Unlock(); return nil } - f := newPlacementView(ask, func(string) {}, put, drop) + f := newPlacementView(ask, func(string) {}, nil, put, drop) f.Bound, f.Wait = 30*time.Millisecond, 10*time.Millisecond reader := viewPlacement(func() *views.Follower { return f }) if _, err := reader.Keys(); err == nil { diff --git a/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go index b1bfaaf2f..6972ba798 100644 --- a/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go @@ -91,28 +91,6 @@ func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) { p.followPlacement() go p.serve(stdout) - // The bus node (novox, running nats) and the node that rotates (g14); no lease key anywhere (issue 1126: the controller - // ended by its unassignment); g14's login adopted from - // the loaded list, its password known to the engine. Written after the vault's first view of the placement, so they - // reach it only as the owner's views events, whichever the vault would otherwise have read first. - p.firstViewAnswered() - for node, modules := range map[string][]string{"novox": {"nats"}, "g14": {"web"}} { - pl := delivery.Placement{V: delivery.Version, Node: node} - for _, m := range modules { - pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m}) - } - values, err := delivery.PlacementValues(pl) - if err != nil { - t.Fatal(err) - } - for k, raw := range values { - kv, _ := p.kv("mesh-delivery-assigner.placement") - if _, err := kv.Put(ctx, k, raw); err != nil { - t.Fatal(err) - } - } - } - p.placementHeld() oldPassword := "the-old-password-of-g14" oldHash, _ := bcrypt.GenerateFromPassword([]byte(oldPassword), bcrypt.MinCost) p.put(t, "mesh-vault.bus-accounts", "node.g14", AccountHash{User: "node.g14", Hash: string(oldHash), Adopted: true}) @@ -176,6 +154,28 @@ func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) { return strings.Contains(strings.Join(p.said, "\n"), event+" ") } reload() + // The bus node (novox, running nats) and the node that rotates (g14); no lease key anywhere (issue 1126: the controller + // ended by its unassignment); g14's login adopted from + // the loaded list, its password known to the engine. Written after the vault's first view of the placement, so they + // reach it only as the owner's views events, and after the bus node's have: the vault heard that have while no node + // held the bus, and adopts from it (learning the loaded list's revision) when the placement names novox. + p.firstViewAnswered() + for node, modules := range map[string][]string{"novox": {"nats"}, "g14": {"web"}} { + pl := delivery.Placement{V: delivery.Version, Node: node} + for _, m := range modules { + pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m}) + } + values, err := delivery.PlacementValues(pl) + if err != nil { + t.Fatal(err) + } + for k, raw := range values { + kv, _ := p.kv("mesh-delivery-assigner.placement") + if _, err := kv.Put(ctx, k, raw); err != nil { + t.Fatal(err) + } + } + } authenticates("before the rotation") enginePub, enginePriv := aSealingKey(t) diff --git a/modules/mesh-vault/cmd/mesh-vault/tools_test.go b/modules/mesh-vault/cmd/mesh-vault/tools_test.go index 89a1194e3..4d48b4ed2 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools_test.go @@ -318,6 +318,28 @@ func TestSecretOwnListsFingerprintsOfOwnSecretsOnly(t *testing.T) { if _, err := v.AdoptFrom("ace", have(), "the test", true); err == nil || !strings.Contains(err.Error(), "does not hold the bus") { t.Errorf("a handing from a node not holding the bus: %v", err) } + // A have heard while the placement names no node holding the bus adopts nothing then; it is kept, and adopted when + // the placement next changes to name its node, once (the placement's event, no timer). + assignedKeys := v.Assignments + v.Assignments = Mirror{Keys: func() ([]string, error) { return []string{"ace.dunst"}, nil }} + handing["node.ace"] = h("k") + if _, err := v.HaveHeard("novox", have(), "the watch"); err == nil || !strings.Contains(err.Error(), "no node is") { + t.Fatalf("a have heard before the placement names the bus node: %v", err) + } + v.PlacementChanged() + if acc.keys["node.ace"].Hash != h("m") { + t.Fatalf("adopted while no node holds the bus: %+v", acc.keys["node.ace"]) + } + v.Assignments = assignedKeys + before := len(v.bus.events) + v.PlacementChanged() + if acc.keys["node.ace"].Hash != h("k") || eventsNamed(v, before, AccountHashedEvent) != 1 { + t.Fatalf("the kept have is not adopted when the placement names its node: %+v", acc.keys["node.ace"]) + } + v.PlacementChanged() + if eventsNamed(v, before, AccountHashedEvent) != 1 { + t.Errorf("the kept have was adopted again at a later placement change") + } listed, err := own.Run(map[string]any{}) if err != nil { t.Fatal(err) diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go index d9081224a..04db7f074 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -65,6 +65,9 @@ type Vault struct { hashes hashing // adoption is the last adoption (adopt.go), under mu. adoption adoptionKept + // heard is each node's latest have of the bus module the watch handed (adopt.go), under mu: one not adopted is + // adopted when the placement next changes. + heard map[string]heardHave // WantRead reads one wanted account afresh (issue-account): nil reads the wants as the watch handed them. WantRead func(user string) ([]byte, bool, error) // Placement reads the seat's placement (placed.go): where modules stand once the assignments are gone.