diff --git a/modules/mesh-vault/cmd/mesh-vault/adopt.go b/modules/mesh-vault/cmd/mesh-vault/adopt.go index c274783d7..60129cf92 100644 --- a/modules/mesh-vault/cmd/mesh-vault/adopt.go +++ b/modules/mesh-vault/cmd/mesh-vault/adopt.go @@ -58,6 +58,79 @@ type adoptionKept struct { failed string // why the last look found nothing to adopt, or "" } +// heardHave is a node's latest have of the bus module, as the watch handed it, and whether it was adopted: a have heard +// while the placement does not yet name its node as holding the bus adopts nothing then, and the have may not change +// again for long, so it is adopted when the placement next changes (PlacementChanged), never on a timer. +type heardHave struct { + raw []byte + by string + adopted bool +} + +// HaveHeard keeps node's latest have of the bus module and adopts from it. +func (v *Vault) HaveHeard(node string, raw []byte, by string) (Adoption, error) { + v.mu.Lock() + if v.heard == nil { + v.heard = map[string]heardHave{} + } + v.heard[node] = heardHave{raw: raw, by: by} + v.mu.Unlock() + a, err := v.AdoptFrom(node, raw, by, false) + if err == nil { + v.markAdopted(node, raw) + } + return a, err +} + +// HaveGone forgets node's have of the bus module: it was deleted. +func (v *Vault) HaveGone(node string) { + v.mu.Lock() + delete(v.heard, node) + v.mu.Unlock() +} + +// PlacementChanged adopts from the kept have of the node the placement now names as holding the bus, when that have was +// not adopted yet: the placement's change is the event that makes it adoptable. +func (v *Vault) PlacementChanged() { + v.mu.Lock() + pending := 0 + for _, h := range v.heard { + if !h.adopted { + pending++ + } + } + v.mu.Unlock() + if pending == 0 { + return + } + bus, err := v.busNode() + if err != nil { + return + } + v.mu.Lock() + h, ok := v.heard[bus] + v.mu.Unlock() + if !ok || h.adopted { + return + } + by := h.by + ", taken again when the placement named " + bus + " as holding the bus" + if _, err := v.AdoptFrom(bus, h.raw, by, false); err != nil { + v.log("adopted nothing from %s.%s when the placement named it as holding the bus: %v", bus, BusModule, err) + return + } + v.markAdopted(bus, h.raw) +} + +// markAdopted marks node's kept have adopted, unless a later have replaced it meanwhile. +func (v *Vault) markAdopted(node string, raw []byte) { + v.mu.Lock() + defer v.mu.Unlock() + if h, ok := v.heard[node]; ok && string(h.raw) == string(raw) { + h.adopted = true + v.heard[node] = h + } +} + // AdoptNow is the tool account_adopt: the bus node's have read now and adopted, though it says the same as before. func (v *Vault) AdoptNow(caller string) (Adoption, error) { if v.Haves == nil { diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index 713b7956b..ebc3b88c5 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -122,7 +122,7 @@ func watchLeavers(env *Env) bool { if node, ok := leaverKey(key); ok { env.Vault.NodeLeft(node) } - }, func(key string, v json.RawMessage) error { + }, env.Vault.PlacementChanged, func(key string, v json.RawMessage) error { _, err := stdio.State(FindingsState).Put(key, v) return err }, @@ -174,11 +174,12 @@ func watchHandings(env *Env) bool { // it changes. var said string err := stdio.State(HaveRead).Watch("*."+BusModule, func(c stdio.StateChange) error { + node := strings.TrimSuffix(c.Key, "."+BusModule) if c.Op == "delete" || len(c.Value) == 0 { + env.Vault.HaveGone(node) return nil } - node := strings.TrimSuffix(c.Key, "."+BusModule) - _, err := env.Vault.AdoptFrom(node, c.Value, "the watch of "+HaveRead, false) + _, err := env.Vault.HaveHeard(node, c.Value, "the watch of "+HaveRead) if line := fmt.Sprint(err); err != nil && line != said { say("adopted nothing from %s: %v", c.Key, err) said = line diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go index a8295d826..8e42369c9 100644 --- a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go @@ -194,46 +194,6 @@ func (p *player) firstViewAnswered() { } } -// placementHeld waits until the vault's picture of the placement holds every key the placement's bucket holds now: -// the owner said each change, and the vault applied it (its first-views tool says the revision its picture holds). What -// the vault does on another state's change (a have) reads the placement from that picture, so a test writes that change -// only after this, never racing the owner's events. -func (p *player) placementHeld() { - p.t.Helper() - kv, err := p.js.KeyValue(p.ctx, "mesh-delivery-assigner_placement") - if err != nil { - p.t.Fatal(err) - } - keys, err := kv.Keys(p.ctx) - if err != nil { - p.t.Fatal(err) - } - var want uint64 - eventually(p.t, "the owner said every key of the placement", func() bool { - p.placeMu.Lock() - defer p.placeMu.Unlock() - for _, k := range keys { - if _, ok := p.placeValues[k]; !ok { - return false - } - } - want = p.placeRevision - return true - }) - eventually(p.t, "the vault's picture of the placement holds every key", func() bool { - raw, err := p.call(FirstViewsTool, "person.jochen", nil) - if err != nil { - return false - } - var a struct { - FirstViews []struct { - Revision uint64 `json:"revision"` - } `json:"first-views"` - } - return json.Unmarshal(raw, &a) == nil && len(a.FirstViews) == 1 && a.FirstViews[0].Revision >= want - }) -} - // bucketOf is the bucket a state names: the vault's own by its local name, another's as `.`. func bucketOf(state string) string { if owner, name, ok := strings.Cut(state, "."); ok { diff --git a/modules/mesh-vault/cmd/mesh-vault/placementview.go b/modules/mesh-vault/cmd/mesh-vault/placementview.go index d69e6bf1b..35b4effc1 100644 --- a/modules/mesh-vault/cmd/mesh-vault/placementview.go +++ b/modules/mesh-vault/cmd/mesh-vault/placementview.go @@ -34,8 +34,9 @@ const FindingsState = "findings" // placementView is the picture; nil until main begins it. var placementView *views.Follower -// newPlacementView makes the follower: ask calls the tool, gone is told of a deleted key. -func newPlacementView(ask func(key string, body any) (json.RawMessage, error), gone func(key string), +// newPlacementView makes the follower: ask calls the tool, gone is told of a deleted key, changed of every change after +// it is held (a first view's keys too). +func newPlacementView(ask func(key string, body any) (json.RawMessage, error), gone func(key string), changed func(), put func(key string, value json.RawMessage) error, drop func(key string) error) *views.Follower { return &views.Follower{Owner: delivery.AssignerModule, Tool: PlacementTool, State: "placement", Logf: say, Ask: func(_ context.Context, after string) (views.Answer, error) { @@ -69,6 +70,9 @@ func newPlacementView(ask func(key string, body any) (json.RawMessage, error), g if deleted { gone(key) } + if changed != nil { + changed() + } }} } diff --git a/modules/mesh-vault/cmd/mesh-vault/placementview_test.go b/modules/mesh-vault/cmd/mesh-vault/placementview_test.go index 99d26a8ca..d7132f10f 100644 --- a/modules/mesh-vault/cmd/mesh-vault/placementview_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/placementview_test.go @@ -30,7 +30,7 @@ func TestAnUnansweredFirstViewOfThePlacementIsAFindingTheAnswerClears(t *testing } put := func(k string, v json.RawMessage) error { mu.Lock(); held[k] = v; mu.Unlock(); return nil } drop := func(k string) error { mu.Lock(); delete(held, k); mu.Unlock(); return nil } - f := newPlacementView(ask, func(string) {}, put, drop) + f := newPlacementView(ask, func(string) {}, nil, put, drop) f.Bound, f.Wait = 30*time.Millisecond, 10*time.Millisecond reader := viewPlacement(func() *views.Follower { return f }) if _, err := reader.Keys(); err == nil { diff --git a/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go index b1bfaaf2f..6972ba798 100644 --- a/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go @@ -91,28 +91,6 @@ func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) { p.followPlacement() go p.serve(stdout) - // The bus node (novox, running nats) and the node that rotates (g14); no lease key anywhere (issue 1126: the controller - // ended by its unassignment); g14's login adopted from - // the loaded list, its password known to the engine. Written after the vault's first view of the placement, so they - // reach it only as the owner's views events, whichever the vault would otherwise have read first. - p.firstViewAnswered() - for node, modules := range map[string][]string{"novox": {"nats"}, "g14": {"web"}} { - pl := delivery.Placement{V: delivery.Version, Node: node} - for _, m := range modules { - pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m}) - } - values, err := delivery.PlacementValues(pl) - if err != nil { - t.Fatal(err) - } - for k, raw := range values { - kv, _ := p.kv("mesh-delivery-assigner.placement") - if _, err := kv.Put(ctx, k, raw); err != nil { - t.Fatal(err) - } - } - } - p.placementHeld() oldPassword := "the-old-password-of-g14" oldHash, _ := bcrypt.GenerateFromPassword([]byte(oldPassword), bcrypt.MinCost) p.put(t, "mesh-vault.bus-accounts", "node.g14", AccountHash{User: "node.g14", Hash: string(oldHash), Adopted: true}) @@ -176,6 +154,28 @@ func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) { return strings.Contains(strings.Join(p.said, "\n"), event+" ") } reload() + // The bus node (novox, running nats) and the node that rotates (g14); no lease key anywhere (issue 1126: the controller + // ended by its unassignment); g14's login adopted from + // the loaded list, its password known to the engine. Written after the vault's first view of the placement, so they + // reach it only as the owner's views events, and after the bus node's have: the vault heard that have while no node + // held the bus, and adopts from it (learning the loaded list's revision) when the placement names novox. + p.firstViewAnswered() + for node, modules := range map[string][]string{"novox": {"nats"}, "g14": {"web"}} { + pl := delivery.Placement{V: delivery.Version, Node: node} + for _, m := range modules { + pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m}) + } + values, err := delivery.PlacementValues(pl) + if err != nil { + t.Fatal(err) + } + for k, raw := range values { + kv, _ := p.kv("mesh-delivery-assigner.placement") + if _, err := kv.Put(ctx, k, raw); err != nil { + t.Fatal(err) + } + } + } authenticates("before the rotation") enginePub, enginePriv := aSealingKey(t) diff --git a/modules/mesh-vault/cmd/mesh-vault/tools_test.go b/modules/mesh-vault/cmd/mesh-vault/tools_test.go index 89a1194e3..4d48b4ed2 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools_test.go @@ -318,6 +318,28 @@ func TestSecretOwnListsFingerprintsOfOwnSecretsOnly(t *testing.T) { if _, err := v.AdoptFrom("ace", have(), "the test", true); err == nil || !strings.Contains(err.Error(), "does not hold the bus") { t.Errorf("a handing from a node not holding the bus: %v", err) } + // A have heard while the placement names no node holding the bus adopts nothing then; it is kept, and adopted when + // the placement next changes to name its node, once (the placement's event, no timer). + assignedKeys := v.Assignments + v.Assignments = Mirror{Keys: func() ([]string, error) { return []string{"ace.dunst"}, nil }} + handing["node.ace"] = h("k") + if _, err := v.HaveHeard("novox", have(), "the watch"); err == nil || !strings.Contains(err.Error(), "no node is") { + t.Fatalf("a have heard before the placement names the bus node: %v", err) + } + v.PlacementChanged() + if acc.keys["node.ace"].Hash != h("m") { + t.Fatalf("adopted while no node holds the bus: %+v", acc.keys["node.ace"]) + } + v.Assignments = assignedKeys + before := len(v.bus.events) + v.PlacementChanged() + if acc.keys["node.ace"].Hash != h("k") || eventsNamed(v, before, AccountHashedEvent) != 1 { + t.Fatalf("the kept have is not adopted when the placement names its node: %+v", acc.keys["node.ace"]) + } + v.PlacementChanged() + if eventsNamed(v, before, AccountHashedEvent) != 1 { + t.Errorf("the kept have was adopted again at a later placement change") + } listed, err := own.Run(map[string]any{}) if err != nil { t.Fatal(err) diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go index d9081224a..04db7f074 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -65,6 +65,9 @@ type Vault struct { hashes hashing // adoption is the last adoption (adopt.go), under mu. adoption adoptionKept + // heard is each node's latest have of the bus module the watch handed (adopt.go), under mu: one not adopted is + // adopted when the placement next changes. + heard map[string]heardHave // WantRead reads one wanted account afresh (issue-account): nil reads the wants as the watch handed them. WantRead func(user string) ([]byte, bool, error) // Placement reads the seat's placement (placed.go): where modules stand once the assignments are gone.