From f1cdabbb0dcbb2b840a6f4f2029d4784e7044c94 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 22:45:18 +0100 Subject: [PATCH] =?UTF-8?q?mesh-vault:=20tell=20where=20modules=20stand=20?= =?UTF-8?q?and=20which=20node=20holds=20the=20bus=20from=20the=20catalogue?= =?UTF-8?q?'s=20placement=20first,=20so=20the=20vault=20does=20not=20depen?= =?UTF-8?q?d=20on=20the=20controller's=20assignments=20for=20them=20(issue?= =?UTF-8?q?=20774,=20ADR=200549=20=C2=A73,=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/mesh-vault/cmd/mesh-vault/adopt.go | 26 ---- modules/mesh-vault/cmd/mesh-vault/main.go | 15 +- modules/mesh-vault/cmd/mesh-vault/ownnode.go | 15 +- modules/mesh-vault/cmd/mesh-vault/placed.go | 134 ++++++++++++++++++ .../mesh-vault/cmd/mesh-vault/placed_test.go | 66 +++++++++ modules/mesh-vault/cmd/mesh-vault/vault.go | 2 + .../mesh-vault/cmd/mesh-vault/vault_test.go | 10 +- modules/mesh-vault/module.json | 3 +- 8 files changed, 231 insertions(+), 40 deletions(-) create mode 100644 modules/mesh-vault/cmd/mesh-vault/placed.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/placed_test.go diff --git a/modules/mesh-vault/cmd/mesh-vault/adopt.go b/modules/mesh-vault/cmd/mesh-vault/adopt.go index 7b80b103..5d005ca3 100644 --- a/modules/mesh-vault/cmd/mesh-vault/adopt.go +++ b/modules/mesh-vault/cmd/mesh-vault/adopt.go @@ -58,32 +58,6 @@ type adoptionKept struct { failed string // why the last look found nothing to adopt, or "" } -// busNode is the node the assignments name as holding the bus module: exactly one. -func (v *Vault) busNode() (string, error) { - if v.Assignments.Keys == nil { - return "", errors.New("the vault cannot read the assignments, so it cannot tell which node holds the bus") - } - keys, err := v.Assignments.Keys() - if err != nil { - return "", fmt.Errorf("the assignments' keys could not be read: %w", err) - } - var nodes []string - for _, k := range keys { - if node, module, ok := strings.Cut(k, "."); ok && module == BusModule { - nodes = append(nodes, node) - } - } - sort.Strings(nodes) - switch len(nodes) { - case 1: - return nodes[0], nil - case 0: - return "", fmt.Errorf("no node is assigned the bus module %s", BusModule) - } - return "", fmt.Errorf("%d nodes are assigned the bus module %s (%s): the vault adopts from one alone", len(nodes), - BusModule, strings.Join(nodes, ", ")) -} - // AdoptNow is the tool account_adopt: the bus node's have read now and adopted, though it says the same as before. func (v *Vault) AdoptNow(caller string) (Adoption, error) { if v.Haves == nil { diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index 85bf3ca7..a2f8fc10 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -51,7 +51,8 @@ func build() *Env { // Where it sees a consumer bind a provider (pair.go): the same mirror, every key. env.Vault = &Vault{Store: store, Own: own, Askers: Declared{Read: readAssignment, Store: store}, Assignments: Mirror{Keys: assignmentKeys, Read: readAssignment}, Emit: emit, Ask: stdio.Ask, - Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, LeaseRead: readLease, WantRead: readWant} + Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, LeaseRead: readLease, WantRead: readWant, + Placement: placement} return env } @@ -114,6 +115,18 @@ func watchWants(env *Env) bool { return true } +// placement reads the indexer's placement through the runtime under the vault's own account (its manifest reads it). +var placement = PlacementReader{ + Keys: func() ([]string, error) { return stdio.State(PlacementRead).Keys() }, + Read: func(key string) ([]byte, error) { + e, err := stdio.State(PlacementRead).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil + }, +} + // readWant is one wanted account of the grants part, read afresh (issue-account). func readWant(user string) ([]byte, bool, error) { e, err := stdio.State(WantedRead).Get(user) diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode.go b/modules/mesh-vault/cmd/mesh-vault/ownnode.go index b873dc44..e8a7224d 100644 --- a/modules/mesh-vault/cmd/mesh-vault/ownnode.go +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode.go @@ -84,17 +84,14 @@ func (v *Vault) credentialRefusal(node string, c CredentialAsked) string { if c.Consumer != node { return fmt.Sprintf("%s asks as a consumer on %s, and is not that machine", node, c.Consumer) } - // The consuming module stands on the machine: read, never assumed (an assignment that cannot be read refuses). - read, why := v.assignmentReader(node) - if why != "" { - return why - } - raw, err := read(node + "." + c.ConsumerModule) + // The consuming module stands on the machine: read from the placement and the assignments, never assumed (one + // that cannot be read refuses). + stands, err := v.standsOn(node, c.ConsumerModule) if err != nil { - return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err) + return fmt.Sprintf(notYetPrefix+"where %s stands could not be read: %v", c.ConsumerModule, err) } - if raw == nil { - return fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", c.ConsumerModule, node) + if !stands { + return fmt.Sprintf("%s is not on %s: neither placed nor assigned there, nor required by a module there", c.ConsumerModule, node) } return "" case "provider": diff --git a/modules/mesh-vault/cmd/mesh-vault/placed.go b/modules/mesh-vault/cmd/mesh-vault/placed.go new file mode 100644 index 00000000..b2532357 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/placed.go @@ -0,0 +1,134 @@ +package main + +// Where modules stand, from the catalogue's placement (novox/hq ADR 0549 §3, §5; ADR 0561; issue 774). The controller's +// assignments mirror goes with its lease, so the vault reads which module stands on which node from the indexer's +// placement first, and the assignments beside it while both stand: a module either names stands. The provisioner's plan, +// which only an assignment's value carries, is read from the assignments still: its home after the lease is a subtask of +// its own (issue 774). + +import ( + "errors" + "fmt" + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// PlacementRead is the indexer's placement, as the vault's manifest reads it. +const PlacementRead = delivery.IndexerModule + "." + delivery.PlacementState + +// PlacementReader reads the indexer's placement: every key, and one key's value (nil when there is none). +type PlacementReader struct { + Keys func() ([]string, error) + Read func(key string) ([]byte, error) +} + +// placements is every node's placement, by node; none when the vault reads no placement. +func (v *Vault) placements() (map[string]delivery.Placement, error) { + out := map[string]delivery.Placement{} + if v.Placement.Keys == nil || v.Placement.Read == nil { + return out, nil + } + keys, err := v.Placement.Keys() + if err != nil { + return nil, fmt.Errorf("the placement's keys could not be read: %w", err) + } + for _, k := range keys { + node, err := delivery.ParsePlacementKey(k) + if err != nil { + continue + } + raw, err := v.Placement.Read(k) + if err != nil { + return nil, fmt.Errorf("the placement of %s could not be read: %w", node, err) + } + if raw == nil { + continue + } + p, err := delivery.DecodePlacement(raw) + if err != nil || p.Node != node { + continue + } + out[node] = p + } + return out, nil +} + +// placedOn says module is placed on node by the catalogue. +func (v *Vault) placedOn(node, module string) (bool, error) { + if v.Placement.Read == nil { + return false, nil + } + key, err := delivery.PlacementKey(node) + if err != nil { + return false, err + } + raw, err := v.Placement.Read(key) + if err != nil || raw == nil { + return false, err + } + p, err := delivery.DecodePlacement(raw) + if err != nil { + return false, fmt.Errorf("the placement of %s is not readable: %w", node, err) + } + return slices.Contains(p.Assigned(), module), nil +} + +// standsOn says module stands on node: placed there, or assigned there while the assignments stand. A read that fails is +// an error, never "not there". +func (v *Vault) standsOn(node, module string) (bool, error) { + placed, err := v.placedOn(node, module) + if err != nil || placed { + return placed, err + } + d, ok := v.Askers.(Declared) + if !ok || d.Read == nil { + if v.Placement.Read == nil { + return false, errors.New("the vault reads neither the placement nor the assignments") + } + return false, nil + } + raw, err := standing(d.Read)(node + "." + module) + return raw != nil, err +} + +// busNode is the node running the bus's module: from the placement, and from the assignments while no placement names +// one. The vault adopts from one node alone. +func (v *Vault) busNode() (string, error) { + placed, err := v.placements() + if err != nil { + return "", err + } + var nodes []string + for node, p := range placed { + if slices.Contains(p.Assigned(), BusModule) { + nodes = append(nodes, node) + } + } + from := "placed" + if len(nodes) == 0 && v.Assignments.Keys != nil { + from = "assigned" + keys, err := v.Assignments.Keys() + if err != nil { + return "", fmt.Errorf("the assignments' keys could not be read: %w", err) + } + for _, k := range keys { + if node, module, ok := strings.Cut(k, "."); ok && module == BusModule { + nodes = append(nodes, node) + } + } + } else if len(nodes) == 0 && len(placed) == 0 { + return "", errors.New("the vault reads neither the placement nor the assignments, so it cannot tell which node holds the bus") + } + sort.Strings(nodes) + switch len(nodes) { + case 1: + return nodes[0], nil + case 0: + return "", fmt.Errorf("no node is %s the bus module %s", from, BusModule) + } + return "", fmt.Errorf("%d nodes are %s the bus module %s (%s): the vault adopts from one alone", len(nodes), from, + BusModule, strings.Join(nodes, ", ")) +} diff --git a/modules/mesh-vault/cmd/mesh-vault/placed_test.go b/modules/mesh-vault/cmd/mesh-vault/placed_test.go new file mode 100644 index 00000000..ef84dc12 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/placed_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// placementsOf is the indexer's placement of each node, as the vault reads it. +func placementsOf(t *testing.T, nodes map[string][]string) PlacementReader { + t.Helper() + values := map[string][]byte{} + for node, modules := range nodes { + p := delivery.Placement{V: delivery.Version, Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + raw, err := delivery.EncodePlacement(p) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.PlacementKey(node) + values[key] = raw + } + return PlacementReader{ + Keys: func() ([]string, error) { + var out []string + for k := range values { + out = append(out, k) + } + return out, nil + }, + Read: func(key string) ([]byte, error) { return values[key], nil }, + } +} + +// 774/AC11 (novox/hq ADR 0561, ADR 0549 §3, §5): with no assignments to read, the vault tells from the catalogue's +// placement which node holds the bus and whether a consumer's module stands on its node; while both stand, a module +// either names stands. +func TestTheVaultReadsWhereModulesStandFromPlacement(t *testing.T) { + v := newVault(t, nil) + v.Placement = placementsOf(t, map[string][]string{"novox": {"nats", "web"}, "ace": {"shop"}}) + if node, err := v.busNode(); err != nil || node != "novox" { + t.Errorf("the bus node from placement: %q, %v", node, err) + } + asked := func(node, module string) CredentialAsked { + return CredentialAsked{Role: "consumer", Provision: "postgres", Consumer: node, ConsumerModule: module, Provider: "novox"} + } + if why := v.credentialRefusal("ace", asked("ace", "shop")); why != "" { + t.Errorf("shop stands on ace by placement, refused: %s", why) + } + if why := v.credentialRefusal("ace", asked("ace", "web")); !strings.Contains(why, "is not on ace") { + t.Errorf("web is not placed on ace, and is answered %q", why) + } + // While both stand, an assignment still names a module placement does not yet. + v.Askers = Declared{Read: assignments{"ace.web": `{"node":"ace","module":"web"}`}.read, Store: v.Store} + if why := v.credentialRefusal("ace", asked("ace", "web")); why != "" { + t.Errorf("web assigned on ace, refused: %s", why) + } + // Two nodes placed with the bus: the vault adopts from neither. + v.Placement = placementsOf(t, map[string][]string{"novox": {"nats"}, "ace": {"nats"}}) + if _, err := v.busNode(); err == nil || !strings.Contains(err.Error(), "2 nodes") { + t.Errorf("two bus nodes: %v", err) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go index 4707dff5..365683f7 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -58,6 +58,8 @@ type Vault struct { adoption adoptionKept // WantRead reads one wanted account afresh (issue-account): nil reads the wants as the watch handed them. WantRead func(user string) ([]byte, bool, error) + // Placement reads the catalogue's placement (placed.go): where modules stand once the assignments are gone. + Placement PlacementReader // LeaseRead reads the controller's lease afresh before a mint (lease.go): nil reads it only as the watch hands it. LeaseRead func() ([]byte, bool, error) // minting is the switch on the lease and what the vault knows of the wanted bus accounts (mint.go). diff --git a/modules/mesh-vault/cmd/mesh-vault/vault_test.go b/modules/mesh-vault/cmd/mesh-vault/vault_test.go index 0dfe93df..16059bea 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault_test.go @@ -131,10 +131,11 @@ func filesUnder(t *testing.T, dir string) map[string]string { // The manifest declares one bucket, and no secret is in it: the vault's only place for a secret is its own store on its // own disk (ADR 0349 §7, ADR 0201 §13). The one bucket is bus-accounts, the bcrypt hashes of the bus accounts it holds -// (ADR 0547 clause 9), opened in accounts.go alone. It reads four buckets of others', and only reads them: the +// (ADR 0547 clause 9), opened in accounts.go alone. It reads five buckets of others', and only reads them: the // controller's assignments, for what each node's plan names (askers.go, design 57 V4), the node-engines' haves, where // the bus node hands the loaded list's hashes sealed to the vault (adopt.go, ADR 0547 clause 9), the grants part's wanted -// accounts and the controller's lease (mint.go, lease.go, ADR 0561 §2, §3), in main.go alone. +// accounts and the controller's lease (mint.go, lease.go, ADR 0561 §2, §3), and the catalogue's placement (placed.go), in +// main.go alone. func TestNoBucketHoldsASecret(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -148,7 +149,7 @@ func TestNoBucketHoldsASecret(t *testing.T) { t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the one "+ "bucket is the accounts' hashes (ADR 0547 clause 9)", s) } - if r, _ := json.Marshal(manifest["reads"]); string(r) != `["mesh-controller.assignments","node-engine.have","mesh-grants.accounts","mesh-controller.lease"]` { + if r, _ := json.Marshal(manifest["reads"]); string(r) != `["mesh-controller.assignments","node-engine.have","mesh-grants.accounts","mesh-controller.lease","mesh-delivery-indexer.placement"]` { t.Errorf("the manifest reads %s: the vault reads the controller's assignments, the haves and the grants part's "+ "wanted accounts alone", r) } @@ -180,6 +181,9 @@ func TestNoBucketHoldsASecret(t *testing.T) { // And the controller's lease, one read and one watch (ADR 0561 §3). code = []byte(strings.Replace(string(code), "stdio.State(LeaseRead).Get(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(LeaseRead).Watch(", "", 1)) + // And the catalogue's placement, where modules stand (placed.go): its keys and one key, read. + code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Keys(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Get(", "", 1)) } if m := bucket.Find(code); m != nil { t.Errorf("%s opens a bucket (%s): the vault keeps every secret in its own store only", f, m) diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index a29c31ae..20d4f415 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -22,7 +22,8 @@ "mesh-controller.assignments", "node-engine.have", "mesh-grants.accounts", - "mesh-controller.lease" + "mesh-controller.lease", + "mesh-delivery-indexer.placement" ], "state": [ {