mesh-vault: tell where modules stand and which node holds the bus from the catalogue's placement first, so the vault does not depend on the controller's assignments for them (issue 774, ADR 0549 §3, §5)

modules/mesh-vault/cmd/mesh-vault/adopt.go

@@ -58,32 +58,6 @@ type adoptionKept struct {
 	failed string // why the last look found nothing to adopt, or ""
 }
 
-// busNode is the node the assignments name as holding the bus module: exactly one.
-func (v *Vault) busNode() (string, error) {
-	if v.Assignments.Keys == nil {
-		return "", errors.New("the vault cannot read the assignments, so it cannot tell which node holds the bus")
-	}
-	keys, err := v.Assignments.Keys()
-	if err != nil {
-		return "", fmt.Errorf("the assignments' keys could not be read: %w", err)
-	}
-	var nodes []string
-	for _, k := range keys {
-		if node, module, ok := strings.Cut(k, "."); ok && module == BusModule {
-			nodes = append(nodes, node)
-		}
-	}
-	sort.Strings(nodes)
-	switch len(nodes) {
-	case 1:
-		return nodes[0], nil
-	case 0:
-		return "", fmt.Errorf("no node is assigned the bus module %s", BusModule)
-	}
-	return "", fmt.Errorf("%d nodes are assigned the bus module %s (%s): the vault adopts from one alone", len(nodes),
-		BusModule, strings.Join(nodes, ", "))
-}
-
 // AdoptNow is the tool account_adopt: the bus node's have read now and adopted, though it says the same as before.
 func (v *Vault) AdoptNow(caller string) (Adoption, error) {
 	if v.Haves == nil {

modules/mesh-vault/cmd/mesh-vault/main.go

@@ -51,7 +51,8 @@ func build() *Env {
 	// Where it sees a consumer bind a provider (pair.go): the same mirror, every key.
 	env.Vault = &Vault{Store: store, Own: own, Askers: Declared{Read: readAssignment, Store: store},
 		Assignments: Mirror{Keys: assignmentKeys, Read: readAssignment}, Emit: emit, Ask: stdio.Ask,
-		Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, LeaseRead: readLease, WantRead: readWant}
+		Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, LeaseRead: readLease, WantRead: readWant,
+		Placement: placement}
 	return env
 }
 
@@ -114,6 +115,18 @@ func watchWants(env *Env) bool {
 	return true
 }
 
+// placement reads the indexer's placement through the runtime under the vault's own account (its manifest reads it).
+var placement = PlacementReader{
+	Keys: func() ([]string, error) { return stdio.State(PlacementRead).Keys() },
+	Read: func(key string) ([]byte, error) {
+		e, err := stdio.State(PlacementRead).Get(key)
+		if err != nil || e == nil {
+			return nil, err
+		}
+		return e.Value, nil
+	},
+}
+
 // readWant is one wanted account of the grants part, read afresh (issue-account).
 func readWant(user string) ([]byte, bool, error) {
 	e, err := stdio.State(WantedRead).Get(user)

modules/mesh-vault/cmd/mesh-vault/ownnode.go

@@ -84,17 +84,14 @@ func (v *Vault) credentialRefusal(node string, c CredentialAsked) string {
 		if c.Consumer != node {
 			return fmt.Sprintf("%s asks as a consumer on %s, and is not that machine", node, c.Consumer)
 		}
-		// The consuming module stands on the machine: read, never assumed (an assignment that cannot be read refuses).
-		read, why := v.assignmentReader(node)
-		if why != "" {
-			return why
-		}
-		raw, err := read(node + "." + c.ConsumerModule)
+		// The consuming module stands on the machine: read from the placement and the assignments, never assumed (one
+		// that cannot be read refuses).
+		stands, err := v.standsOn(node, c.ConsumerModule)
 		if err != nil {
-			return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err)
+			return fmt.Sprintf(notYetPrefix+"where %s stands could not be read: %v", c.ConsumerModule, err)
 		}
-		if raw == nil {
-			return fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", c.ConsumerModule, node)
+		if !stands {
+			return fmt.Sprintf("%s is not on %s: neither placed nor assigned there, nor required by a module there", c.ConsumerModule, node)
 		}
 		return ""
 	case "provider":

modules/mesh-vault/cmd/mesh-vault/placed.go

@@ -0,0 +1,134 @@
+package main
+
+// Where modules stand, from the catalogue's placement (novox/hq ADR 0549 §3, §5; ADR 0561; issue 774). The controller's
+// assignments mirror goes with its lease, so the vault reads which module stands on which node from the indexer's
+// placement first, and the assignments beside it while both stand: a module either names stands. The provisioner's plan,
+// which only an assignment's value carries, is read from the assignments still: its home after the lease is a subtask of
+// its own (issue 774).
+
+import (
+	"errors"
+	"fmt"
+	"slices"
+	"sort"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// PlacementRead is the indexer's placement, as the vault's manifest reads it.
+const PlacementRead = delivery.IndexerModule + "." + delivery.PlacementState
+
+// PlacementReader reads the indexer's placement: every key, and one key's value (nil when there is none).
+type PlacementReader struct {
+	Keys func() ([]string, error)
+	Read func(key string) ([]byte, error)
+}
+
+// placements is every node's placement, by node; none when the vault reads no placement.
+func (v *Vault) placements() (map[string]delivery.Placement, error) {
+	out := map[string]delivery.Placement{}
+	if v.Placement.Keys == nil || v.Placement.Read == nil {
+		return out, nil
+	}
+	keys, err := v.Placement.Keys()
+	if err != nil {
+		return nil, fmt.Errorf("the placement's keys could not be read: %w", err)
+	}
+	for _, k := range keys {
+		node, err := delivery.ParsePlacementKey(k)
+		if err != nil {
+			continue
+		}
+		raw, err := v.Placement.Read(k)
+		if err != nil {
+			return nil, fmt.Errorf("the placement of %s could not be read: %w", node, err)
+		}
+		if raw == nil {
+			continue
+		}
+		p, err := delivery.DecodePlacement(raw)
+		if err != nil || p.Node != node {
+			continue
+		}
+		out[node] = p
+	}
+	return out, nil
+}
+
+// placedOn says module is placed on node by the catalogue.
+func (v *Vault) placedOn(node, module string) (bool, error) {
+	if v.Placement.Read == nil {
+		return false, nil
+	}
+	key, err := delivery.PlacementKey(node)
+	if err != nil {
+		return false, err
+	}
+	raw, err := v.Placement.Read(key)
+	if err != nil || raw == nil {
+		return false, err
+	}
+	p, err := delivery.DecodePlacement(raw)
+	if err != nil {
+		return false, fmt.Errorf("the placement of %s is not readable: %w", node, err)
+	}
+	return slices.Contains(p.Assigned(), module), nil
+}
+
+// standsOn says module stands on node: placed there, or assigned there while the assignments stand. A read that fails is
+// an error, never "not there".
+func (v *Vault) standsOn(node, module string) (bool, error) {
+	placed, err := v.placedOn(node, module)
+	if err != nil || placed {
+		return placed, err
+	}
+	d, ok := v.Askers.(Declared)
+	if !ok || d.Read == nil {
+		if v.Placement.Read == nil {
+			return false, errors.New("the vault reads neither the placement nor the assignments")
+		}
+		return false, nil
+	}
+	raw, err := standing(d.Read)(node + "." + module)
+	return raw != nil, err
+}
+
+// busNode is the node running the bus's module: from the placement, and from the assignments while no placement names
+// one. The vault adopts from one node alone.
+func (v *Vault) busNode() (string, error) {
+	placed, err := v.placements()
+	if err != nil {
+		return "", err
+	}
+	var nodes []string
+	for node, p := range placed {
+		if slices.Contains(p.Assigned(), BusModule) {
+			nodes = append(nodes, node)
+		}
+	}
+	from := "placed"
+	if len(nodes) == 0 && v.Assignments.Keys != nil {
+		from = "assigned"
+		keys, err := v.Assignments.Keys()
+		if err != nil {
+			return "", fmt.Errorf("the assignments' keys could not be read: %w", err)
+		}
+		for _, k := range keys {
+			if node, module, ok := strings.Cut(k, "."); ok && module == BusModule {
+				nodes = append(nodes, node)
+			}
+		}
+	} else if len(nodes) == 0 && len(placed) == 0 {
+		return "", errors.New("the vault reads neither the placement nor the assignments, so it cannot tell which node holds the bus")
+	}
+	sort.Strings(nodes)
+	switch len(nodes) {
+	case 1:
+		return nodes[0], nil
+	case 0:
+		return "", fmt.Errorf("no node is %s the bus module %s", from, BusModule)
+	}
+	return "", fmt.Errorf("%d nodes are %s the bus module %s (%s): the vault adopts from one alone", len(nodes), from,
+		BusModule, strings.Join(nodes, ", "))
+}

modules/mesh-vault/cmd/mesh-vault/placed_test.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// placementsOf is the indexer's placement of each node, as the vault reads it.
+func placementsOf(t *testing.T, nodes map[string][]string) PlacementReader {
+	t.Helper()
+	values := map[string][]byte{}
+	for node, modules := range nodes {
+		p := delivery.Placement{V: delivery.Version, Node: node}
+		for _, m := range modules {
+			p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
+		}
+		raw, err := delivery.EncodePlacement(p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		key, _ := delivery.PlacementKey(node)
+		values[key] = raw
+	}
+	return PlacementReader{
+		Keys: func() ([]string, error) {
+			var out []string
+			for k := range values {
+				out = append(out, k)
+			}
+			return out, nil
+		},
+		Read: func(key string) ([]byte, error) { return values[key], nil },
+	}
+}
+
+// 774/AC11 (novox/hq ADR 0561, ADR 0549 §3, §5): with no assignments to read, the vault tells from the catalogue's
+// placement which node holds the bus and whether a consumer's module stands on its node; while both stand, a module
+// either names stands.
+func TestTheVaultReadsWhereModulesStandFromPlacement(t *testing.T) {
+	v := newVault(t, nil)
+	v.Placement = placementsOf(t, map[string][]string{"novox": {"nats", "web"}, "ace": {"shop"}})
+	if node, err := v.busNode(); err != nil || node != "novox" {
+		t.Errorf("the bus node from placement: %q, %v", node, err)
+	}
+	asked := func(node, module string) CredentialAsked {
+		return CredentialAsked{Role: "consumer", Provision: "postgres", Consumer: node, ConsumerModule: module, Provider: "novox"}
+	}
+	if why := v.credentialRefusal("ace", asked("ace", "shop")); why != "" {
+		t.Errorf("shop stands on ace by placement, refused: %s", why)
+	}
+	if why := v.credentialRefusal("ace", asked("ace", "web")); !strings.Contains(why, "is not on ace") {
+		t.Errorf("web is not placed on ace, and is answered %q", why)
+	}
+	// While both stand, an assignment still names a module placement does not yet.
+	v.Askers = Declared{Read: assignments{"ace.web": `{"node":"ace","module":"web"}`}.read, Store: v.Store}
+	if why := v.credentialRefusal("ace", asked("ace", "web")); why != "" {
+		t.Errorf("web assigned on ace, refused: %s", why)
+	}
+	// Two nodes placed with the bus: the vault adopts from neither.
+	v.Placement = placementsOf(t, map[string][]string{"novox": {"nats"}, "ace": {"nats"}})
+	if _, err := v.busNode(); err == nil || !strings.Contains(err.Error(), "2 nodes") {
+		t.Errorf("two bus nodes: %v", err)
+	}
+}

modules/mesh-vault/cmd/mesh-vault/vault.go

@@ -58,6 +58,8 @@ type Vault struct {
 	adoption adoptionKept
 	// WantRead reads one wanted account afresh (issue-account): nil reads the wants as the watch handed them.
 	WantRead func(user string) ([]byte, bool, error)
+	// Placement reads the catalogue's placement (placed.go): where modules stand once the assignments are gone.
+	Placement PlacementReader
 	// LeaseRead reads the controller's lease afresh before a mint (lease.go): nil reads it only as the watch hands it.
 	LeaseRead func() ([]byte, bool, error)
 	// minting is the switch on the lease and what the vault knows of the wanted bus accounts (mint.go).

modules/mesh-vault/cmd/mesh-vault/vault_test.go

@@ -131,10 +131,11 @@ func filesUnder(t *testing.T, dir string) map[string]string {
 
 // The manifest declares one bucket, and no secret is in it: the vault's only place for a secret is its own store on its
 // own disk (ADR 0349 §7, ADR 0201 §13). The one bucket is bus-accounts, the bcrypt hashes of the bus accounts it holds
-// (ADR 0547 clause 9), opened in accounts.go alone. It reads four buckets of others', and only reads them: the
+// (ADR 0547 clause 9), opened in accounts.go alone. It reads five buckets of others', and only reads them: the
 // controller's assignments, for what each node's plan names (askers.go, design 57 V4), the node-engines' haves, where
 // the bus node hands the loaded list's hashes sealed to the vault (adopt.go, ADR 0547 clause 9), the grants part's wanted
-// accounts and the controller's lease (mint.go, lease.go, ADR 0561 §2, §3), in main.go alone.
+// accounts and the controller's lease (mint.go, lease.go, ADR 0561 §2, §3), and the catalogue's placement (placed.go), in
+// main.go alone.
 func TestNoBucketHoldsASecret(t *testing.T) {
 	raw, err := os.ReadFile("../../module.json")
 	if err != nil {
@@ -148,7 +149,7 @@ func TestNoBucketHoldsASecret(t *testing.T) {
 		t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the one "+
 			"bucket is the accounts' hashes (ADR 0547 clause 9)", s)
 	}
-	if r, _ := json.Marshal(manifest["reads"]); string(r) != `["mesh-controller.assignments","node-engine.have","mesh-grants.accounts","mesh-controller.lease"]` {
+	if r, _ := json.Marshal(manifest["reads"]); string(r) != `["mesh-controller.assignments","node-engine.have","mesh-grants.accounts","mesh-controller.lease","mesh-delivery-indexer.placement"]` {
 		t.Errorf("the manifest reads %s: the vault reads the controller's assignments, the haves and the grants part's "+
 			"wanted accounts alone", r)
 	}
@@ -180,6 +181,9 @@ func TestNoBucketHoldsASecret(t *testing.T) {
 			// And the controller's lease, one read and one watch (ADR 0561 §3).
 			code = []byte(strings.Replace(string(code), "stdio.State(LeaseRead).Get(", "", 1))
 			code = []byte(strings.Replace(string(code), "stdio.State(LeaseRead).Watch(", "", 1))
+			// And the catalogue's placement, where modules stand (placed.go): its keys and one key, read.
+			code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Keys(", "", 1))
+			code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Get(", "", 1))
 		}
 		if m := bucket.Find(code); m != nil {
 			t.Errorf("%s opens a bucket (%s): the vault keeps every secret in its own store only", f, m)

modules/mesh-vault/module.json

@@ -22,7 +22,8 @@
     "mesh-controller.assignments",
     "node-engine.have",
     "mesh-grants.accounts",
-    "mesh-controller.lease"
+    "mesh-controller.lease",
+    "mesh-delivery-indexer.placement"
   ],
   "state": [
     {