mesh-contracts/provisioning/go/clients/meshprovisionerclient/provider/tables_test.go

package provider

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"slices"
	"strings"
	"testing"
	"time"

	provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/catalogue"
)

// TestEntries groups the cases below, each one a subtest.
func TestEntries(t *testing.T) {
	// A provider answers each consumer that binds it with an entry of its own in its own bucket: composed from its
	// served values and the consumer's identity, naming the credential the vault keeps for that consumer, and never
	// the credential itself; its adapter is handed each consumer's own value read from the vault.
	t.Run("EachConsumerGetsAnEntryOfItsOwnAndTheVaultsValue", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{
			"pair/novox/postgres-database/ace/letta": "letta-secret-1",
			"pair/novox/postgres-database/g14/umami": "umami-secret-2",
		}}
		assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
		assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres"))
		p := postgres(bus, v, m)
		out, err := p.Pass(context.Background())
		if err != nil {
			t.Fatal(err)
		}
		if got := bus.keys(BucketOf("postgres")); !slices.Equal(got, []string{"ace.letta.postgres-database",
			"g14.umami.postgres-database"}) {
			t.Fatalf("entries %v", got)
		}
		e := entryOf(t, bus, "ace.letta.postgres-database")
		if e.Provider != "novox" || e.ProviderModule != "postgres" || e.Consumer != "ace" || e.Module != "letta" ||
			e.As != "mesh_ace_letta" || e.User != "mesh_ace_letta" ||
			e.VaultName != "pair/novox/postgres-database/ace/letta" || e.Refused != "" {
			t.Fatalf("entry %+v", e)
		}
		if e.Values["database"] != "mesh_ace_letta" || e.Values["host"] != "postgres.novox.mesh" ||
			e.Values["port"] != float64(5432) {
			t.Fatalf("values %v", e.Values)
		}
		if m.made["ace.letta.postgres-database"] != "letta-secret-1" ||
			m.made["g14.umami.postgres-database"] != "umami-secret-2" {
			t.Fatalf("made %v", m.made)
		}
		if bytes.Contains(bus.everything(), []byte("secret-")) {
			t.Fatal("a credential is in a bucket")
		}
		for _, o := range out {
			if o.Status != Answered {
				t.Fatalf("outcome %+v", o)
			}
		}
	})

	// A provider whose offer shares one credential (ADR 0158) names the vault's one value for the provision, and its
	// one user name, to every consumer.
	t.Run("ASharedCredentialIsTheVaultsOneValueAndTheProvidersOneUser", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{"provision/novox/sonarr-api": "one-value"}}
		assign(t, bus, "ace", "overseerr", bind("sonarr-api", "", "novox", "sonarr"))
		assign(t, bus, "g14", "homepage", bind("sonarr-api", "", "novox", "sonarr"))
		p := New(Options{Node: "novox", Module: "sonarr", Bus: bus, Vault: v, Make: m.Make, Logf: func(string, ...any) {},
			Offers: map[string]Offer{"sonarr-api": {Credential: Shared, User: "sonarr",
				Serves: map[string]any{"url": "http://sonarr.novox.mesh:8989"}}}})
		if _, err := p.Pass(context.Background()); err != nil {
			t.Fatal(err)
		}
		for _, key := range []string{"ace.overseerr.sonarr-api", "g14.homepage.sonarr-api"} {
			var e Entry
			if err := json.Unmarshal(bus.get(BucketOf("sonarr"), key), &e); err != nil {
				t.Fatalf("%s: %v", key, err)
			}
			if e.VaultName != "provision/novox/sonarr-api" || e.User != "sonarr" || e.Values["url"] == nil {
				t.Fatalf("%s: %+v", key, e)
			}
			if m.made[key] != "one-value" {
				t.Fatalf("%s made with %q", key, m.made[key])
			}
		}
		if bytes.Contains(bus.everything(), []byte("one-value")) {
			t.Fatal("the shared credential is in a bucket")
		}
	})

	// A consumer that keeps several holders of one provision is served no derived value: its entry refuses it, naming
	// both ends, so its client can report it (ADR 0349 clause 12, ADR 0202 §10); nothing is made for it.
	t.Run("AMultiHolderConsumerIsRefusedADerivedValueNamingBothEnds", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta/archive": "x-secret"}}
		assign(t, bus, "ace", "letta", bind("postgres-database", "archive", "novox", "postgres"))
		out, err := postgres(bus, v, m).Pass(context.Background())
		if err != nil {
			t.Fatal(err)
		}
		e := entryOf(t, bus, "ace.letta.postgres-database.archive")
		for _, says := range []string{"letta on ace", "postgres on novox", "postgres-database", "archive"} {
			if !strings.Contains(e.Refused, says) {
				t.Fatalf("the refusal does not name %q: %s", says, e.Refused)
			}
		}
		if e.Values != nil || e.VaultName != "" {
			t.Fatalf("a refused entry carries values: %+v", e)
		}
		if m.calls != 0 {
			t.Fatal("the adapter made a refused consumer")
		}
		if len(out) != 1 || out[0].Status != Refused {
			t.Fatalf("outcomes %+v", out)
		}
	})

	// A consumer bound under a local name is served when the offer derives nothing from it: its credential is its own
	// pair row under that local name.
	t.Run("ALocalNameIsServedWhenNothingIsDerived", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{"pair/novox/redis-cache/ace/letta/queue": "q-secret"}}
		assign(t, bus, "ace", "letta", bind("redis-cache", "queue", "novox", "redis"))
		p := New(Options{Node: "novox", Module: "redis", Bus: bus, Vault: v, Make: m.Make, Logf: func(string, ...any) {},
			Offers: map[string]Offer{"redis-cache": {Credential: PerConsumer, Serves: map[string]any{"port": 6379}}}})
		if _, err := p.Pass(context.Background()); err != nil {
			t.Fatal(err)
		}
		var e Entry
		if err := json.Unmarshal(bus.get(BucketOf("redis"), "ace.letta.redis-cache.queue"), &e); err != nil {
			t.Fatal(err)
		}
		if e.Refused != "" || e.VaultName != "pair/novox/redis-cache/ace/letta/queue" || e.Local != "queue" {
			t.Fatalf("entry %+v", e)
		}
		if m.made["ace.letta.redis-cache.queue"] != "q-secret" {
			t.Fatalf("made %v", m.made)
		}
	})

	// A consumer's plan entry may carry what its module contributes to the provider (issue 776): the helper hands it to
	// the provider's adapter with the consumer, and does not write it to the bucket.
	t.Run("AConsumersContributionIsHandedToTheProviderPerConsumer", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{
			"pair/novox/postgres-database/ace/nextcloud": "a", "pair/novox/postgres-database/g14/umami": "b"}}
		var pp catalogue.PlanProvision
		if err := json.Unmarshal([]byte(`{"provision":"postgres-database","provider":"novox","provider-module":"postgres",`+
			`"contributes":{"name":"nextcloud","extensions":["pg_trgm"]}}`), &pp); err != nil {
			t.Fatal(err)
		}
		planned(t, bus, "ace", "nextcloud", pp)
		assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres"))
		if _, err := postgres(bus, v, m).Pass(context.Background()); err != nil {
			t.Fatal(err)
		}
		want := map[string]any{"name": "nextcloud", "extensions": []any{"pg_trgm"}}
		if got := m.contributed["ace.nextcloud.postgres-database"]; !reflect.DeepEqual(got, want) {
			t.Fatalf("the adapter was handed the contribution %v, want %v", got, want)
		}
		if got := m.contributed["g14.umami.postgres-database"]; got != nil {
			t.Fatalf("a consumer contributing nothing was handed %v", got)
		}
		if bytes.Contains(bus.get(BucketOf("postgres"), "ace.nextcloud.postgres-database"), []byte("pg_trgm")) {
			t.Fatal("a contribution is in the provider's bucket")
		}
	})

	// A consumer's plan entry carries where its provider is, which machine, and who the consumer is (issue 788): the
	// entry's values carry `at` and `from` as the binding file gives them, and its `as` is the plan's, so a module with a
	// slug keeps the login it has today. An entry of a plan without them is as it was.
	t.Run("AConsumersEntryCarriesAtFromAndTheAsOfItsPlan", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{
			"pair/novox/postgres-database/ace/claude-code": "a", "pair/novox/postgres-database/g14/umami": "b"}}
		var pp catalogue.PlanProvision
		if err := json.Unmarshal([]byte(`{"provision":"postgres-database","provider":"novox","provider-module":"postgres",`+
			`"at":"10.7.0.1","from":"novox","as":"mesh_ace_agent"}`), &pp); err != nil {
			t.Fatal(err)
		}
		planned(t, bus, "ace", "claude-code", pp)
		assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres"))
		if _, err := postgres(bus, v, m).Pass(context.Background()); err != nil {
			t.Fatal(err)
		}
		var e Entry
		if err := json.Unmarshal(bus.get(BucketOf("postgres"), "ace.claude-code.postgres-database"), &e); err != nil {
			t.Fatal(err)
		}
		if e.Values["at"] != "10.7.0.1" || e.Values["from"] != "novox" || e.Values["port"] == nil {
			t.Fatalf("the entry's values are %v", e.Values)
		}
		if e.As != "mesh_ace_agent" || e.User != "mesh_ace_agent" || m.as["ace.claude-code.postgres-database"] != "mesh_ace_agent" {
			t.Fatalf("the entry is %+v, made as %q", e, m.as["ace.claude-code.postgres-database"])
		}
		var plain Entry
		if err := json.Unmarshal(bus.get(BucketOf("postgres"), "g14.umami.postgres-database"), &plain); err != nil {
			t.Fatal(err)
		}
		if _, has := plain.Values["at"]; has || plain.As != "mesh_g14_umami" {
			t.Fatalf("an entry of a plan without them changed: %+v", plain)
		}
	})

	// A consumer's plan entry may carry the names the controller composed for it (issue 792): each is copied into the
	// entry's values under its key, over anything the provider serves under it. An entry without them is as it was.
	t.Run("AConsumersEntryCarriesTheNamesOfItsPlan", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{
			"pair/novox/postgres-database/ace/gitea": "a", "pair/novox/postgres-database/g14/umami": "b"}}
		var pp catalogue.PlanProvision
		if err := json.Unmarshal([]byte(`{"provision":"postgres-database","provider":"novox","provider-module":"postgres",`+
			`"names":{"name":"git.example.org","internal-name":"git.mesh","name-web":"web.example.org"}}`), &pp); err != nil {
			t.Fatal(err)
		}
		planned(t, bus, "ace", "gitea", pp)
		assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres"))
		if _, err := postgres(bus, v, m).Pass(context.Background()); err != nil {
			t.Fatal(err)
		}
		var e Entry
		if err := json.Unmarshal(bus.get(BucketOf("postgres"), "ace.gitea.postgres-database"), &e); err != nil {
			t.Fatal(err)
		}
		if e.Values["name"] != "git.example.org" || e.Values["internal-name"] != "git.mesh" ||
			e.Values["name-web"] != "web.example.org" || e.Values["port"] == nil {
			t.Fatalf("the entry's values are %v", e.Values)
		}
		var plain Entry
		if err := json.Unmarshal(bus.get(BucketOf("postgres"), "g14.umami.postgres-database"), &plain); err != nil {
			t.Fatal(err)
		}
		if _, has := plain.Values["name"]; has {
			t.Fatalf("an entry of a plan without names changed: %+v", plain)
		}
	})

	// The plan's credential form and holder name win over the offer's; the offer's form is the default (issue 825,
	// finding 4).
	t.Run("ThePlansCredentialFormWinsOverTheOffers", func(t *testing.T) {
		run := func(offer CredentialForm, plan map[string]string) Entry {
			bus, m := newStandIn(), &maker{}
			v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "p",
				"provision/novox/postgres-database": "s", "provision/novox/postgres-database/ace/letta": "h"}}
			b := bind("postgres-database", "", "novox", "postgres")
			for k, val := range plan {
				b[k] = val
			}
			assign(t, bus, "ace", "letta", b)
			p := New(Options{Node: "novox", Module: "postgres", Bus: bus, Vault: v, Make: m.Make, Logf: func(string, ...any) {},
				Offers: map[string]Offer{"postgres-database": {Credential: offer, User: "shared-user"}}})
			out, err := p.Pass(context.Background())
			if err != nil || len(out) != 1 || out[0].Status != Answered {
				t.Fatalf("%v %+v", err, out)
			}
			return entryOf(t, bus, "ace.letta.postgres-database")
		}
		if e := run(PerConsumer, map[string]string{"credential-form": "shared"}); e.VaultName !=
			"provision/novox/postgres-database" || e.User != "shared-user" {
			t.Fatalf("shared over pair: %+v", e)
		}
		if e := run(Shared, map[string]string{"credential-form": "pair"}); e.VaultName !=
			"pair/novox/postgres-database/ace/letta" || e.User != "mesh_ace_letta" {
			t.Fatalf("pair over shared: %+v", e)
		}
		for _, form := range []string{"issued", "given"} {
			if e := run(PerConsumer, map[string]string{"credential-form": form, "name": "main"}); e.VaultName !=
				"provision/novox/postgres-database/ace/letta" || e.User != "mesh_ace_letta" {
				t.Fatalf("%s holder: %+v", form, e)
			}
		}
		if e := run(Shared, nil); e.VaultName != "provision/novox/postgres-database" {
			t.Fatalf("the offer's form is the default: %+v", e)
		}
		{
			// An offer that declares no credential is answered with none whatever form the plan says (the controller writes
			// "pair" for such an offer): no vault name, and the Vault is never asked, so a provider with no Vault works (issue 825,
			// finding 4; issue 923).
			bus := newStandIn()
			b := bind("route", "", "novox", "route-proxy")
			b["credential-form"] = "pair"
			assign(t, bus, "ace", "letta", b)
			p := New(Options{Node: "novox", Module: "route-proxy", Bus: bus, Logf: func(string, ...any) {},
				Offers: map[string]Offer{"route": {Credential: NoCredential}}})
			out, err := p.Pass(context.Background())
			if err != nil || len(out) != 1 || out[0].Status != Answered {
				t.Fatalf("%v %+v", err, out)
			}
			var e Entry
			if err := json.Unmarshal(bus.get(BucketOf("route-proxy"), "ace.letta.route"), &e); err != nil || e.VaultName != "" ||
				e.User != "" {
				t.Fatalf("%v %+v", err, e)
			}
		}
	})

	// Issue 1126 review 1 (mcp-endpoint asked on ace and shanks, 2026-11-02 11:56): a consumer's key composed from its newest
	// build carries the plan entry it was composed from, though its wanted build's plan names no such provision; the
	// provider binds it from the key and provides it, so the want that waits for it moves. A key of this provider that
	// carries no entry and no plan names is said, never bound silently or left without a word.
	t.Run("AKeyNoPlanNamesIsBoundFromTheEntryItCarries", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "letta-secret-1"}}
		// letta's wanted plan names nothing of postgres; its key, from its newest build, carries the entry.
		planned(t, bus, "ace", "letta")
		entry := catalogue.PlanProvision{Provision: "postgres-database", Provider: "novox", ProviderModule: "postgres", As: "mesh_ace_letta"}
		pr := provisions.Provision{Provision: "postgres-database", Consumer: provisions.End{Module: "letta", Node: "ace"},
			Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Asked, Plan: &entry}
		raw, err := provisions.Encode(pr)
		if err != nil {
			t.Fatal(err)
		}
		_ = bus.Put(context.Background(), provisions.ProvisionsBucket, "ace.letta.postgres-database", raw)
		// And umami's key carries none: asked, but no plan names it.
		bare := provisions.Provision{Provision: "postgres-database", Consumer: provisions.End{Module: "umami", Node: "g14"},
			Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Asked}
		raw, _ = provisions.Encode(bare)
		_ = bus.Put(context.Background(), provisions.ProvisionsBucket, "g14.umami.postgres-database", raw)

		out, err := postgres(bus, v, m).Pass(context.Background())
		if err != nil {
			t.Fatal(err)
		}
		if e := entryOf(t, bus, "ace.letta.postgres-database"); e.Consumer != "ace" || e.Module != "letta" || e.Refused != "" {
			t.Errorf("the entry %+v", e)
		}
		said := map[string]Outcome{}
		for _, o := range out {
			said[o.Key] = o
		}
		if said["ace.letta.postgres-database"].Status != Answered {
			t.Errorf("letta: %+v", said["ace.letta.postgres-database"])
		}
		if o := said["g14.umami.postgres-database"]; o.Status != Waiting || !strings.Contains(o.Why, "no plan names it") {
			t.Errorf("umami: %+v", o)
		}
		if bus.get(BucketOf("postgres"), "g14.umami.postgres-database") != nil {
			t.Error("a key with no entry and no plan was bound")
		}
	})
}

// TestBinds groups the cases below, each one a subtest.
func TestBinds(t *testing.T) {
	// A consumer that unbinds loses its entry and the adapter hears it is gone; an entry another machine's instance of
	// the same provider wrote is not this one's to remove.
	t.Run("AnUnbindRemovesTheEntryAndLeavesAnotherProvidersAlone", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s1"}}
		assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
		other, _ := json.Marshal(Entry{Provider: "shanks", ProviderModule: "postgres", Provision: "postgres-database",
			Consumer: "g14", Module: "gitea"})
		_ = bus.Put(context.Background(), BucketOf("postgres"), "g14.gitea.postgres-database", other)
		p := postgres(bus, v, m)
		if _, err := p.Pass(context.Background()); err != nil {
			t.Fatal(err)
		}
		retire(t, bus, "ace", "letta", "postgres-database", "novox", "postgres")
		out, err := p.Pass(context.Background())
		if err != nil {
			t.Fatal(err)
		}
		if got := bus.keys(BucketOf("postgres")); !slices.Equal(got, []string{"g14.gitea.postgres-database"}) {
			t.Fatalf("entries left %v", got)
		}
		if !slices.Equal(m.gone, []string{"ace.letta.postgres-database"}) {
			t.Fatalf("gone %v", m.gone)
		}
		if len(out) != 1 || out[0].Status != Removed {
			t.Fatalf("outcomes %+v", out)
		}
	})

	// Only the binds naming this provider, on this machine, are its own; a bind naming it for a provision it does not
	// offer is refused, naming both ends.
	t.Run("OnlyBindsNamingThisProviderAreAnswered", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s"}}
		assign(t, bus, "ace", "letta", bind("postgres-database", "", "shanks", "postgres"),
			bind("mongodb-database", "", "novox", "postgres"))
		assign(t, bus, "ace", "other", bind("postgres-database", "", "novox", "mssql"))
		out, err := postgres(bus, v, m).Pass(context.Background())
		if err != nil {
			t.Fatal(err)
		}
		if got := bus.keys(BucketOf("postgres")); !slices.Equal(got, []string{"ace.letta.mongodb-database"}) {
			t.Fatalf("entries %v", got)
		}
		e := entryOf(t, bus, "ace.letta.mongodb-database")
		if !strings.Contains(e.Refused, "postgres on novox") || !strings.Contains(e.Refused, "letta on ace") {
			t.Fatalf("refusal %q", e.Refused)
		}
		if len(out) != 1 || m.calls != 0 {
			t.Fatalf("outcomes %+v, %d made", out, m.calls)
		}
	})

	// A plan that cannot be read is skipped and said, by its key and why; the rest of the pass goes on (issue 825,
	// finding 3). Its consumer's own entry is left alone, since a bind missed would read as an unbind.
	t.Run("AnUnreadableAssignmentIsSkippedAndSaidAndTheRestGoesOn", func(t *testing.T) {
		for name, bad := range map[string]string{"not json": "{not json",
			"another key": `{"v":1,"node":"g14","module":"umami","plan":{"templates":{"source":"","digest":""},"files":null}}`} {
			bus, m := newStandIn(), &maker{}
			v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s",
				"pair/novox/postgres-database/g14/umami": "u"}}
			assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
			p := postgres(bus, v, m)
			if _, err := p.Pass(context.Background()); err != nil {
				t.Fatal(err)
			}
			_ = bus.Put(context.Background(), provisions.PlansBucket, "ace.letta", []byte(bad))
			assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres"))
			var logged []string
			p.opts.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
			out, err := p.Pass(context.Background())
			if err != nil {
				t.Fatalf("%s: %v", name, err)
			}
			if bus.get(BucketOf("postgres"), "g14.umami.postgres-database") == nil {
				t.Fatalf("%s: the good consumer was not answered", name)
			}
			if bus.get(BucketOf("postgres"), "ace.letta.postgres-database") == nil || len(m.gone) != 0 {
				t.Fatalf("%s: the entry of the unreadable assignment's consumer was removed", name)
			}
			said := false
			for _, o := range out {
				if o.Key == "ace.letta" && o.Status == Failed && o.Why != "" {
					said = true
				}
			}
			if !said || !strings.Contains(strings.Join(logged, "\n"), "ace.letta") {
				t.Fatalf("%s: the skipped assignment is not said: %+v, %q", name, out, logged)
			}
		}
	})

	// Run watches the binds on the stand-in bus: a consumer that binds is answered, and one that unbinds is removed,
	// without a pass asked by hand.
	t.Run("RunAnswersBindsAndUnbindsAsTheyAreWatched", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s"}}
		p := postgres(bus, v, m)
		p.opts.Every = time.Hour
		ctx, cancel := context.WithCancel(context.Background())
		done := make(chan error, 1)
		go func() { done <- p.Run(ctx) }()
		assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
		waitFor(t, func() bool { return bus.get(BucketOf("postgres"), "ace.letta.postgres-database") != nil })
		retire(t, bus, "ace", "letta", "postgres-database", "novox", "postgres")
		waitFor(t, func() bool { return bus.get(BucketOf("postgres"), "ace.letta.postgres-database") == nil })
		cancel()
		if err := <-done; err != nil && !errors.Is(err, context.Canceled) {
			t.Fatal(err)
		}
	})

	// An instance deletes only an entry it wrote, and only while the entry is as it read it: one that another instance of
	// the provider wrote after the consumer moved stands (issue 825, finding 5).
	t.Run("AnEntryAnotherInstanceWroteAfterTheMoveStands", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s"}}
		assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
		a := postgres(bus, v, m)
		if _, err := a.Pass(context.Background()); err != nil {
			t.Fatal(err)
		}
		// The consumer moves to the instance on shanks: A's pass reads its own entry and the unbind, and while it lets
		// the consumer go, B writes the consumer's entry.
		retire(t, bus, "ace", "letta", "postgres-database", "novox", "postgres")
		theirs, _ := json.Marshal(Entry{Provider: "shanks", ProviderModule: "postgres", Provision: "postgres-database",
			Consumer: "ace", Module: "letta", VaultName: "pair/shanks/postgres-database/ace/letta"})
		a.opts.Gone = func(ctx context.Context, e Entry) error {
			return bus.Put(ctx, BucketOf("postgres"), e.Key(), theirs)
		}
		out, err := a.Pass(context.Background())
		if err != nil {
			t.Fatal(err)
		}
		if got := bus.get(BucketOf("postgres"), "ace.letta.postgres-database"); !bytes.Equal(got, theirs) {
			t.Fatalf("the other instance's entry was removed: %s", got)
		}
		for _, o := range out {
			if o.Status == Removed {
				t.Fatalf("outcomes %+v", out)
			}
		}
	})
}

// TestCredentialsAndKeys groups the cases below, each one a subtest.
func TestCredentialsAndKeys(t *testing.T) {
	// A bind whose credential the vault does not hold yet is waited for: nothing is made and no entry is written, so a
	// consumer is never told of a login its provider has not made; once the vault holds it, it is answered.
	t.Run("ACredentialTheVaultDoesNotHoldYetIsWaitedFor", func(t *testing.T) {
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{}}
		assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
		p := postgres(bus, v, m)
		out, err := p.Pass(context.Background())
		if err != nil {
			t.Fatal(err)
		}
		if len(out) != 1 || out[0].Status != Waiting || !strings.Contains(out[0].Why, "pair/novox/postgres-database/ace/letta") {
			t.Fatalf("outcomes %+v", out)
		}
		if bus.keys(BucketOf("postgres")) != nil || m.calls != 0 {
			t.Fatal("a consumer was answered before its credential was there")
		}
		v.values["pair/novox/postgres-database/ace/letta"] = "late"
		if _, err := p.Pass(context.Background()); err != nil {
			t.Fatal(err)
		}
		if entryOf(t, bus, "ace.letta.postgres-database").Refused != "" || m.made["ace.letta.postgres-database"] != "late" {
			t.Fatal("not answered once the vault held the value")
		}
	})

	// An adapter that fails is said with the credential taken out of its words, and its entry says it failed, serving
	// nothing (novox/hq ADR 0570 clause 3: the provider's outcome is its entry, which the holder of mesh-provisioner reads).
	t.Run("AFailedMakeIsSaidWithoutTheCredential", func(t *testing.T) {
		bus, m := newStandIn(), &maker{fail: errors.New(`password "hunter2-secret" rejected`)}
		v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "hunter2-secret"}}
		assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
		var logged []string
		p := postgres(bus, v, m)
		p.opts.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
		out, err := p.Pass(context.Background())
		if err != nil {
			t.Fatal(err)
		}
		if len(out) != 1 || out[0].Status != Failed || strings.Contains(out[0].Why, "hunter2") {
			t.Fatalf("outcomes %+v", out)
		}
		if strings.Contains(strings.Join(logged, "\n"), "hunter2") || len(logged) == 0 {
			t.Fatalf("logged %q", logged)
		}
		var e Entry
		raw := bus.get(BucketOf("postgres"), "ace.letta.postgres-database")
		if err := json.Unmarshal(raw, &e); err != nil || e.State != "failed" || e.Values != nil || e.VaultName != "" ||
			strings.Contains(string(raw), "hunter2") {
			t.Fatalf("the failed entry is %s", raw)
		}

		// A provider not ready yet (gitea after a restart, its keys not read; the regression of 2026-10-30) is no failure:
		// a first entry waits serving nothing, and once made, a later not ready keeps serving what it served.
		m.fail = fmt.Errorf("keys not read yet: %w", ErrNotReady)
		_ = bus.Delete(context.Background(), BucketOf("postgres"), "ace.letta.postgres-database")
		if out, _ := p.Pass(context.Background()); len(out) != 1 || out[0].Status != Waiting {
			t.Fatalf("not ready at first: %+v", out)
		}
		raw = bus.get(BucketOf("postgres"), "ace.letta.postgres-database")
		if err := json.Unmarshal(raw, &e); err != nil || e.State != "waits" || !strings.Contains(e.Why, "keys not read yet") || e.Values != nil {
			t.Fatalf("the first entry not ready is %s", raw)
		}
		m.fail = nil
		if out, _ := p.Pass(context.Background()); len(out) != 1 || out[0].Status != Answered {
			t.Fatalf("made: %+v", out)
		}
		var made Entry
		_ = json.Unmarshal(bus.get(BucketOf("postgres"), "ace.letta.postgres-database"), &made)
		m.fail = ErrNotReady
		if out, _ := p.Pass(context.Background()); len(out) != 1 || out[0].Status != Waiting {
			t.Fatalf("not ready after a restart: %+v", out)
		}
		raw = bus.get(BucketOf("postgres"), "ace.letta.postgres-database")
		if err := json.Unmarshal(raw, &e); err != nil || e.State != "waits" || e.VaultName != made.VaultName || e.User != made.User ||
			fmt.Sprint(e.Values) != fmt.Sprint(made.Values) || len(made.Values) == 0 {
			t.Fatalf("not ready after it served, the entry is %s, it served %+v", raw, made)
		}
	})

	// A failed Make is said with the credential taken out in the forms it can come back in: as it is, in base64 (standard
	// and URL, with and without padding) and as a JSON string says it (issue 825, finding 6).
	t.Run("AFailedMakeIsSaidWithoutTheCredentialInAnyOfItsForms", func(t *testing.T) {
		cred := "pa/ss+w\"rd?>>x"
		b := []byte(cred)
		jsonQuoted, _ := json.Marshal(cred)
		forms := map[string]string{
			"raw": cred, "std": base64.StdEncoding.EncodeToString(b), "std unpadded": base64.RawStdEncoding.EncodeToString(b),
			"url": base64.URLEncoding.EncodeToString(b), "url unpadded": base64.RawURLEncoding.EncodeToString(b),
			"json": string(jsonQuoted[1 : len(jsonQuoted)-1]),
		}
		if forms["std"] == forms["url"] || forms["json"] == forms["raw"] {
			t.Fatal("the test credential does not tell the forms apart")
		}
		for name, form := range forms {
			bus, m := newStandIn(), &maker{fail: errors.New("login failed for " + form + " at the server")}
			v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": cred}}
			assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
			out, err := postgres(bus, v, m).Pass(context.Background())
			if err != nil || len(out) != 1 {
				t.Fatalf("%v %+v", err, out)
			}
			if strings.Contains(out[0].Why, form) || !strings.Contains(out[0].Why, "<credential>") {
				t.Fatalf("%s form: %q", name, out[0].Why)
			}
		}
	})

	// The consumer's identity is the mesh's (ADR 0202 §1): mesh_<node>_<module>, lower case, `_` for anything else,
	// and its DNS form writes `-` for `_`.
	t.Run("TheIdentityIsTheMeshsAndItsDNSForm", func(t *testing.T) {
		if got := ConsumerIdentity("Home.Server", "my-app"); got != "mesh_home_server_my_app" {
			t.Fatal(got)
		}
		got, err := ServedTo(map[string]any{"bucket": "b-${consumer:as:dns}", "n": 3}, "mesh_ace_letta")
		if err != nil || got["bucket"] != "b-mesh-ace-letta" || got["n"] != 3 {
			t.Fatalf("%v %v", got, err)
		}
		if _, err := ServedTo(map[string]any{"x": "${consumer:node}"}, "mesh_ace_letta"); err == nil {
			t.Fatal("a fact the mesh does not state was filled")
		}
	})

	// Every entry the helper writes passes the state guard's key rule (issue 945), and names where the vault keeps the
	// credential under `vault-name`, for a login per consumer and for a shared one.
	t.Run("AnEntryHasNoKeyTheStateGuardRefuses", func(t *testing.T) {
		for name, form := range map[string]CredentialForm{"pair": PerConsumer, "shared": Shared} {
			bus, m := newStandIn(), &maker{}
			v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s1",
				"provision/novox/postgres-database": "s2"}}
			assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
			p := New(Options{Node: "novox", Module: "postgres", Bus: bus, Vault: v, Make: m.Make, Logf: func(string, ...any) {},
				Offers: map[string]Offer{"postgres-database": {Credential: form, User: "shared-user"}}})
			if _, err := p.Pass(context.Background()); err != nil {
				t.Fatal(err)
			}
			var doc map[string]any
			if err := json.Unmarshal(bus.get(BucketOf("postgres"), "ace.letta.postgres-database"), &doc); err != nil {
				t.Fatal(err)
			}
			var keys []string
			keysOf(doc, &keys)
			for _, k := range keys {
				if refusedByTheStateGuard(k) {
					t.Fatalf("%s: the entry carries the key %q, which the state guard refuses", name, k)
				}
			}
			want := map[string]string{"pair": "pair/novox/postgres-database/ace/letta",
				"shared": "provision/novox/postgres-database"}[name]
			if doc["vault-name"] != want {
				t.Fatalf("%s: vault-name is %v, want %s", name, doc["vault-name"], want)
			}
		}
	})

	// novox/hq ADR 0570 clauses 3 and 5 (issue 1954 L6-P3, P4): a consumer the seat mesh-provisioner plans is bound by its
	// plan and its provision key, never by its assignment. Its entry is made only once its key is asked and every provision
	// it depends on is provided, and says provided; a plan gone retires nothing by itself; a retiring key is let go only on
	// the operator's yes.
	t.Run("TheSeatsKeysSayWhenAProviderActsAndWhenItLetsGo", func(t *testing.T) {
		ctx := context.Background()
		bus, m := newStandIn(), &maker{}
		v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s1"}}
		// The same consumer's assignment names another provider: the seat's plan wins.
		assign(t, bus, "ace", "letta", bind("postgres-database", "", "shanks", "postgres"))
		plan, _ := provisions.EncodePlan(provisions.Plan{Node: "ace", Module: "letta", Plan: catalogue.Plan{
			Provisions: []catalogue.PlanProvision{{Provision: "postgres-database", Provider: "novox", ProviderModule: "postgres"}}}})
		_ = bus.Put(ctx, provisions.PlansBucket, "ace.letta", plan)
		key := "ace.letta.postgres-database"
		put := func(k string, state string, depends []string, warrant *provisions.Warrant) {
			pr := provisions.Provision{Provision: "postgres-database", Consumer: provisions.End{Module: "letta", Node: "ace"},
				Provider: provisions.End{Module: "postgres", Node: "novox"}, DependsOn: depends, State: state, Warrant: warrant}
			if k != key {
				_, _, pr.Provision, _, _ = provisions.ParseKey(k)
				pr.Provider = provisions.End{Module: "mesh-vault", Node: "novox"}
			}
			if state != provisions.Asked && state != provisions.Provided {
				pr.Why = "said"
			}
			raw, err := provisions.Encode(pr)
			if err != nil {
				t.Fatal(err)
			}
			_ = bus.Put(ctx, provisions.ProvisionsBucket, k, raw)
		}
		p := postgres(bus, v, m)
		steps := []struct {
			name   string
			before func()
			status string
			says   string
			entry  bool
		}{
			{name: "no key yet", status: Waiting, says: "holds no key"},
			{name: "a dependency asked", before: func() {
				put("ace.letta.secret", provisions.Asked, nil, nil)
				put(key, provisions.Asked, []string{"ace.letta.secret"}, nil)
			}, status: Waiting, says: "depends on ace.letta.secret, which is asked"},
			{name: "the dependency provided", before: func() { put("ace.letta.secret", provisions.Provided, nil, nil) },
				status: Answered, entry: true},
			{name: "its plan gone: nothing let go by itself", before: func() { _ = bus.Delete(ctx, provisions.PlansBucket, "ace.letta") },
				entry: true},
			{name: "retiring, not yet answered", before: func() { put(key, provisions.Retiring, nil, nil) }, entry: true},
			{name: "retiring, the operator's yes", before: func() {
				put(key, provisions.Retiring, nil, &provisions.Warrant{Ask: "a-1", Option: "yes", By: "the operator"})
			}, status: Removed},
		}
		for _, s := range steps {
			if s.before != nil {
				s.before()
			}
			out, err := p.Pass(ctx)
			if err != nil {
				t.Fatalf("%s: %v", s.name, err)
			}
			var mine []Outcome
			for _, o := range out {
				if o.Key == key {
					mine = append(mine, o)
				}
			}
			if s.status != "" && (len(mine) != 1 || mine[0].Status != s.status || !strings.Contains(mine[0].Why, s.says)) {
				t.Errorf("%s: %+v", s.name, mine)
			}
			if held := bus.get(BucketOf("postgres"), key) != nil; held != s.entry {
				t.Errorf("%s: an entry held %v", s.name, held)
			}
			if s.entry && entryOf(t, bus, key).State != provisions.Provided {
				t.Errorf("%s: the entry does not say provided", s.name)
			}
		}
		if !slices.Equal(m.gone, []string{key}) || m.made[key] != "s1" {
			t.Errorf("gone %v, made %v: let go once, on the warrant, and made from the vault's value", m.gone, m.made)
		}
	})
}

// TestConsumersFileAndStateBus groups the cases below, each one a subtest.
func TestConsumersFileAndStateBus(t *testing.T) {
	// The provider reads its consumers' credentials only from the file its machine's provisioner client fills (novox/hq
	// ADR 0373 clause 3, issue 679): one entry per consumer, naming the consumer and the vault's name for its credential,
	// and carrying the value. The helper is wired to it as its Vault and asks nothing of the vault itself.
	t.Run("TheConsumersFileAnswersEachConsumersCredential", func(t *testing.T) {
		dir := t.TempDir()
		at := filepath.Join(dir, "consumers.json")
		file := `[{"consumer":"ace","module":"letta","credential":"pair/novox/postgres-database/ace/letta","value":"s3cret"},
	 {"consumer":"g14","module":"gitea","local":"cache","credential":"pair/novox/postgres-database/g14/gitea/cache","value":"other"}]`
		if err := os.WriteFile(at, []byte(file), 0o600); err != nil {
			t.Fatal(err)
		}
		v := ConsumersFile{Paths: []string{filepath.Join(dir, "absent.json"), at}}
		got, err := v.Value(context.Background(), PairName("novox", "postgres-database", "ace", "letta", ""))
		if err != nil || string(got) != "s3cret" {
			t.Fatalf("ace's letta: %q, %v; want s3cret", got, err)
		}
		got, err = v.Value(context.Background(), PairName("novox", "postgres-database", "g14", "gitea", "cache"))
		if err != nil || string(got) != "other" {
			t.Fatalf("g14's gitea under cache: %q, %v; want other", got, err)
		}
		// A consumer the file does not carry is waited for, and never told of a login its provider has not made.
		_, err = v.Value(context.Background(), PairName("novox", "postgres-database", "shanks", "photos", ""))
		if err == nil || !strings.Contains(err.Error(), "pair/novox/postgres-database/shanks/photos") {
			t.Fatalf("a consumer the file does not carry: %v", err)
		}
		if strings.Contains(err.Error(), "s3cret") || strings.Contains(err.Error(), "other") {
			t.Fatalf("an error names a credential: %v", err)
		}

		// Wired as the helper's Vault, the file is what makes each consumer's login.
		bus := newStandIn()
		assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
		var made []string
		p := New(Options{Node: "novox", Module: "postgres", Bus: bus, Vault: v,
			Offers: map[string]Offer{"postgres-database": {Serves: map[string]any{"port": 5432}}},
			Make: func(_ context.Context, c Consumer) error {
				made = append(made, c.Entry.User+"="+string(c.Credential))
				return nil
			}, Logf: func(string, ...any) {}})
		if _, err := p.Pass(context.Background()); err != nil {
			t.Fatal(err)
		}
		if len(made) != 1 || made[0] != "mesh_ace_letta=s3cret" {
			t.Fatalf("made %v, want ace's letta under the file's credential", made)
		}
	})

	// A file that is not the client's form is said, never read as no consumers.
	t.Run("AConsumersFileNotInItsFormIsRefused", func(t *testing.T) {
		at := filepath.Join(t.TempDir(), "consumers.json")
		if err := os.WriteFile(at, []byte(`{"not": "a list"}`), 0o600); err != nil {
			t.Fatal(err)
		}
		_, err := ConsumersFile{Paths: []string{at}}.Value(context.Background(), "pair/novox/p/ace/m")
		if err == nil || !strings.Contains(err.Error(), at) {
			t.Fatalf("a file not in the client's form: %v", err)
		}
	})

	t.Run("StandingIsFalseOnlyForAnEndedOrPendingValue", func(t *testing.T) {
		for raw, want := range map[string]bool{
			`{"node":"a","module":"b"}`:                                   true,
			`{"node":"a","module":"b","ended":"","state":"","x":{"y":1}}`: true,
			`{"node":"a","module":"b","ended":null}`:                      true,
			`{"node":"a","module":"b","build":"b1","commit":"c","rollout":"r","since":"t","artifact":{"name":"n"}}`: true,
			`{"node":"a","module":"b","ended":"2026-10-18T01:00:00Z"}`:                                              false,
			`{"node":"a","module":"b","state":"pending"}`:                                                           false,
		} {
			if got := Standing([]byte(raw)); got != want {
				t.Errorf("%s: %v, want %v", raw, got, want)
			}
		}
	})

	// The bus of a moved provider maps its own bucket to the state `bound` and the seat's plans to the state
	// `mesh-provisioner.plans` (novox/hq ADR 0570 clause 2), writes only the own bucket's state, and refuses any other
	// bucket, the controller's assignments among them.
	t.Run("StateBusMapsTheProvidersBucketsToTheRuntimesStates", func(t *testing.T) {
		bus := StateBus("postgres")
		rt := serve(t, []stdio.Tool{
			{Name: "watch", Run: func(a map[string]any) (any, error) {
				return nil, bus.Watch(context.Background(), a["bucket"].(string), func(string, []byte) {})
			}},
			{Name: "put", Run: func(map[string]any) (any, error) {
				return "put", bus.Put(context.Background(), BucketOf("postgres"), "k", []byte(`{"a":1}`))
			}},
			{Name: "delete", Run: func(map[string]any) (any, error) {
				return "deleted", bus.Delete(context.Background(), BucketOf("postgres"), "k")
			}},
		})
		for i, c := range []struct{ bucket, state string }{
			{BucketOf("postgres"), "bound"},
			{provisions.PlansBucket, "mesh-provisioner.plans"},
		} {
			rt.call(i+1, "watch", map[string]any{"bucket": c.bucket})
			w := rt.next()
			if w["method"] != "mesh/state.watch" || w["params"].(map[string]any)["state"] != c.state {
				t.Fatalf("bucket %s asked %v, want the state %s", c.bucket, w, c.state)
			}
			rt.send(map[string]any{"jsonrpc": "2.0", "id": w["id"], "result": map[string]any{}})
			rt.next()
		}
		rt.call(3, "put", nil)
		p := rt.next()
		if p["method"] != "mesh/state.put" || p["params"].(map[string]any)["state"] != "bound" || p["params"].(map[string]any)["key"] != "k" {
			t.Fatalf("put asked %v", p)
		}
		rt.send(map[string]any{"jsonrpc": "2.0", "id": p["id"], "result": map[string]any{"revision": 1}})
		rt.next()
		rt.call(4, "delete", nil)
		d := rt.next()
		if d["method"] != "mesh/state.delete" || d["params"].(map[string]any)["state"] != "bound" {
			t.Fatalf("delete asked %v", d)
		}
		rt.send(map[string]any{"jsonrpc": "2.0", "id": d["id"], "result": map[string]any{}})
		rt.next()

		for _, bucket := range []string{"other_bound", "elsewhere", "mesh-controller_assignments"} {
			if err := bus.Watch(context.Background(), bucket, func(string, []byte) {}); err == nil || !strings.Contains(err.Error(), "not a bucket") {
				t.Fatalf("a watch of %s gave %v", bucket, err)
			}
			if err := bus.Put(context.Background(), bucket, "k", nil); err == nil {
				t.Fatalf("a put to %s was allowed", bucket)
			}
			if err := bus.Delete(context.Background(), bucket, "k"); err == nil {
				t.Fatalf("a delete in %s was allowed", bucket)
			}
		}
	})

	// A watch whose ctx has ended calls its handler no more, though the runtime keeps sending changes: the Bus contract
	// says a watch lasts until ctx ends, and a stdio state watch lasts as long as the process.
	t.Run("StateBusWatchCallsItsHandlerNoMoreOnceItsContextEnded", func(t *testing.T) {
		bus := StateBus("postgres")
		ctx, cancel := context.WithCancel(context.Background())
		handled := make(chan string, 8)
		rt := serve(t, []stdio.Tool{
			{Name: "start", Run: func(map[string]any) (any, error) {
				return "started", bus.Watch(ctx, provisions.PlansBucket, func(key string, value []byte) {
					handled <- key + "=" + string(value)
				})
			}},
			{Name: "stop", Run: func(map[string]any) (any, error) { cancel(); return "stopped", nil }},
		})
		change := func(id int, watch any, key, op string) {
			rt.send(map[string]any{"jsonrpc": "2.0", "id": id, "method": "mesh/state", "params": map[string]any{"watch": watch,
				"change": map[string]any{"state": "mesh-controller.assignments", "key": key, "op": op, "value": "v"}}})
			if r := rt.next(); r["id"] != float64(id) || r["error"] != nil {
				t.Fatalf("a change was answered %v", r)
			}
		}
		rt.call(1, "start", nil)
		w := rt.next()
		rt.send(map[string]any{"jsonrpc": "2.0", "id": w["id"], "result": map[string]any{}})
		rt.next()

		change(50, w["id"], "before", "put")
		if got := <-handled; got != `before="v"` {
			t.Fatalf("handled %q", got)
		}
		rt.call(2, "stop", nil)
		rt.next()
		change(51, w["id"], "after", "put")
		change(52, w["id"], "after", "delete")
		select {
		case got := <-handled:
			t.Fatalf("the handler was called after its context ended: %q", got)
		case <-time.After(100 * time.Millisecond):
		}
	})
}