mesh-controller/cmd/mesh-controller/assignment_accounts_test.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/chain"
	"github.com/novox/mesh-controller/internal/inventory"
)

// An assignment that comes to stand issues its node's bus accounts and the module's grant (ADR 0543, issue 1945): on
// 2026-10-25 at 17:19 mesh-build-node and mesh-build-go, built long before, were assigned to ace, and nothing issued
// ace's accounts nor put them in the bus's user list.

func anAssignmentValue(t *testing.T, a chain.Assignment) []byte {
	t.Helper()
	raw, err := json.Marshal(a)
	if err != nil {
		t.Fatal(err)
	}
	return raw
}

// Only a write that makes the key stand turns it: a rewrite of a standing assignment (the provisioner's plan kept in its
// value), an ended one, a pending one or a module only pulled in by another does not. The process's first watch hands
// the assignments standing when it starts to the start's look (not fresh), says once that they are all read, and hands
// each assignment written afterwards as fresh; a watch opened again after one ended hands every standing assignment it
// has not seen stand as fresh and never calls the start's look (the review of issue 1945).
func TestAnAssignmentThatComesToStand(t *testing.T) {
	at := time.Date(2026, 10, 25, 16, 19, 42, 0, time.UTC)
	stands := chain.Assignment{Node: "ace", Module: "mesh-build-node", Assigned: at}
	ended := stands
	ended.Ended = at.Add(time.Hour)
	pending := stands
	pending.State = chain.StatePending
	pulled := chain.Assignment{Node: "ace", Module: "mesh-build-node", RequiredBy: "mesh-build-go"}
	replanned := stands
	replanned.Provisioning = json.RawMessage(`{"plan":1}`)

	type write struct {
		value   *chain.Assignment // nil for a delete
		turning bool
	}
	for _, c := range []struct {
		name   string
		writes []write
	}{
		{"a new assignment turns", []write{{&stands, true}}},
		{"a rewrite of one that stands does not", []write{{&stands, true}, {&replanned, false}, {&stands, false}}},
		{"an ended one does not", []write{{&ended, false}}},
		{"assigned again after it ended turns again", []write{{&stands, true}, {&ended, false}, {&stands, true}}},
		{"a pending one does not, and turns once made", []write{{&pending, false}, {&stands, true}}},
		{"a module only pulled in does not", []write{{&pulled, false}}},
		{"one deleted and written again turns again", []write{{&stands, true}, {nil, false}, {&stands, true}}},
	} {
		t.Run(c.name, func(t *testing.T) {
			turns := &assignmentTurns{}
			for i, w := range c.writes {
				var value []byte
				if w.value != nil {
					value = anAssignmentValue(t, *w.value)
				}
				a, turned := turns.heard("ace.mesh-build-node", value)
				if turned != w.turning {
					t.Fatalf("write %d: turned %v, want %v", i, turned, w.turning)
				}
				if turned && (a.Node != "ace" || a.Module != "mesh-build-node") {
					t.Fatalf("write %d: the assignment turned is %s on %s", i, a.Module, a.Node)
				}
			}
		})
	}

	type handed struct {
		key   string
		fresh bool
	}
	for _, first := range []bool{true, false} {
		t.Run(fmt.Sprintf("the watch, first %v", first), func(t *testing.T) {
			bucket := chain.NewMemory()
			put := func(a chain.Assignment) {
				t.Helper()
				value := anAssignmentValue(t, a)
				if e, held, _ := bucket.Get(t.Context(), a.Key()); held {
					if _, err := bucket.Update(t.Context(), a.Key(), value, e.Revision); err != nil {
						t.Fatal(err)
					}
					return
				}
				if _, err := bucket.Create(t.Context(), a.Key(), value); err != nil {
					t.Fatal(err)
				}
			}
			put(chain.Assignment{Node: "g14", Module: "mesh-build-go", Assigned: at})
			got := make(chan handed, 8)
			read := make(chan struct{}, 8)
			ctx, stop := context.WithCancel(t.Context())
			done := make(chan error, 1)
			go func() {
				done <- watchAssignmentsOnce(ctx, bucket.Watch, &assignmentTurns{}, first,
					func(a chain.Assignment, fresh bool) { got <- handed{a.Key(), fresh} },
					func() { read <- struct{}{} })
			}()
			next := func() handed {
				t.Helper()
				select {
				case h := <-got:
					return h
				case <-time.After(5 * time.Second):
					t.Fatal("nothing handed")
					return handed{}
				}
			}
			// Standing when the watch opened: the start's look on the first watch, fresh on any other.
			if h := next(); h != (handed{"g14.mesh-build-go", !first}) {
				t.Fatalf("the standing assignment was handed %+v", h)
			}
			put(chain.Assignment{Node: "g14", Module: "mesh-build-go", Assigned: at, Provisioning: json.RawMessage(`{"plan":2}`)})
			put(chain.Assignment{Node: "ace", Module: "mesh-build-node", Assigned: at})
			if h := next(); h != (handed{"ace.mesh-build-node", true}) {
				t.Fatalf("handed %+v, want ace.mesh-build-node as fresh (and nothing for the rewrite before it)", h)
			}
			stop()
			if err := <-done; err != nil {
				t.Fatal(err)
			}
			if want := map[bool]int{true: 1, false: 0}[first]; len(read) != want {
				t.Fatalf("the start's look was called %d time(s), want %d", len(read), want)
			}
		})
	}
	// A first watch that fails to open, or ends before its Caught mark, is no start's look: the next watch is, and hands
	// every standing assignment to it once, none as fresh (the review of issue 1945: a burst of fresh settles, or keys
	// handed and dropped).
	for _, firstWatch := range []string{"fails to open", "ends before its Caught mark"} {
		t.Run("a first watch that "+firstWatch, func(t *testing.T) {
			bucket := chain.NewMemory()
			standing := chain.Assignment{Node: "g14", Module: "mesh-build-go", Assigned: at}
			if _, err := bucket.Create(t.Context(), standing.Key(), anAssignmentValue(t, standing)); err != nil {
				t.Fatal(err)
			}
			watches := 0
			watch := func(ctx context.Context) (<-chan chain.Entry, error) {
				watches++
				if watches > 1 {
					return bucket.Watch(ctx)
				}
				if firstWatch == "fails to open" {
					return nil, errors.New("the chain is not open yet")
				}
				cut := make(chan chain.Entry, 1)
				cut <- chain.Entry{Key: standing.Key(), Value: anAssignmentValue(t, standing)}
				close(cut)
				return cut, nil
			}
			var looks [][]chain.Assignment
			var fresh []chain.Assignment
			ctx, stop := context.WithCancel(t.Context())
			defer stop()
			runAssignmentWatches(ctx, watch, func(a chain.Assignment) { fresh = append(fresh, a) },
				func(look []chain.Assignment) { looks = append(looks, look); stop() },
				func() bool { return ctx.Err() == nil })
			if len(looks) != 1 || len(looks[0]) != 1 || looks[0][0].Key() != standing.Key() || len(fresh) != 0 {
				t.Fatalf("after %d watch(es): looks %v, fresh %v; want one look at %s and nothing fresh",
					watches, looks, fresh, standing.Key())
			}
		})
	}
}

// settle issues what is missing and then puts it on the bus: the node holding the bus composed (its user list) and the
// grants written, even when every account was minted before (a module assigned again keeps its account). The start's
// look reads the catalogue once, issues for each standing assignment, and composes the bus's node and writes the grants
// once when it looked at any. Neither acts for a controller that does not hold the lease, nor for the controller's own
// module.
func TestSettlingAndTheStartsLook(t *testing.T) {
	ace := func(m string) chain.Assignment { return chain.Assignment{Node: "ace", Module: m} }
	cases := []struct {
		name               string
		look               []chain.Assignment // nil: settle mesh-build-node on ace
		module             string
		notHolding         bool
		issued             bool
		issueErr, composeE error
		holder             string
		wantIssues         int
		wantComposed       []string
		wantGrants         bool
		wantErr            string
		stale              []string // what forgetting the accounts of modules no longer on their nodes says
	}{
		{name: "an account not minted", module: "mesh-build-node", issued: true, holder: "novox", wantIssues: 1,
			wantComposed: []string{"novox"}, wantGrants: true},
		{name: "an account minted before: the bus and the grants still", module: "mesh-build-go", holder: "novox",
			wantIssues: 1, wantComposed: []string{"novox"}, wantGrants: true},
		{name: "no lease: nothing, and tried again", module: "mesh-build-node", notHolding: true, issued: true,
			holder: "novox", wantErr: errNotHolding.Error()},
		{name: "the controller's own module is the terminal's", module: controllerModule, issued: true, holder: "novox"},
		{name: "no bus holder: the grants still", module: "mesh-build-node", issued: true, wantIssues: 1, wantGrants: true},
		{name: "an issue that fails is the error, nothing composed", module: "mesh-build-node",
			issueErr: errors.New("the bus cannot be reached"), holder: "novox", wantIssues: 1, wantErr: "the bus cannot be reached"},
		{name: "a composition that fails is the error, no grant", module: "mesh-build-node", issued: true, holder: "novox",
			composeE: errors.New("store refused"), wantIssues: 1, wantComposed: []string{"novox"}, wantErr: "store refused"},
		{name: "the start's look: each issued, the bus and the grants once",
			look: []chain.Assignment{ace("mesh-build-node"), ace(controllerModule), ace("mesh-build-go")}, issued: true,
			holder: "novox", wantIssues: 2, wantComposed: []string{"novox"}, wantGrants: true},
		{name: "the start's look with every account minted still covers once",
			look: []chain.Assignment{ace("mesh-build-node"), ace("mesh-build-go")}, holder: "novox", wantIssues: 2,
			wantComposed: []string{"novox"}, wantGrants: true},
		{name: "the start's look at nothing does nothing", look: []chain.Assignment{ace(controllerModule)}, holder: "novox"},
		{name: "the start's look forgets the accounts of modules no longer on their nodes first",
			look: []chain.Assignment{ace("mesh-build-node")}, holder: "novox", wantIssues: 1, wantComposed: []string{"novox"},
			wantGrants: true, stale: []string{"mesh-build-go no longer runs on ace: its bus accounts forgotten: ace.mesh-build-go"}},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			var composed []string
			issues, catalogues, grants := 0, 0, 0
			forgets := 0
			steps := assignmentAccountSteps{
				holds: func(context.Context) bool { return !c.notHolding },
				forgetStale: func(context.Context) ([]string, error) {
					forgets++
					return c.stale, nil
				},
				catalogue: func(context.Context) (map[string]catalogue.Manifest, error) {
					catalogues++
					return map[string]catalogue.Manifest{}, nil
				},
				issue: func(context.Context, map[string]catalogue.Manifest, string, string) (bool, string, error) {
					issues++
					if c.issued {
						return true, "its bus credential issued", c.issueErr
					}
					return false, "", c.issueErr
				},
				busHolder: func(context.Context) (string, error) { return c.holder, nil },
				compose: func(_ context.Context, nodes []string) error {
					composed = append(composed, nodes...)
					return c.composeE
				},
				grants: func(context.Context) error { grants++; return nil },
			}
			var said string
			var err error
			if c.look != nil {
				said, err = steps.look(t.Context(), c.look)
			} else {
				said, err = steps.settle(t.Context(), "ace", c.module)
			}
			if c.wantErr != "" {
				if err == nil || !strings.Contains(err.Error(), c.wantErr) {
					t.Fatalf("error %v, want one saying %q", err, c.wantErr)
				}
			} else if err != nil {
				t.Fatal(err)
			}
			if issues != c.wantIssues || catalogues > 1 {
				t.Fatalf("%d issue(s) from %d reading(s) of the catalogue, want %d from at most one", issues, catalogues, c.wantIssues)
			}
			if strings.Join(composed, ",") != strings.Join(c.wantComposed, ",") {
				t.Fatalf("composed %v, want %v", composed, c.wantComposed)
			}
			if want := map[bool]int{true: 1}[c.wantGrants]; grants != want {
				t.Fatalf("grants written %d time(s), want %d", grants, want)
			}
			if want := map[bool]int{true: 1}[len(c.look) > 0 && !c.notHolding]; forgets != want {
				t.Fatalf("forgot stale accounts %d time(s), want %d (only from a reading that holds assignments)", forgets, want)
			}
			for _, line := range c.stale {
				if !strings.Contains(said, line) {
					t.Fatalf("said %q, want it to say %q", said, line)
				}
			}
			if c.wantGrants && c.wantErr == "" && !strings.Contains(said, "the grants written") {
				t.Fatalf("said %q, want it to name the grants", said)
			}
		})
	}
}

// The accounts a module speaks with on a node: the module's own when it reads `broker`, its provisioner client's when
// it runs one alone, none when its accounts are its processes' or it does not speak on the bus. A process's account is
// held in the provisioner client's form, named for the process and carrying no claims, and may be issued only when
// the process declares its own secret broker-<process id> (ADR 0436 §2; issue 1181).
func TestTheAccountsAModuleSpeaksWithOnANode(t *testing.T) {
	client := aProvisioner()
	delete(client.OwnSecrets, "broker")
	for _, c := range []struct {
		name string
		m    catalogue.Manifest
		user string
		own  bool
	}{
		{"a module reading broker", catalogue.Manifest{Module: "mesh-build-node",
			OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh-build-node/broker"}}},
			broker.Principal{Kind: broker.KindModule, Node: "ace", Module: "mesh-build-node"}.Username(), true},
		{"a module reading broker and running the client", aProvisioner(),
			broker.Principal{Kind: broker.KindModule, Node: "ace", Module: "mesh-provisioner"}.Username(), true},
		{"the provisioner client alone", client,
			broker.Principal{Kind: broker.KindProvisioner, Node: "ace", Module: "mesh-provisioner"}.Username(), true},
		{"no broker, no client", catalogue.Manifest{Module: "hello-web"}, "", false},
	} {
		t.Run(c.name, func(t *testing.T) {
			user, own := ownAccountOf(c.m, "ace")
			if user != c.user || own != c.own {
				t.Fatalf("got %q %v, want %q %v", user, own, c.user, c.own)
			}
		})
	}

	t.Run("a process's account", func(t *testing.T) {
		user := broker.Principal{Kind: broker.KindProcess, Node: "novox", Module: "mesh-mail", Process: "send"}.Username()
		if user != "novox.mesh-mail.send" {
			t.Fatalf("the process's username: %q", user)
		}
		held, err := heldProcessAccount(broker.Broker{Fingerprint: "abc"}, "novox.example:4222", "novox", "mesh-mail",
			user, "a-password")
		if err != nil {
			t.Fatal(err)
		}
		var got map[string]any
		if err := json.Unmarshal(held, &got); err != nil {
			t.Fatal(err)
		}
		want := map[string]any{"url": "nats://novox.example:4222", "fingerprint": "abc", "node": "novox",
			"module": "mesh-mail", "user": "novox.mesh-mail.send", "password": "a-password"}
		for k, v := range want {
			if got[k] != v {
				t.Errorf("%s: %v, want %v", k, got[k], v)
			}
		}
		if len(got) != len(want) {
			t.Errorf("the held account carries %v; want exactly %v (never claims — a process holds no seat)", got, want)
		}
	})

	// An issue mints and seals an account only when it is not issued: no row, or a row with no sealed credential (issue
	// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go had a row from 17:19 and no credential), under the lock on
	// the account; a seal that fails leaves it unsealed for the next issue. A rotation always mints.
	for _, c := range []struct {
		name           string
		only           bool
		minted, sealed bool
		sealErr        error
		want           string
	}{
		{"no row: minted unsealed, sealed, marked", true, false, false, nil, "lock mint-unsealed seal mark release"},
		{"a row sealed: nothing", true, true, true, nil, "lock release"},
		{"a row with no sealed credential: minted and sealed again", true, true, false, nil, "lock mint-unsealed seal mark release"},
		{"a seal that fails: not marked", true, false, false, errors.New("the vault did not take it"), "lock mint-unsealed seal release"},
		{"a rotation mints whatever stands", false, true, true, nil, "mint seal"},
	} {
		t.Run("issuing an account: "+c.name, func(t *testing.T) {
			acts := &anAccountStore{minted: c.minted, sealed: c.sealed}
			ctx, got := withIssued(t.Context())
			if c.only {
				ctx = onlyIfAbsent(ctx)
			}
			err := issueAccount(ctx, acts, inventory.BusUser{Username: "ace.mesh-build-go", Kind: inventory.BusModule},
				func(password string) error {
					acts.did("seal")
					if password == "" {
						t.Error("sealed no password")
					}
					return c.sealErr
				})
			if (err != nil) != (c.sealErr != nil) {
				t.Fatalf("error %v, want %v", err, c.sealErr)
			}
			if strings.Join(acts.acts, " ") != c.want {
				t.Fatalf("acts %q, want %q", strings.Join(acts.acts, " "), c.want)
			}
			if issued := len(got.list()) == 1; issued != (c.sealErr == nil && strings.Contains(c.want, "seal")) {
				t.Fatalf("said issued %v", got.list())
			}
		})
	}

	// The accounts of a module that no longer resolves on its node are stale, its provisioner client's among them; a
	// module still resolved (assigned or required), an account of another kind, and a node whose resolution is unknown
	// are left alone (issue 1945: ace's mesh-build-go rows from 17:19, marked sealed by migration 0108).
	t.Run("stale accounts", func(t *testing.T) {
		u := func(kind, node, module string) inventory.BusUser {
			return inventory.BusUser{Username: node + "." + module + "." + kind, Kind: kind, Node: node, Module: module}
		}
		users := map[string]inventory.BusUser{}
		for _, b := range []inventory.BusUser{u(inventory.BusModule, "ace", "mesh-build-go"),
			u(inventory.BusProcess, "ace", "mesh-build-go"), u(inventory.BusProvisioner, "ace", "mesh-provisioner"),
			u(inventory.BusModule, "ace", "mesh-build-node"), u(inventory.BusNode, "ace", ""),
			u(inventory.BusModule, "g14", "mesh-build-go")} {
			users[b.Username] = b
		}
		got := staleAccounts(users, map[string]map[string]bool{"ace": {"mesh-build-node": true}})
		if want := map[string][]string{"ace": {"mesh-build-go", "mesh-provisioner"}}; !reflect.DeepEqual(got, want) {
			t.Fatalf("stale %v, want %v", got, want)
		}
	})

	t.Run("a process may be issued only with its own broker secret", func(t *testing.T) {
		// mesh-mail's real shape: no `broker` of its own, its process declaring bus (review round 1 of issue 1181).
		if err := mayIssue(mailSenderWithProcess()); err != nil {
			t.Fatalf("a module with no broker of its own, whose process declares bus, was refused: %v", err)
		}
		m := mailSenderWithProcess()
		delete(m.OwnSecrets, "broker-send")
		err := mayIssue(m)
		if err == nil {
			t.Fatal("a process with no broker-<id> own secret was issued an account")
		}
		for _, want := range []string{"mesh-mail", "send", "broker-send", "own-secrets"} {
			if !strings.Contains(err.Error(), want) {
				t.Errorf("the refusal does not say %q: %v", want, err)
			}
		}
	})
}

// anAccountStore is an accountIssuer that writes down what was asked of it.
type anAccountStore struct {
	minted, sealed bool
	acts           []string
}

func (a *anAccountStore) did(act string) { a.acts = append(a.acts, act) }
func (a *anAccountStore) LockBusAccount(context.Context, string) (func(), error) {
	a.did("lock")
	return func() { a.did("release") }, nil
}
func (a *anAccountStore) BusAccountIssued(context.Context, string) (bool, bool, error) {
	return a.minted, a.sealed, nil
}
func (a *anAccountStore) MintBusPassword(context.Context, inventory.BusUser) (string, error) {
	a.did("mint")
	return "a-password", nil
}
func (a *anAccountStore) MintBusPasswordUnsealed(context.Context, inventory.BusUser) (string, error) {
	a.did("mint-unsealed")
	return "a-password", nil
}
func (a *anAccountStore) MarkBusAccountSealed(context.Context, string) error {
	a.did("mark")
	return nil
}