mesh-catalog/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go

package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/nats-io/nats-server/v2/server"
	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"
)

// The client's test list of novox/hq ADR 0349 ("How it is checked"), against a real bus (design 57 P5, issue 671):
// a template fills from each owner's keys; a missing value keeps the old file and reports what it waits for; a change
// refills, writes once and triggers the declared reaction; an owned file changed on disk is written back, a seed
// file is left, a managed file keeps the program's other keys; a module taken off the node loses its files and
// needs; a node coming back fills from current values.
//
// The bus is a real server, its accounts the client's as mesh-controller composes them (testdata/accounts.conf), so
// every read and write the client makes is one its grants allow. The node-engine is a stand-in on a real unix
// socket speaking its format (mesh-host internal/filling), writing each file as its ownership says — the
// node-engine's own tests prove its writes; these prove the client asks for the right ones — and counting each
// write that moved a file, which is what the node-engine reloads or restarts the program for.

const node = "g14"

// aBus is a server of this test alone with the client's buckets, the registry written as mesh-provisioner writes it.
type aBus struct {
	url    string
	domain jetstream.JetStream
	conn   *nats.Conn
	kv     map[string]jetstream.KeyValue
}

func startBus(t *testing.T) *aBus {
	t.Helper()
	accounts, err := os.ReadFile("testdata/accounts.conf")
	if err != nil {
		t.Fatal(err)
	}
	conf := filepath.Join(t.TempDir(), "bus.conf")
	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: \""+t.TempDir()+"\" }\n"+
		string(accounts)), 0o600); err != nil {
		t.Fatal(err)
	}
	opts, err := server.ProcessConfigFile(conf)
	if err != nil {
		t.Fatal(err)
	}
	opts.NoLog, opts.NoSigs = true, true
	s, err := server.NewServer(opts)
	if err != nil {
		t.Fatal(err)
	}
	go s.Start()
	if !s.ReadyForConnections(10 * time.Second) {
		t.Fatal("the bus did not come up")
	}
	t.Cleanup(s.Shutdown)
	nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("domain", "x"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(nc.Close)
	js, err := jetstream.New(nc)
	if err != nil {
		t.Fatal(err)
	}
	b := &aBus{url: s.ClientURL(), domain: js, conn: nc, kv: map[string]jetstream.KeyValue{}}
	for _, name := range []string{PlansBucket, PlacementBucket, ConfigBucket, RegistryBucket, AppliedBucket,
		CollectBucket, delivery.HaveBucket} {
		kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: name})
		if err != nil {
			t.Fatal(err)
		}
		b.kv[name] = kv
	}
	b.put(t, RegistryBucket, "config", `{"kind":"config","domain":"mesh-config","from":"bucket",`+
		`"bucket":"mesh-config_resolved","key":"<node>.<module>.<key>"}`)
	b.put(t, RegistryBucket, "secret", `{"kind":"secret","domain":"mesh-vault","from":"ask","ask":"mesh-vault.get"}`)
	for _, kind := range append([]string{CollectedKind}, oldCollectionKinds...) {
		b.put(t, RegistryBucket, kind, `{"kind":"`+kind+`","domain":"mesh-provisioner","from":"bucket",`+
			`"bucket":"mesh-provisioner_collect","key":"<node>.<collection>.<contributor>"}`)
	}
	for _, kind := range placementKinds {
		b.put(t, RegistryBucket, kind, `{"kind":"`+kind+`","domain":"mesh-placement","from":"bucket",`+
			`"bucket":"mesh-provisioner_plans","key":"<node>.<module>"}`)
	}
	return b
}

func (b *aBus) put(t *testing.T, bucket, key, value string) {
	t.Helper()
	if bucket == PlansBucket {
		// The seat mesh-provisioner keeps a plan, not an assignment (ADR 0570 clause 2): a module with no plan, or an
		// ended one, has no key.
		raw, ok := planned(value)
		if !ok {
			if err := b.kv[bucket].Delete(context.Background(), key); err != nil {
				t.Fatal(err)
			}
			return
		}
		value = string(raw)
	}
	if _, err := b.kv[bucket].PutString(context.Background(), key, value); err != nil {
		t.Fatalf("writing %s in %s: %v", key, bucket, err)
	}
}

// place writes this node's placement as the seat mesh-placement keeps it, naming exactly these modules.
func (b *aBus) place(t *testing.T, modules ...string) {
	t.Helper()
	p := delivery.Placement{V: 1, Node: node}
	for _, m := range modules {
		p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
	}
	keys := delivery.NewPlacementKeys()
	ctx := context.Background()
	have, _ := b.kv[PlacementBucket].Keys(ctx)
	for _, k := range have {
		if e, err := b.kv[PlacementBucket].Get(ctx, k); err == nil {
			_, _ = keys.Put(k, e.Value(), e.Revision(), false)
		}
	}
	writes, err := keys.Writes(node, &p)
	if err != nil {
		t.Fatal(err)
	}
	for _, w := range writes {
		if w.Value == nil {
			b.del(t, PlacementBucket, w.Key)
			continue
		}
		if _, err := b.kv[PlacementBucket].Put(ctx, w.Key, w.Value); err != nil {
			t.Fatal(err)
		}
	}
}

func (b *aBus) del(t *testing.T, bucket, key string) {
	t.Helper()
	if err := b.kv[bucket].Delete(context.Background(), key); err != nil {
		t.Fatal(err)
	}
}

// config writes a resolved value as mesh-config does.
func (b *aBus) config(t *testing.T, module, key, value string) {
	t.Helper()
	b.put(t, ConfigBucket, node+"."+module+"."+key, fmt.Sprintf(`{"node":%q,"module":%q,"key":%q,"value":%s,`+
		`"source":"node"}`, node, module, key, value))
}

// assign writes a module's assignment with its plan, as the controller does.
func (b *aBus) assign(t *testing.T, module string, plan Plan) {
	t.Helper()
	p, err := json.Marshal(plan)
	if err != nil {
		t.Fatal(err)
	}
	b.put(t, PlansBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+
		`"provisioning":%s}`, node, module, p))
}

// keys is the bucket's keys under prefix.
func (b *aBus) keys(t *testing.T, bucket, prefix string) []string {
	t.Helper()
	all, err := b.kv[bucket].Keys(context.Background())
	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
		t.Fatal(err)
	}
	var out []string
	for _, k := range all {
		if strings.HasPrefix(k, prefix) {
			out = append(out, k)
		}
	}
	return out
}

// reports collects what is said on prov.applied.
type reports struct {
	mu  sync.Mutex
	all []Report
}

func (b *aBus) listen(t *testing.T) *reports {
	t.Helper()
	r := &reports{}
	sub, err := b.conn.Subscribe("prov.applied."+node+".>", func(m *nats.Msg) {
		var rep Report
		if err := json.Unmarshal(m.Data, &rep); err == nil {
			r.mu.Lock()
			r.all = append(r.all, rep)
			r.mu.Unlock()
		}
	})
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = sub.Unsubscribe() })
	_ = b.conn.Flush()
	return r
}

func (r *reports) last(module string) (Report, int) {
	r.mu.Lock()
	defer r.mu.Unlock()
	n := 0
	var out Report
	for _, rep := range r.all {
		if rep.Module == module {
			out, n = rep, n+1
		}
	}
	return out, n
}

// engine stands in for the node-engine's write socket: each path written under root, as its ownership says.
type engine struct {
	root   string
	socket string
	mu     sync.Mutex
	owner  map[string]string // path → owned, seed or managed
	moved  map[string]int    // writes that changed the file: what the program is reloaded for
	asked  map[string]int
	wrote  map[string]string // what it last wrote
}

func startEngine(t *testing.T, owners map[string]string) *engine {
	t.Helper()
	e := newEngine(t, owners)
	e.listen(t)
	return e
}

// newEngine is an engine whose socket is not made yet: a node-engine not started, or replacing itself.
func newEngine(t *testing.T, owners map[string]string) *engine {
	t.Helper()
	// A unix socket's path is short: under /tmp, not the test's own directory.
	dir, err := os.MkdirTemp("/tmp", "p5-")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { os.RemoveAll(dir) })
	e := &engine{root: t.TempDir(), socket: filepath.Join(dir, "fill.sock"), owner: owners,
		moved: map[string]int{}, asked: map[string]int{}, wrote: map[string]string{}}
	return e
}

// listen makes the socket and answers on it until stop, which removes it as a node-engine stopping does.
func (e *engine) listen(t *testing.T) (stop func()) {
	t.Helper()
	l, err := net.Listen("unix", e.socket)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { l.Close() })
	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				return
			}
			go func() {
				defer conn.Close()
				var req FillRequest
				dec := json.NewDecoder(conn)
				dec.DisallowUnknownFields()
				if err := dec.Decode(&req); err != nil {
					_ = json.NewEncoder(conn).Encode(FillAnswer{Refused: "unreadable: " + err.Error()})
					return
				}
				_ = json.NewEncoder(conn).Encode(e.write(req))
			}()
		}
	}()
	return func() { l.Close(); os.Remove(e.socket) }
}

func (e *engine) at(path string) string { return filepath.Join(e.root, path) }

func (e *engine) write(req FillRequest) FillAnswer {
	e.mu.Lock()
	defer e.mu.Unlock()
	e.asked[req.Path]++
	owner, declared := e.owner[req.Path]
	if !declared {
		return FillAnswer{Path: req.Path, Refused: req.Path + " is not a file this node's declaration names as filled"}
	}
	at := e.at(req.Path)
	have, err := os.ReadFile(at)
	exists := err == nil
	want := string(req.Content)
	switch owner {
	case "seed":
		if exists {
			return FillAnswer{Path: req.Path, Action: "kept"}
		}
	case "managed":
		// The mesh's block between its marks; the program's lines around it kept.
		const begin, end = "# >>> mesh\n", "# <<< mesh\n"
		block := begin + want + end
		if exists {
			text := string(have)
			if i, j := strings.Index(text, begin), strings.Index(text, end); i >= 0 && j > i {
				block = text[:i] + block + text[j+len(end):]
			} else {
				block = text + block
			}
		}
		want = block
	}
	action := "created"
	switch {
	case exists && string(have) == want:
		return FillAnswer{Path: req.Path, Action: "unchanged", Digest: digestOf(have)}
	case exists && string(have) != e.wrote[req.Path]:
		action = "corrected"
	case exists:
		action = "updated"
	}
	_ = os.MkdirAll(filepath.Dir(at), 0o755)
	if err := os.WriteFile(at, []byte(want), 0o644); err != nil {
		return FillAnswer{Path: req.Path, Failed: err.Error()}
	}
	e.wrote[req.Path] = want
	e.moved[req.Path]++
	return FillAnswer{Path: req.Path, Action: action, Digest: digestOf([]byte(want))}
}

func (e *engine) counts(path string) (moved, asked int) {
	e.mu.Lock()
	defer e.mu.Unlock()
	return e.moved[path], e.asked[path]
}

func (e *engine) read(t *testing.T, path string) string {
	t.Helper()
	b, err := os.ReadFile(e.at(path))
	if err != nil {
		return ""
	}
	return string(b)
}

// templates is an artifact store holding one archive per name, and the plan's Templates for it.
func templates(t *testing.T, files map[string]string) (Templates, *httptest.Server) {
	t.Helper()
	var buf bytes.Buffer
	z := gzip.NewWriter(&buf)
	w := tar.NewWriter(z)
	for _, name := range sortedKeys(files) {
		if err := w.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(files[name])),
			Typeflag: tar.TypeReg}); err != nil {
			t.Fatal(err)
		}
		if _, err := w.Write([]byte(files[name])); err != nil {
			t.Fatal(err)
		}
	}
	if err := w.Close(); err != nil {
		t.Fatal(err)
	}
	if err := z.Close(); err != nil {
		t.Fatal(err)
	}
	body := buf.Bytes()
	srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { _, _ = rw.Write(body) }))
	t.Cleanup(srv.Close)
	return Templates{Source: srv.URL + "/files.tar.gz", Digest: digestOf(body)}, srv
}

// aClient runs a client on g14's account until the returned stop is called.
func aClient(t *testing.T, b *aBus, e *engine, state string) (*Client, func()) {
	t.Helper()
	nc, err := nats.Connect(b.url, nats.UserInfo("provisioner."+node, "x"),
		nats.CustomInboxPrefix("_INBOX.provisioner."+node),
		nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {
			t.Errorf("the bus refused the client: %v", err)
		}))
	if err != nil {
		t.Fatal(err)
	}
	js, err := jetstream.New(nc)
	if err != nil {
		t.Fatal(err)
	}
	secrets, err := NewVaultSecrets(nc, "provisioner."+node)
	if err != nil {
		t.Fatal(err)
	}
	secrets.Within = 2 * time.Second
	ctx, cancel := context.WithCancel(context.Background())
	up, err := watchSocket(ctx, e.socket)
	if err != nil {
		t.Fatal(err)
	}
	c := &Client{Node: node, JS: js, Conn: nc, Secrets: secrets, SocketUp: up,
		Fill:      SocketFiller{Socket: e.socket, Busy: 10 * time.Millisecond, Tries: 3},
		Templates: &templateStore{dir: state, fetch: HTTPFetch}}
	done := make(chan error, 1)
	go func() { done <- c.Run(ctx) }()
	stopped := false
	stop := func() {
		if stopped {
			return
		}
		stopped = true
		cancel()
		if err := <-done; err != nil && !errors.Is(err, context.Canceled) {
			t.Errorf("the client stopped with %v", err)
		}
		nc.Close()
	}
	t.Cleanup(stop)
	return c, stop
}

func eventually(t *testing.T, what string, ok func() bool) {
	t.Helper()
	deadline := time.Now().Add(30 * time.Second)
	for !ok() {
		if time.Now().After(deadline) {
			t.Fatalf("not within 30s: %s", what)
		}
		time.Sleep(20 * time.Millisecond)
	}
}

const dunstrc = "/home/nox/.config/dunst/dunstrc"

// dunst is a module with one owned file naming a config value and a placement value.
func dunst(t *testing.T) Plan {
	tpl, _ := templates(t, map[string]string{
		"dunstrc": "[global]\nfont = Monospace ${config:font-size}\nicons = ${machine:account-home}/.icons\n"})
	return Plan{Templates: tpl, Files: []PlannedFile{{ID: "dunstrc", Path: dunstrc, Template: "dunstrc",
		OwnerAtRuntime: "owned", Takes: "reload-on"}},
		Placement: map[string]string{"machine:account-home": "/home/nox"}}
}

func TestNatsATemplateFillsFromEachOwnersKeys(t *testing.T) {
	b := startBus(t)
	heard := b.listen(t)
	e := startEngine(t, map[string]string{dunstrc: "owned"})
	b.config(t, "dunst", "font-size", "13")
	b.assign(t, "dunst", dunst(t))
	aClient(t, b, e, t.TempDir())

	want := "[global]\nfont = Monospace 13\nicons = /home/nox/.icons\n"
	eventually(t, "dunstrc is filled", func() bool { return e.read(t, dunstrc) == want })
	eventually(t, "prov.applied says applied", func() bool {
		r, _ := heard.last("dunst")
		return r.Status == StatusApplied
	})
	r, _ := heard.last("dunst")
	if r.Templates == "" || len(r.Files) != 1 || r.Files[0].Action != "created" || r.Files[0].Takes != "reload-on" {
		t.Errorf("prov.applied said %+v", r)
	}
	kept, err := b.kv[AppliedBucket].Get(context.Background(), node+".dunst")
	if err != nil || !strings.Contains(string(kept.Value()), `"status":"applied"`) {
		t.Errorf("mesh-provisioner_applied keeps %v (%v)", kept, err)
	}
}

func TestNatsAMissingValueKeepsTheOldFileAndSaysWhatItWaitsFor(t *testing.T) {
	b := startBus(t)
	heard := b.listen(t)
	e := startEngine(t, map[string]string{dunstrc: "owned"})
	b.assign(t, "dunst", dunst(t))
	aClient(t, b, e, t.TempDir())

	// No value yet: nothing written, and the wait said.
	eventually(t, "prov.applied says it waits", func() bool {
		r, _ := heard.last("dunst")
		return r.Status == StatusWaiting
	})
	r, _ := heard.last("dunst")
	if len(r.WaitsFor) != 1 || !strings.HasPrefix(r.WaitsFor[0], "config:font-size") {
		t.Errorf("it waits for %v", r.WaitsFor)
	}
	if _, asked := e.counts(dunstrc); asked != 0 {
		t.Errorf("a file waiting for a value was written %d times", asked)
	}
	// Filled once it is there; and a value gone keeps the file there is.
	b.config(t, "dunst", "font-size", "13")
	eventually(t, "dunstrc is filled", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 13") })
	b.del(t, ConfigBucket, node+".dunst.font-size")
	eventually(t, "prov.applied says it waits again", func() bool {
		r, _ := heard.last("dunst")
		return r.Status == StatusWaiting
	})
	if !strings.Contains(e.read(t, dunstrc), "Monospace 13") {
		t.Errorf("a value gone changed the file there was: %q", e.read(t, dunstrc))
	}
	if _, asked := e.counts(dunstrc); asked != 1 {
		t.Errorf("dunstrc was asked for %d times, want 1", asked)
	}
}

func TestNatsAChangeRefillsWritesOnceAndTheProgramTakesIt(t *testing.T) {
	b := startBus(t)
	heard := b.listen(t)
	e := startEngine(t, map[string]string{dunstrc: "owned"})
	b.config(t, "dunst", "font-size", "13")
	b.assign(t, "dunst", dunst(t))
	aClient(t, b, e, t.TempDir())
	eventually(t, "dunstrc is filled", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 13") })

	b.config(t, "dunst", "font-size", "15")
	eventually(t, "dunstrc takes the change", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 15") })
	eventually(t, "prov.applied says it", func() bool {
		r, _ := heard.last("dunst")
		return r.Status == StatusApplied && len(r.Files) == 1 && r.Files[0].Action == "updated"
	})
	// The same value again, and a key the template does not name: the next fill asks the node-engine for the file
	// again (ADR 0349 clause 5), which answers unchanged, so the program is not reloaded for it.
	b.config(t, "dunst", "font-size", "15")
	b.config(t, "dunst", "geometry", `"300x5"`)
	eventually(t, "the node-engine is asked for the file again by each fill", func() bool { _, asked := e.counts(dunstrc); return asked >= 3 })
	time.Sleep(300 * time.Millisecond) // any stray extra write would come now
	if moved, asked := e.counts(dunstrc); moved != 2 || asked != 3 {
		t.Errorf("two values wrote dunstrc %d times (asked %d); want 2 and 3: the program is reloaded once a change, "+
			"the file asked again each fill", moved, asked)
	}
}

func TestNatsAnOwnedFileIsWrittenBackASeedLeftAndAManagedFileKeepsTheProgramsLines(t *testing.T) {
	const seed, managed = "/home/nox/.config/app/first.conf", "/home/nox/.config/app/settings.ini"
	b := startBus(t)
	heard := b.listen(t)
	e := startEngine(t, map[string]string{dunstrc: "owned", seed: "seed", managed: "managed"})
	tpl, _ := templates(t, map[string]string{"dunstrc": "font = ${config:font-size}\n",
		"first.conf": "start = ${config:font-size}\n", "settings.ini": "size = ${config:font-size}\n"})
	b.config(t, "app", "font-size", "13")
	b.assign(t, "app", Plan{Templates: tpl, Files: []PlannedFile{
		{ID: "rc", Path: dunstrc, Template: "dunstrc", OwnerAtRuntime: "owned"},
		{ID: "first", Path: seed, Template: "first.conf", OwnerAtRuntime: "seed"},
		{ID: "settings", Path: managed, Template: "settings.ini", OwnerAtRuntime: "managed", Takes: "read-on-use"}}})
	aClient(t, b, e, t.TempDir())
	eventually(t, "all three are placed", func() bool {
		r, _ := heard.last("app")
		return r.Status == StatusApplied
	})

	// The program changes each on disk.
	for path, text := range map[string]string{dunstrc: "font = 99\n", seed: "start = mine\n",
		managed: "theme = dark\n" + e.read(t, managed) + "recent = a,b\n"} {
		if err := os.WriteFile(e.at(path), []byte(text), 0o644); err != nil {
			t.Fatal(err)
		}
	}

	// 1129/AC3: with nothing reaching the module over the bus, an owned file changed on disk stays as the program
	// left it (well over any former drift interval).
	time.Sleep(1500 * time.Millisecond)
	if got := e.read(t, dunstrc); got != "font = 99\n" {
		t.Errorf("an owned file changed on disk, with nothing over the bus, reads %q, want the program's", got)
	}

	// 1129/AC2: a config value of the module changes over the bus, so every file of the module is written again
	// through the node-engine (ADR 0349 clause 5): the owned file is put back, the seed keeps the program's text,
	// and the managed file keeps the program's lines with the mesh's part.
	b.config(t, "app", "font-size", "14")
	eventually(t, "the owned file is written back", func() bool { return e.read(t, dunstrc) == "font = 14\n" })
	eventually(t, "prov.applied says it was put back", func() bool {
		r, _ := heard.last("app")
		return r.Status == StatusApplied && len(r.Files) == 3 && r.Files[0].Action == "corrected"
	})
	if got := e.read(t, seed); got != "start = mine\n" {
		t.Errorf("the seed file is %q: written once, then the program's", got)
	}
	if got := e.read(t, managed); !strings.Contains(got, "theme = dark\n") || !strings.Contains(got, "recent = a,b\n") ||
		!strings.Contains(got, "size = 14\n") {
		t.Errorf("the managed file is %q: the mesh's block and the program's own lines", got)
	}

	// A further reconcile that moves nothing is said, because it reads unchanged where the last read corrected; the
	// one after that, which reads the same as it, is not said again.
	_, afterCorrected := heard.last("app")
	b.config(t, "app", "geometry", `"300x5"`) // a value the templates do not name: a reason to look, not to move
	eventually(t, "a reconcile that moves nothing is said once, reading unchanged", func() bool {
		_, n := heard.last("app")
		return n == afterCorrected+1
	})
	_, afterUnchanged := heard.last("app")
	b.config(t, "app", "geometry", `"300x5"`)
	time.Sleep(600 * time.Millisecond)
	if _, after := heard.last("app"); after != afterUnchanged {
		t.Errorf("a reconcile whose report reads the same as the last one said %d more reports", after-afterUnchanged)
	}
}

// 1129/AC1: with every module filled and nothing reaching the client over the bus, it asks the node-engine for no
// write, asks the vault nothing, and publishes no report — there is no timer that asks again on its own.
func TestNatsWithNothingOverTheBusItAsksForNoWriteNoSecretAndNoReport(t *testing.T) {
	b := startBus(t)
	heard := b.listen(t)
	vault := startVault(t, b)
	vault.keep("own/g14/app/app-token", "tok", false)
	vault.keep("own/g14/app/db-password", "pw", false)
	e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"})
	b.config(t, "app", "font-size", "13")
	b.assign(t, "app", app(t))
	aClient(t, b, e, t.TempDir())

	eventually(t, "app is filled", func() bool {
		r, _ := heard.last("app")
		return r.Status == StatusApplied
	})
	_, askedBefore := e.counts(appConf)
	vaultBefore := vault.times("own/g14/app/app-token") + vault.times("own/g14/app/db-password")
	_, reportsBefore := heard.last("app")

	// Well over any former drift interval used in these tests (100ms, 200ms): nothing reaches the client.
	time.Sleep(1500 * time.Millisecond)

	if _, askedAfter := e.counts(appConf); askedAfter != askedBefore {
		t.Errorf("the node-engine was asked to write %s again, with nothing over the bus", appConf)
	}
	if n := vault.times("own/g14/app/app-token") + vault.times("own/g14/app/db-password"); n != vaultBefore {
		t.Error("the vault was asked again, with nothing over the bus")
	}
	if _, after := heard.last("app"); after != reportsBefore {
		t.Error("a report was published again, with nothing over the bus")
	}
}

func TestNatsAModuleTakenOffTheNodeLosesItsReport(t *testing.T) {
	b := startBus(t)
	heard := b.listen(t)
	e := startEngine(t, map[string]string{dunstrc: "owned"})
	b.config(t, "dunst", "font-size", "13")
	b.assign(t, "dunst", dunst(t))
	aClient(t, b, e, t.TempDir())
	eventually(t, "its report is kept", func() bool { return len(b.keys(t, AppliedBucket, node+".dunst")) == 1 })

	b.del(t, PlansBucket, node+".dunst")
	eventually(t, "its report goes", func() bool { return len(b.keys(t, AppliedBucket, node+".dunst")) == 0 })
	eventually(t, "prov.applied says it is removed", func() bool {
		r, _ := heard.last("dunst")
		return r.Status == StatusRemoved
	})
}

// Issue 1126 B12 (ADR 0590): the controller's assignments are frozen and go, so what places a module here is the seat
// mesh-placement's placement alone. A module stays while placement names it, even with no plan of the seat
// mesh-provisioner (nats on novox runs without a want): its last plan keeps being filled, and it leaves only when
// neither its plan nor its placement is there.
func TestNatsAModuleStaysWhilePlacementNamesItAndLeavesOnlyWhenNeitherPlacesIt(t *testing.T) {
	b := startBus(t)
	heard := b.listen(t)
	e := startEngine(t, map[string]string{dunstrc: "owned"})
	b.config(t, "dunst", "font-size", "13")
	b.place(t, "dunst")
	b.assign(t, "dunst", dunst(t))
	aClient(t, b, e, t.TempDir())
	eventually(t, "dunstrc is filled from the seat's plan", func() bool {
		return strings.Contains(e.read(t, dunstrc), "Monospace 13")
	})

	// The seat's plan goes, placement stays: the module stays, filled from its last plan, its report kept.
	b.del(t, PlansBucket, node+".dunst")
	b.config(t, "dunst", "font-size", "15")
	eventually(t, "dunstrc follows a change with only placement naming it", func() bool {
		return strings.Contains(e.read(t, dunstrc), "Monospace 15")
	})
	if r, _ := heard.last("dunst"); r.Status == StatusRemoved || len(b.keys(t, AppliedBucket, node+".dunst")) != 1 {
		t.Fatalf("the module left when its plan went, though placement still names it: %+v", r)
	}

	// Neither places it: it leaves.
	b.place(t)
	eventually(t, "prov.applied says it is removed", func() bool {
		r, _ := heard.last("dunst")
		return r.Status == StatusRemoved
	})
}

func TestNatsANodeComingBackFillsFromTheCurrentValues(t *testing.T) {
	b := startBus(t)
	const zshrc = "/home/nox/.zshrc"
	e := startEngine(t, map[string]string{dunstrc: "owned", zshrc: "owned"})
	b.config(t, "dunst", "font-size", "13")
	b.assign(t, "dunst", dunst(t))
	b.config(t, "zsh", "prompt", `"%"`)
	tpl, _ := templates(t, map[string]string{"zshrc": "PS1=${config:prompt}\n"})
	b.assign(t, "zsh", Plan{Templates: tpl, Files: []PlannedFile{{ID: "zshrc", Path: zshrc, Template: "zshrc",
		OwnerAtRuntime: "owned"}}})
	state := t.TempDir()
	_, stop := aClient(t, b, e, state)
	eventually(t, "both are filled", func() bool {
		return strings.Contains(e.read(t, dunstrc), "Monospace 13") && e.read(t, zshrc) == "PS1=%\n"
	})

	// Away: a value changes and a module leaves, and the artifact store is gone.
	stop()
	b.config(t, "dunst", "font-size", "17")
	b.del(t, PlansBucket, node+".zsh")

	heard := b.listen(t)
	aClient(t, b, e, state)
	eventually(t, "dunstrc has the value set while it was away", func() bool {
		return strings.Contains(e.read(t, dunstrc), "Monospace 17")
	})
	eventually(t, "and its report", func() bool {
		r, _ := heard.last("zsh")
		return r.Status == StatusRemoved && len(b.keys(t, AppliedBucket, node+".zsh")) == 0
	})
}

// A plan that cannot be acted on, templates that are not the ones named, and a secret the plan does not name each
// fail, with the reason; nothing is written.
func TestNatsWhatCannotBeFilledFailsWithTheReason(t *testing.T) {
	b := startBus(t)
	heard := b.listen(t)
	e := startEngine(t, map[string]string{dunstrc: "owned", "/etc/app.conf": "owned"})
	aClient(t, b, e, t.TempDir())

	b.put(t, PlansBucket, node+".broken", `{"node":"g14","module":"broken","provisioning":{"templates":`+
		`{"source":"x","digest":"sha256:00"},"files":[],"surprise":1}}`)
	p := dunst(t)
	p.Templates.Digest = "sha256:" + strings.Repeat("0", 64)
	b.assign(t, "forged", p)
	tpl, _ := templates(t, map[string]string{"app.conf": "token = ${secret:app-token}\n"})
	b.assign(t, "secretive", Plan{Templates: tpl, Files: []PlannedFile{{ID: "conf", Path: "/etc/app.conf",
		Template: "app.conf", OwnerAtRuntime: "owned"}}})

	for module, why := range map[string]string{"broken": "surprise", "forged": "and the plan names",
		"secretive": "the plan names no secret app-token"} {
		eventually(t, module+" fails", func() bool {
			r, _ := heard.last(module)
			return r.Status == StatusFailed && strings.Contains(r.Reason, why)
		})
	}
	if _, asked := e.counts("/etc/app.conf"); asked != 0 {
		t.Errorf("a file that cannot be filled was written")
	}
}

// TestNatsResticsFilesFillFromTheBackupPlacement is the backup lines as placement (issue 794): restic's two files
// are the plan's `backup:lines` and `backup:data`, each exactly the text the controller composed, trailing newline
// and all, and a changed plan refills them.
func TestNatsResticsFilesFillFromTheBackupPlacement(t *testing.T) {
	const backups, data = "/var/lib/restic/backups.conf", "/var/lib/restic/data.conf"
	b := startBus(t)
	e := startEngine(t, map[string]string{backups: "owned", data: "owned"})
	tpl, _ := templates(t, map[string]string{
		"backups.conf": "# What the modules on this machine back up, composed by the mesh. Do not edit.\n${backup:lines}",
		"data.conf":    "# The data the modules on this machine declare, composed by the mesh. Do not edit.\n${backup:data}"})
	plan := func(lines, declared string) Plan {
		return Plan{Templates: tpl, Files: []PlannedFile{
			{ID: "declared", Path: backups, Template: "backups.conf", OwnerAtRuntime: "owned"},
			{ID: "data-declared", Path: data, Template: "data.conf", OwnerAtRuntime: "owned"}},
			Placement: map[string]string{"backup:lines": lines, "backup:data": declared}}
	}
	b.assign(t, "restic", plan("# a\n/srv/a\n# b\n/srv/b\n", "# a\nstate\n"))
	aClient(t, b, e, t.TempDir())
	head := "# What the modules on this machine back up, composed by the mesh. Do not edit.\n"
	eventually(t, "backups.conf is filled", func() bool { return e.read(t, backups) == head+"# a\n/srv/a\n# b\n/srv/b\n" })
	eventually(t, "data.conf is filled", func() bool {
		return e.read(t, data) == "# The data the modules on this machine declare, composed by the mesh. Do not edit.\n# a\nstate\n"
	})
	b.assign(t, "restic", plan("# a\n/srv/a\n", "# a\nstate\n"))
	eventually(t, "a module taken away refills backups.conf", func() bool { return e.read(t, backups) == head+"# a\n/srv/a\n" })
}

// defaults writes the defaults a module's manifest declares as mesh-config resolves them (source default), for a
// moved module whose templates read config values (issue 1780).
func (b *aBus) defaults(t *testing.T, module string, manifest []byte) {
	t.Helper()
	var m struct {
		Config map[string]struct {
			Default json.RawMessage `json:"default"`
		} `json:"config"`
	}
	if err := json.Unmarshal(manifest, &m); err != nil {
		t.Fatal(err)
	}
	for key, d := range m.Config {
		if len(d.Default) > 0 && string(d.Default) != "null" {
			b.put(t, ConfigBucket, node+"."+module+"."+key, fmt.Sprintf(`{"node":%q,"module":%q,"key":%q,"value":%s,`+
				`"source":"default"}`, node, module, key, d.Default))
		}
	}
}

// The node-engine moving to a new composition of the node's own fills every module again: an apply may have left a
// filled file as the composition had it (issue 1945, live on 2026-10-25: release 6's node-engines wrote filled files
// empty, and the client, which fills at its start and on a change of a value, filled nothing again).
func TestNatsTheNodesOwnHaveMovingFillsEveryModuleAgain(t *testing.T) {
	b := startBus(t)
	e := startEngine(t, map[string]string{dunstrc: "owned"})
	own := func(digit string) {
		raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: "sha256:" + strings.Repeat(digit, 64)})
		if err != nil {
			t.Fatal(err)
		}
		b.put(t, delivery.HaveBucket, node+".node", string(raw))
	}
	own("a")
	b.config(t, "dunst", "font-size", "13")
	b.assign(t, "dunst", dunst(t))
	aClient(t, b, e, t.TempDir())
	eventually(t, "dunstrc is filled", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 13") })

	// The node's own composition empties it, and the node-engine says it runs that composition.
	if err := os.WriteFile(e.at(dunstrc), nil, 0o644); err != nil {
		t.Fatal(err)
	}
	own("b")
	eventually(t, "dunstrc is filled again", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 13") })

	// The same composition said again fills nothing.
	_, before := e.counts(dunstrc)
	own("b")
	b.config(t, "dunst", "font-size", "15")
	eventually(t, "the change is written", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 15") })
	if _, after := e.counts(dunstrc); after-before != 1 {
		t.Errorf("dunstrc was asked %d times for one change", after-before)
	}
}

// planned is an assignment's value as the seat mesh-provisioner keeps its plan, `{v, node, module, plan}`; false for one
// with no plan, or ended, or in a state, which has no key there.
func planned(assignment string) ([]byte, bool) {
	var a struct {
		Node         string          `json:"node"`
		Module       string          `json:"module"`
		Provisioning json.RawMessage `json:"provisioning"`
		Ended        json.RawMessage `json:"ended"`
		State        json.RawMessage `json:"state"`
	}
	if json.Unmarshal([]byte(assignment), &a) != nil || len(a.Provisioning) == 0 || string(a.Provisioning) == "null" ||
		len(a.Ended) > 0 || len(a.State) > 0 {
		return nil, false
	}
	raw, _ := json.Marshal(map[string]any{"v": 1, "node": a.Node, "module": a.Module, "plan": a.Provisioning})
	return raw, true
}

// A standing failure keeps when it began while only its words change, so the controller's apply-failing alarm judges
// its age (the sweep of 2026-10-30, issue 774: novox's reason changed at 16:56 and the alarm cleared for five minutes
// while every module still failed); it begins anew after any other status, and a report from before the field gives
// its time.
func TestAFailureKeepsWhenItBeganWhileItsWordsChange(t *testing.T) {
	t0 := time.Date(2026, 10, 30, 15, 0, 0, 0, time.UTC)
	now := t0.Add(time.Hour)
	failed := func(why string, since time.Time) *Report {
		return &Report{Status: StatusFailed, Reason: why, At: since.Add(time.Minute), FailingSince: since}
	}
	for _, row := range []struct {
		name   string
		r      Report
		before *Report
		want   time.Time
	}{
		{"a first failure", Report{Status: StatusFailed, Reason: "a"}, nil, now},
		{"the same failure in other words", Report{Status: StatusFailed, Reason: "the vault refused it"}, failed("bus refused", t0), t0},
		{"a failure after an apply", Report{Status: StatusFailed, Reason: "a"}, &Report{Status: StatusApplied, At: t0}, now},
		{"a failure after waiting", Report{Status: StatusFailed, Reason: "a"}, &Report{Status: StatusWaiting, At: t0}, now},
		{"a report from before the field", Report{Status: StatusFailed, Reason: "b"}, &Report{Status: StatusFailed, Reason: "a", At: t0}, t0},
		{"an apply", Report{Status: StatusApplied}, failed("a", t0), time.Time{}},
	} {
		if got := failingSince(row.r, row.before, now); !got.Equal(row.want) {
			t.Errorf("%s: failing since %v, want %v", row.name, got, row.want)
		}
	}
}

// A plan of provisions alone, a module not handed whose route the helper answers (issue 774), names no templates:
// nothing is fetched and nothing filled, where a fetch of no source failed the module's report.
func TestAPlanOfProvisionsAloneFetchesNoTemplates(t *testing.T) {
	s := &templateStore{dir: t.TempDir(), fetch: func(context.Context, string) ([]byte, error) {
		t.Fatal("a plan with no templates fetched")
		return nil, nil
	}}
	texts, err := s.load(context.Background(), Templates{})
	if err != nil || len(texts) != 0 {
		t.Fatalf("loaded %v: %v", texts, err)
	}
	if targets, wrong := fillTargets(Plan{}, texts); wrong != "" || len(targets) != 0 {
		t.Fatalf("filled %v: %s", targets, wrong)
	}
	// And such a plan hands its module to nobody (catalogue.Plan.Hands): read as no plan, never refused as one that
	// names no source, and the module's files stay its node's.
	routeOnly := `{"node":"g14","module":"grafana","provisioning":{"templates":{"digest":"","source":""},"files":null,` +
		`"provisions":[{"provision":"route","provider":"g14","provider-module":"route-proxy"}]}}`
	if module, _, err := planOf("g14", "g14.grafana", []byte(routeOnly)); module != "grafana" || !errors.Is(err, errNoPlan) {
		t.Fatalf("a plan of provisions alone reads as %s: %v, want no plan", module, err)
	}
}