mesh-host/internal/agent/agent_test.go

package agent

import (
	"context"
	"crypto/ed25519"
	"encoding/json"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

const node = "n1"

// The keys of the tests: the controller's and the operator's are trusted, a stranger's is not.
var controllerPub, controllerKey = newKey()
var operatorPub, operatorKey = newKey()
var _, strangerKey = newKey()

func newKey() (ed25519.PublicKey, ed25519.PrivateKey) {
	pub, key, err := ed25519.GenerateKey(nil)
	if err != nil {
		panic(err)
	}
	return pub, key
}

func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }

// A watcher the test writes to; it says Caught once the keys written before the watch began are out.
type feed struct {
	mu    sync.Mutex
	ch    chan Update
	rev   uint64
	first []Update
}

func newFeed() *feed { return &feed{ch: make(chan Update, 256)} }

func (f *feed) Watch(_ context.Context, _ string) (<-chan Update, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	for _, u := range f.first {
		f.ch <- u
	}
	f.ch <- Update{Caught: true}
	return f.ch, nil
}

// before writes a key as if it was there when the watch began.
func (f *feed) before(key string, value []byte) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.rev++
	f.first = append(f.first, Update{Key: key, Value: value, Revision: f.rev})
}

func (f *feed) put(key string, value []byte) {
	f.mu.Lock()
	f.rev++
	r := f.rev
	f.mu.Unlock()
	f.ch <- Update{Key: key, Value: value, Revision: r}
}

type haves struct {
	mu      sync.Mutex
	stored  map[string]Have
	log     []Have
	removed []string
}

func (h *haves) Read(_ context.Context, module string) (Have, bool) {
	h.mu.Lock()
	defer h.mu.Unlock()
	x, ok := h.stored[module]
	return x, ok
}
func (h *haves) Write(_ context.Context, x Have) {
	h.mu.Lock()
	defer h.mu.Unlock()
	if h.stored == nil {
		h.stored = map[string]Have{}
	}
	h.stored[x.Module] = x
	h.log = append(h.log, x)
}
func (h *haves) Removed(_ context.Context, m string) {
	h.mu.Lock()
	defer h.mu.Unlock()
	h.removed = append(h.removed, m)
}
func (h *haves) last(module string) Have {
	h.mu.Lock()
	defer h.mu.Unlock()
	return h.stored[module]
}

type bed struct {
	t        *testing.T
	wants    *feed
	comps    *feed
	haves    *haves
	mu       sync.Mutex
	applied  []string
	removed  []string
	backups  []string
	failWith string
	backup   func(module string) Backup
	kept     map[string][]json.RawMessage
	others   [][]delivery.Composed
	said     []string
	env      Env
}

func newBed(t *testing.T) *bed {
	b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}}
	b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves,
		Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed) Result {
			b.mu.Lock()
			defer b.mu.Unlock()
			b.others = append(b.others, others)
			name := c.Module
			if name == "" {
				name = NodeModule
			}
			b.applied = append(b.applied, name)
			if b.failWith != "" {
				return Result{Failed: true, Detail: b.failWith}
			}
			return Result{}
		},
		Remove: func(_ context.Context, m string, _ []delivery.Composed) Result {
			b.mu.Lock()
			defer b.mu.Unlock()
			b.removed = append(b.removed, m)
			return Result{}
		},
		Backup: func(_ context.Context, m string) Backup {
			b.mu.Lock()
			defer b.mu.Unlock()
			b.backups = append(b.backups, m)
			if b.backup != nil {
				return b.backup(m)
			}
			return Backup{Done: true}
		},
		Kept:    func() map[string][]json.RawMessage { return b.kept },
		Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub, operatorPub} },
		Say: func(s string) {
			t.Log(s)
			b.mu.Lock()
			b.said = append(b.said, s)
			b.mu.Unlock()
		},
	}
	return b
}

func (b *bed) run() (stop func()) {
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() { defer close(done); _ = New(b.env).Run(ctx) }()
	return func() { cancel(); <-done }
}

func file(module, id, content string) json.RawMessage {
	raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id,
		"content": content, "module": module})
	return raw
}

// composition is a module's composition at a fingerprint: its key, its bytes and its digest, from the writer's code.
func composition(t *testing.T, module, fp string, data bool, rs ...json.RawMessage) (key string, raw []byte, digest string) {
	t.Helper()
	c := delivery.Composed{Node: node, Module: module, FP: fp, Data: data, Resources: rs}
	var err error
	if module == NodeModule {
		c.Module, c.FP = "", ""
		key, err = delivery.ComposedNodeKey(node)
	} else {
		key, err = delivery.ComposedKey(node, module, fp)
	}
	if err != nil {
		t.Fatal(err)
	}
	return signedBy(t, c, key, controllerKey)
}

// signedBy is c signed with signer, as the writer's code signs it: its key, its bytes and its digest.
func signedBy(t *testing.T, c delivery.Composed, key string, signer ed25519.PrivateKey) (string, []byte, string) {
	t.Helper()
	signed, err := delivery.SignComposed(c, signer)
	if err != nil {
		t.Fatal(err)
	}
	raw, err := delivery.EncodeComposed(signed)
	if err != nil {
		t.Fatal(err)
	}
	digest, err := delivery.ComposedDigestOf(signed)
	if err != nil {
		t.Fatal(err)
	}
	return key, raw, digest
}

func wantFor(t *testing.T, module, fp, digest, backup string) (string, []byte) {
	t.Helper()
	key, err := delivery.WantKey(node, module)
	if err != nil {
		t.Fatal(err)
	}
	raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest, Backup: backup})
	if err != nil {
		t.Fatal(err)
	}
	return key, raw
}

func eventually(t *testing.T, what string, ok func() bool) {
	t.Helper()
	deadline := time.Now().Add(3 * time.Second)
	for !ok() {
		if time.Now().After(deadline) {
			t.Fatal(what)
		}
		time.Sleep(5 * time.Millisecond)
	}
}

func (b *bed) appliedList() []string {
	b.mu.Lock()
	defer b.mu.Unlock()
	return append([]string(nil), b.applied...)
}

// R6-A AC1 (the agent, with a bus that is a fake): given a want naming a digest and that composition, the module is
// applied and its have names the fingerprint and the digest; a composition whose digest no want names is not applied.
func TestAWantAndItsCompositionApplyTheModuleAndTheHaveNamesBoth(t *testing.T) {
	b := newBed(t)
	stop := b.run()
	defer stop()

	fp := fpOf("a")
	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
	// A composition no want names yet is not applied.
	b.comps.put(ck, craw)
	time.Sleep(100 * time.Millisecond)
	if got := b.appliedList(); len(got) != 0 {
		t.Fatalf("a composition no want names was applied: %v", got)
	}
	wk, wraw := wantFor(t, "web", fp, dig, "")
	b.wants.put(wk, wraw)
	eventually(t, "the module was not applied", func() bool { return len(b.appliedList()) == 1 })
	eventually(t, "the have does not name both", func() bool {
		h := b.haves.last("web")
		return h.FP == fp && h.Composed == dig && h.Phase == string(delivery.PhaseRunning)
	})

	// A want naming a digest the composition at that fingerprint does not have: nothing is applied.
	fp2 := fpOf("b")
	ck2, craw2, _ := composition(t, "web", fp2, false, file("web", "x", "2"))
	b.comps.put(ck2, craw2)
	wk2, wraw2 := wantFor(t, "web", fp2, "sha256:"+strings.Repeat("0", 64), "")
	b.wants.put(wk2, wraw2)
	time.Sleep(100 * time.Millisecond)
	if got := b.appliedList(); len(got) != 1 {
		t.Fatalf("a want naming another digest applied the composition: %v", got)
	}
}

// R6-A AC2: given a kept declaration and wants and compositions equal to it, when the agent starts, nothing is applied
// and every have names its fingerprint and digest.
func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) {
	b := newBed(t)
	fp := fpOf("a")
	rs := []json.RawMessage{file("web", "x", "1"), file("web", "y", "2")}
	ck, craw, dig := composition(t, "web", fp, false, rs...)
	wk, wraw := wantFor(t, "web", fp, dig, "")
	b.comps.before(ck, craw)
	b.wants.before(wk, wraw)
	// The kept declaration holds the same resources, in another order and spacing.
	b.kept = map[string][]json.RawMessage{"web": {json.RawMessage(strings.ReplaceAll(string(rs[1]), ",", " , ")), rs[0]}}
	stop := b.run()
	defer stop()
	eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig })
	if h := b.haves.last("web"); h.FP != fp || h.Phase != string(delivery.PhaseRunning) {
		t.Fatalf("the have: %+v", h)
	}
	time.Sleep(50 * time.Millisecond)
	if got := b.appliedList(); len(got) != 0 {
		t.Fatalf("a module that already runs what the want names was applied: %v", got)
	}
}

// What the node-engine itself last applied is what runs after a restart, not the kept declaration.
func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
	b := newBed(t)
	fp := fpOf("a")
	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
	wk, wraw := wantFor(t, "web", fp, dig, "")
	b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: "running"}}
	b.comps.before(ck, craw)
	b.wants.before(wk, wraw)
	stop := b.run()
	defer stop()
	time.Sleep(150 * time.Millisecond)
	if got := b.appliedList(); len(got) != 0 {
		t.Fatalf("a module that runs what the want names was applied after a restart: %v", got)
	}
}

// A module found running on its own have after a restart is said again, once, through the haves: the session then holds
// it and writes the health this engine judges. Not said, its key kept the health an older engine wrote: xorg's have on
// g14 and shanks said unknown from before the engine said a module with nothing to judge ok, and the assigner held every
// module requiring x11-display back for it (2026-10-25).
func TestStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines(t *testing.T) {
	b := newBed(t)
	fp := fpOf("a")
	ck, craw, dig := composition(t, "xorg", fp, false, file("xorg", "x", "1"))
	wk, wraw := wantFor(t, "xorg", fp, dig, "")
	b.haves.stored = map[string]Have{"xorg": {Module: "xorg", FP: fp, Composed: dig, Phase: "running", Detail: "kept"}}
	b.comps.before(ck, craw)
	b.wants.before(wk, wraw)
	stop := b.run()
	defer stop()
	eventually(t, "the have found running was not said again", func() bool {
		b.haves.mu.Lock()
		defer b.haves.mu.Unlock()
		return len(b.haves.log) > 0
	})
	time.Sleep(50 * time.Millisecond)
	b.haves.mu.Lock()
	log := append([]Have(nil), b.haves.log...)
	b.haves.mu.Unlock()
	if len(log) != 1 {
		t.Fatalf("the have was said %d times, want once: %+v", len(log), log)
	}
	if h := log[0]; h.Module != "xorg" || h.FP != fp || h.Composed != dig || h.Phase != string(delivery.PhaseRunning) || h.Detail != "kept" {
		t.Fatalf("the have said again: %+v", h)
	}
	if got := b.appliedList(); len(got) != 0 {
		t.Fatalf("a module that runs what the want names was applied: %v", got)
	}
}

// A module whose kept resources differ from the composition is applied, and a failed apply is retried only by a change
// of either key.
func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
	b := newBed(t)
	b.failWith = "web.x: no space"
	stop := b.run()
	defer stop()
	fp := fpOf("a")
	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
	wk, wraw := wantFor(t, "web", fp, dig, "")
	b.comps.put(ck, craw)
	b.wants.put(wk, wraw)
	eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
	eventually(t, "the have does not say failed", func() bool {
		h := b.haves.last("web")
		return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "no space") && h.FP == fp
	})
	// An unrelated key changes: not a retry.
	ck9, craw9, _ := composition(t, "web", fpOf("9"), false, file("web", "x", "9"))
	b.comps.put(ck9, craw9)
	time.Sleep(150 * time.Millisecond)
	if n := len(b.appliedList()); n != 1 {
		t.Fatalf("a failed apply was retried by an unrelated key (%d applies)", n)
	}
	// The want is written again: the retry.
	b.mu.Lock()
	b.failWith = ""
	b.mu.Unlock()
	b.wants.put(wk, wraw)
	eventually(t, "the change of the want did not retry", func() bool { return len(b.appliedList()) == 2 })
	eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
}

// R6-A AC5: a data module that ran before asks its backup before any resource changes; a refusal holds it, nothing is
// applied and the have says held; when the want then says skip, the module is applied.
func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
	b := newBed(t)
	fp1, fp2 := fpOf("a"), fpOf("b")
	ck1, craw1, dig1 := composition(t, "db", fp1, true, file("db", "x", "1"))
	wk1, wraw1 := wantFor(t, "db", fp1, dig1, "")
	b.comps.before(ck1, craw1)
	b.wants.before(wk1, wraw1)
	b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}}
	refuse := true
	b.backup = func(string) Backup {
		if refuse {
			return Backup{Words: "restic: the repository is locked"}
		}
		return Backup{Done: true}
	}
	stop := b.run()
	defer stop()
	eventually(t, "baseline", func() bool { return b.haves.last("db").Composed == dig1 })

	ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
	b.comps.put(ck2, craw2)
	wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
	b.wants.put(wk2, wraw2)
	eventually(t, "the backup was not asked", func() bool {
		b.mu.Lock()
		defer b.mu.Unlock()
		return len(b.backups) == 1
	})
	eventually(t, "the have does not say held", func() bool {
		h := b.haves.last("db")
		return h.Phase == string(delivery.PhaseHeld) && strings.Contains(h.Detail, "repository is locked") && h.FP == fp1
	})
	if got := b.appliedList(); len(got) != 0 {
		t.Fatalf("something was applied behind a failed backup: %v", got)
	}

	// The answer arrives in the module's own want: back up again (it fails again), then skip.
	wk, again := wantFor(t, "db", fp2, dig2, delivery.BackupAgain)
	b.wants.put(wk, again)
	eventually(t, "back up again did not ask the backup", func() bool {
		b.mu.Lock()
		defer b.mu.Unlock()
		return len(b.backups) == 2
	})
	if got := b.appliedList(); len(got) != 0 {
		t.Fatalf("applied behind a second failed backup: %v", got)
	}
	_, skip := wantFor(t, "db", fp2, dig2, delivery.BackupSkip)
	b.wants.put(wk, skip)
	eventually(t, "skip did not apply the module", func() bool { return len(b.appliedList()) == 1 })
	b.mu.Lock()
	n := len(b.backups)
	b.mu.Unlock()
	if n != 2 {
		t.Fatalf("a backup was asked after the answer was to skip: %d asks", n)
	}
	eventually(t, "not running at the new fingerprint", func() bool {
		h := b.haves.last("db")
		return h.Phase == string(delivery.PhaseRunning) && h.FP == fp2 && h.Composed == dig2
	})
}

// A data module with a good backup is applied after it; one that never ran has nothing to back up.
func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) {
	b := newBed(t)
	stop := b.run()
	defer stop()
	fp := fpOf("c")
	ck, craw, dig := composition(t, "db", fp, true, file("db", "x", "1"))
	wk, wraw := wantFor(t, "db", fp, dig, "")
	b.comps.put(ck, craw)
	b.wants.put(wk, wraw)
	eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
	b.mu.Lock()
	n := len(b.backups)
	b.mu.Unlock()
	if n != 0 {
		t.Fatalf("a module that never ran was backed up (%d)", n)
	}
	fp2 := fpOf("d")
	ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
	wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
	b.comps.put(ck2, craw2)
	b.wants.put(wk2, wraw2)
	eventually(t, "second not applied", func() bool { return len(b.appliedList()) == 2 })
	b.mu.Lock()
	n = len(b.backups)
	b.mu.Unlock()
	if n != 1 {
		t.Fatalf("a module that ran before was backed up %d times, want once", n)
	}
}

// R6-A AC4 at the agent: a removed want removes that module only, and its have is written removed.
func TestARemovedWantRemovesThatModuleOnly(t *testing.T) {
	b := newBed(t)
	stop := b.run()
	defer stop()
	fp := fpOf("a")
	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
	b.comps.put(ck, craw)
	wk, wraw := wantFor(t, "web", fp, dig, "")
	b.wants.put(wk, wraw)
	eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
	gone, _ := delivery.EncodeWant(delivery.Want{Removed: true})
	b.wants.put(wk, gone)
	eventually(t, "not removed", func() bool {
		b.mu.Lock()
		defer b.mu.Unlock()
		return len(b.removed) == 1 && b.removed[0] == "web"
	})
	eventually(t, "the have was not written removed", func() bool {
		b.haves.mu.Lock()
		defer b.haves.mu.Unlock()
		return len(b.haves.removed) == 1
	})
}

// The node's own composition is applied before any module's, whichever key came first.
func TestTheNodesOwnCompositionComesBeforeAnyModules(t *testing.T) {
	b := newBed(t)
	fp := fpOf("a")
	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
	wk, wraw := wantFor(t, "web", fp, dig, "")
	nk, nraw, ndig := composition(t, NodeModule, "", false)
	nwk, err := delivery.WantNodeKey(node)
	if err != nil {
		t.Fatal(err)
	}
	nwraw, err := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
	if err != nil {
		t.Fatal(err)
	}
	b.comps.before(ck, craw)
	b.comps.before(nk, nraw)
	b.wants.before(wk, wraw)
	b.wants.before(nwk, nwraw)
	stop := b.run()
	defer stop()
	eventually(t, "both not applied", func() bool { return len(b.appliedList()) == 2 })
	if got := b.appliedList(); got[0] != NodeModule || got[1] != "web" {
		t.Fatalf("the order was %v, the node's own first", got)
	}
}

// R6-A AC9: a composition signed by either trusted key is applied; one signed by another key, one changed after it was
// signed and one unsigned are refused and nothing of them is applied (ADR 0539 §3).
func TestOnlyACompositionSignedByATrustedKeyIsApplied(t *testing.T) {
	b := newBed(t)
	stop := b.run()
	defer stop()

	plain := func(module, fp, content string) delivery.Composed {
		return delivery.Composed{Node: node, Module: module, FP: fp, Resources: []json.RawMessage{file(module, "x", content)}}
	}
	put := func(c delivery.Composed, signer ed25519.PrivateKey, tamper func([]byte) []byte) {
		key, _ := delivery.ComposedKey(node, c.Module, c.FP)
		var raw []byte
		var dig string
		if signer == nil {
			raw, _ = delivery.EncodeComposed(c)
			dig, _ = delivery.ComposedDigestOf(c)
		} else {
			_, raw, dig = signedBy(t, c, key, signer)
		}
		if tamper != nil {
			raw = tamper(raw)
		}
		b.comps.put(key, raw)
		wk, wraw := wantFor(t, c.Module, c.FP, dig, "")
		b.wants.put(wk, wraw)
	}

	// Refused: signed by a stranger, changed after signing, unsigned.
	put(plain("stranger", fpOf("1"), "x"), strangerKey, nil)
	put(plain("changed", fpOf("2"), "before"), controllerKey, func(raw []byte) []byte {
		return []byte(strings.Replace(string(raw), "before", "after!", 1))
	})
	put(plain("unsigned", fpOf("3"), "x"), nil, nil)
	time.Sleep(200 * time.Millisecond)
	if got := b.appliedList(); len(got) != 0 {
		t.Fatalf("a composition that is not signed by a trusted key was applied: %v", got)
	}

	// Applied: signed by the controller, signed by the operator.
	put(plain("bycontroller", fpOf("4"), "x"), controllerKey, nil)
	put(plain("byoperator", fpOf("5"), "x"), operatorKey, nil)
	eventually(t, "the trusted signers' compositions were not applied", func() bool { return len(b.appliedList()) == 2 })
	got := b.appliedList()
	if !(got[0] == "bycontroller" && got[1] == "byoperator") && !(got[0] == "byoperator" && got[1] == "bycontroller") {
		t.Fatalf("applied %v", got)
	}
	if h := b.haves.last("byoperator"); h.Phase != string(delivery.PhaseRunning) || h.FP != fpOf("5") {
		t.Fatalf("the operator's module has no running have: %+v", h)
	}
}

// With no key trusted, nothing is applied: a node that does not know its signers fails closed.
func TestWithNoTrustedKeyNothingIsApplied(t *testing.T) {
	b := newBed(t)
	b.env.Trusted = nil
	stop := b.run()
	defer stop()
	fp := fpOf("a")
	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
	b.comps.put(ck, craw)
	wk, wraw := wantFor(t, "web", fp, dig, "")
	b.wants.put(wk, wraw)
	time.Sleep(200 * time.Millisecond)
	if got := b.appliedList(); len(got) != 0 {
		t.Fatalf("applied with no trusted key: %v", got)
	}
}

// What a module's apply must not take is every other module that runs, those the agent holds and those still running from
// the kept declaration (ADR 0429 §4): the resources of the kept declaration reach Apply as the others.
func TestApplyingOneModuleHoldsOutTheModulesStillRunningFromTheKeptDeclaration(t *testing.T) {
	b := newBed(t)
	b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}, NodeModule: {file("node", "n", "n")}}
	stop := b.run()
	defer stop()
	fp := fpOf("a")
	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
	b.comps.put(ck, craw)
	wk, wraw := wantFor(t, "web", fp, dig, "")
	b.wants.put(wk, wraw)
	eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
	b.mu.Lock()
	defer b.mu.Unlock()
	var names []string
	for _, c := range b.others[0] {
		names = append(names, c.Module)
	}
	if len(names) != 2 || !strings.Contains(strings.Join(names, ","), "db") {
		t.Fatalf("the others of web were %v: db and the node's own run from the kept declaration", names)
	}
}

// Everything is what runs: the agent's compositions and the kept resources of each module it does not hold.
func TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest(t *testing.T) {
	b := newBed(t)
	b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}, "web": {file("web", "x", "old")}}
	a := New(b.env)
	if got := a.Everything(); len(got) != 2 {
		t.Fatalf("before the agent holds anything: %d compositions, want the 2 kept", len(got))
	}
	a.running["web"] = delivery.Have{FP: fpOf("a"), Composed: "sha256:" + strings.Repeat("1", 64), Phase: delivery.PhaseRunning}
	ck, craw, dig := composition(t, "web", fpOf("a"), false, file("web", "x", "new"))
	a.running["web"] = delivery.Have{FP: fpOf("a"), Composed: dig, Phase: delivery.PhaseRunning}
	a.takeComposition(Update{Key: ck, Value: craw, Revision: 1})
	got := a.Everything()
	if len(got) != 2 {
		t.Fatalf("%d compositions", len(got))
	}
	for _, c := range got {
		if c.Module == "web" && !strings.Contains(string(c.Resources[0]), "new") {
			t.Fatal("web is the kept declaration's, not the composition the agent runs")
		}
	}
	if a.HoldsNode() {
		t.Fatal("the agent holds no node composition")
	}
}

// A failed apply is applied again after a restart even when the kept declaration holds the composition: once this
// node-engine wrote its haves, they are what it knows (the kept declaration is only for the first start after the switch).
func TestAFailedApplyIsAppliedAgainAfterARestartWhateverIsKept(t *testing.T) {
	b := newBed(t)
	fp := fpOf("a")
	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
	wk, wraw := wantFor(t, "web", fp, dig, "")
	b.comps.before(ck, craw)
	b.wants.before(wk, wraw)
	b.kept = map[string][]json.RawMessage{"web": {file("web", "x", "1")}}
	b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed)}}
	stop := b.run()
	defer stop()
	eventually(t, "a failed apply was marked running from the kept declaration", func() bool { return len(b.appliedList()) == 1 })
}

// A have written before the switch carries no composition: then the kept declaration says what runs.
func TestAHaveWithNoCompositionDefersToTheKeptDeclaration(t *testing.T) {
	b := newBed(t)
	fp := fpOf("a")
	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
	wk, wraw := wantFor(t, "web", fp, dig, "")
	b.comps.before(ck, craw)
	b.wants.before(wk, wraw)
	b.kept = map[string][]json.RawMessage{"web": {file("web", "x", "1")}}
	b.haves.stored = map[string]Have{"web": {Module: "web", Phase: "running"}}
	stop := b.run()
	defer stop()
	eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig })
	time.Sleep(50 * time.Millisecond)
	if got := b.appliedList(); len(got) != 0 {
		t.Fatalf("applied %v", got)
	}
}

// A module that leaves the node's own composition for a composition of its own keeps its resources until that one is
// applied: the node's own apply holds out every module whose composition the node's bucket holds and the agent does not
// run yet (issue 1945, live on 2026-10-25 at 02:32: shanks' node-engine applied a node's own composition that no longer
// carried systemd-resolved, whose own composition waited for its credential, and removed its guard, so the machine lost
// its resolver).
func TestTheNodesOwnApplyHoldsOutAModuleThatMovedToItsOwnComposition(t *testing.T) {
	b := newBed(t)
	fp := fpOf("b")
	// systemd-resolved's own composition is written; no want names it yet.
	rk, rraw, _ := composition(t, "systemd-resolved", fp, false, file("systemd-resolved", "guard", "g"))
	b.comps.before(rk, rraw)
	nk, nraw, ndig := composition(t, NodeModule, "", false)
	nwk, err := delivery.WantNodeKey(node)
	if err != nil {
		t.Fatal(err)
	}
	nwraw, err := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
	if err != nil {
		t.Fatal(err)
	}
	b.comps.before(nk, nraw)
	b.wants.before(nwk, nwraw)
	stop := b.run()
	defer stop()
	eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 })
	b.mu.Lock()
	defer b.mu.Unlock()
	var names []string
	for _, c := range b.others[0] {
		names = append(names, c.Module)
	}
	if !strings.Contains(strings.Join(names, ","), "systemd-resolved") {
		t.Fatalf("the others of the node's own apply were %v: systemd-resolved's resources would be removed", names)
	}
}

// A module the assigner removed holds nothing back, its compositions still in the bucket or not: its resources are its
// removal's to take (issue 1945).
func TestARemovedModulesCompositionHoldsNothingBack(t *testing.T) {
	b := newBed(t)
	fp := fpOf("b")
	rk, rraw, _ := composition(t, "gone", fp, false, file("gone", "x", "1"))
	b.comps.before(rk, rraw)
	gk, err := delivery.WantKey(node, "gone")
	if err != nil {
		t.Fatal(err)
	}
	graw, err := delivery.EncodeWant(delivery.Want{Removed: true})
	if err != nil {
		t.Fatal(err)
	}
	b.wants.before(gk, graw)
	nk, nraw, ndig := composition(t, NodeModule, "", false)
	nwk, _ := delivery.WantNodeKey(node)
	nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
	b.comps.before(nk, nraw)
	b.wants.before(nwk, nwraw)
	stop := b.run()
	defer stop()
	eventually(t, "the node's own not applied", func() bool {
		for _, m := range b.appliedList() {
			if m == NodeModule {
				return true
			}
		}
		return false
	})
	b.mu.Lock()
	defer b.mu.Unlock()
	for _, others := range b.others {
		for _, c := range others {
			if c.Module == "gone" {
				t.Fatal("a removed module's composition was held out")
			}
		}
	}
}

// The node's own composition is not applied while a module's composition in the node's bucket cannot be read or is not
// signed by a trusted key: what that one carries may be what left the node's own, and the node's own apply would remove
// it (issue 1945, the review's gap 2). It is said, naming the composition, and applied once that one reads.
func TestTheNodesOwnWaitsWhileAModulesCompositionCannotBeRead(t *testing.T) {
	b := newBed(t)
	bad, err := delivery.ComposedKey(node, "broken", fpOf("c"))
	if err != nil {
		t.Fatal(err)
	}
	b.comps.before(bad, []byte(`{"v":1,"not":"a composition"}`))
	nk, nraw, ndig := composition(t, NodeModule, "", false)
	nwk, _ := delivery.WantNodeKey(node)
	nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
	b.comps.before(nk, nraw)
	b.wants.before(nwk, nwraw)
	stop := b.run()
	defer stop()
	eventually(t, "the wait not said", func() bool {
		b.mu.Lock()
		defer b.mu.Unlock()
		for _, s := range b.said {
			if strings.Contains(s, bad) {
				return true
			}
		}
		return false
	})
	if got := b.appliedList(); len(got) != 0 {
		t.Fatalf("applied %v while a composition could not be read", got)
	}
	// Read again, the node's own is applied.
	gk, graw, _ := composition(t, "broken", fpOf("c"), false, file("broken", "x", "1"))
	b.comps.put(gk, graw)
	eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 })
}

// A stale refused composition beside a newer readable one of the same module does not hold the node's own apply back:
// only the composition the module's want names, else its newest, counts (the review of issue 1945: the bucket keeps
// compositions written before the filled paths, which the SDK now refuses, and nobody applies them).
func TestAStaleRefusedCompositionDoesNotHoldTheNodesOwnBack(t *testing.T) {
	b := newBed(t)
	stale, err := delivery.ComposedKey(node, "power", fpOf("a"))
	if err != nil {
		t.Fatal(err)
	}
	b.comps.before(stale, []byte(`{"v":1,"not":"a composition"}`))
	gk, graw, _ := composition(t, "power", fpOf("b"), false, file("power", "x", "1"))
	b.comps.before(gk, graw)
	nk, nraw, ndig := composition(t, NodeModule, "", false)
	nwk, _ := delivery.WantNodeKey(node)
	nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
	b.comps.before(nk, nraw)
	b.wants.before(nwk, nwraw)
	stop := b.run()
	defer stop()
	eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 })
}

// A composition the kept declaration or the node's own have says runs is applied all the same when the engine holds no
// record of its resources: they were removed since (issue 1945, live on 2026-10-25: ace's and g14's node-engines removed
// ca-trust's, power's and matrix's resources in a node's own apply, and the wants that named those modules' own
// compositions were judged already running from the kept declaration, so nothing made them again).
func TestACompositionWhoseResourcesWereRemovedIsAppliedAgain(t *testing.T) {
	for _, from := range []string{"kept", "have"} {
		t.Run(from, func(t *testing.T) {
			b := newBed(t)
			fp := fpOf("a")
			rs := []json.RawMessage{file("ca-trust", "trust", "1")}
			ck, craw, dig := composition(t, "ca-trust", fp, false, rs...)
			wk, wraw := wantFor(t, "ca-trust", fp, dig, "")
			b.comps.before(ck, craw)
			b.wants.before(wk, wraw)
			if from == "kept" {
				b.kept = map[string][]json.RawMessage{"ca-trust": rs}
			} else {
				b.haves.stored = map[string]Have{"ca-trust": {Module: "ca-trust", FP: fp, Composed: dig, Phase: "running"}}
			}
			b.env.Recorded = func(context.Context, delivery.Composed) bool { return false }
			stop := b.run()
			defer stop()
			eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
		})
	}
}

// The composition a want names is applied from the key's history when a newer one has replaced it under the same key:
// a want held at a fingerprint keeps naming the digest it was written with, and the controller writes every new
// composition at that fingerprint over the same key (ADR 0539 clause 1, the sweep's kept revisions; issue 1945, live on
// 2026-10-25: shanks' node-tools composition had digest 575a5c96, its held want named a36e63c8, and nothing was applied).
func TestTheRevisionAWantNamesIsAppliedFromHistory(t *testing.T) {
	b := newBed(t)
	fp := fpOf("a")
	ck, oldRaw, oldDig := composition(t, "node-tools", fp, false, file("node-tools", "x", "old"))
	_, newRaw, _ := composition(t, "node-tools", fp, false, file("node-tools", "x", "new"))
	b.comps.before(ck, newRaw) // the key's latest value
	wk, wraw := wantFor(t, "node-tools", fp, oldDig, "")
	b.wants.before(wk, wraw)
	b.env.Revisions = func(_ context.Context, key string) ([]Update, error) {
		if key != ck {
			return nil, nil
		}
		return []Update{{Key: ck, Value: oldRaw, Revision: 3}, {Key: ck, Value: newRaw, Revision: 7}}, nil
	}
	stop := b.run()
	defer stop()
	eventually(t, "the revision the want names was not applied", func() bool { return len(b.appliedList()) == 1 })
	if h := b.haves.last("node-tools"); h.Composed != oldDig {
		t.Fatalf("the have names %s, want %s", h.Composed, oldDig)
	}
}

// 1945/AC18 (unit): a module that still runs from the kept declaration carries the kept declaration's filled paths of
// its files, so the filling writes them ("not named as filled" for claude-code and screen-lock, 2026-10-25).
func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) {
	b := newBed(t)
	b.kept = map[string][]json.RawMessage{"claude-code": {file("claude-code", "settings", "")}}
	b.env.KeptFilled = func() map[string][]string {
		return map[string][]string{"claude-code": {"/tmp/r6/claude-code/settings"}}
	}
	a := New(b.env)
	got := a.Everything()
	if len(got) != 1 || got[0].Module != "claude-code" || strings.Join(got[0].Filled, ",") != "/tmp/r6/claude-code/settings" {
		t.Fatalf("what runs from the kept declaration: %+v", got)
	}
}