mesh-host/internal/link/have_test.go

package link

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
	nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
	sdkkeys "git.novox.be/novox/mesh-sdk/go/keys"
	"github.com/novox/mesh-host/internal/compositions"
	"slices"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"github.com/nats-io/nats.go"
)

// What the node-engine writes beside its report: a module's have, and the node's own facts (novox/hq ADR 0429 §4,
// issue 1132).

// haveLink is a link with a bucket behind it that keeps each key's revision as the bus does.
type haveLink struct {
	*quietLink
	hmu                  sync.Mutex
	kv                   map[string]haveEntry
	sets                 []string // keys written, in order
	events               []string // "have <key>", in the order they happened
	failing              bool
	binds                int
	setTries             int
	bindFails, bindStuck bool
	// bindNoResponders fails that many binds with the bus's no responders, as while it reloads its accounts.
	bindNoResponders int
	// stated is what the tool runner's PING says of itself (Stater); statedErr is the runner not answering.
	stated    map[string]string
	statedErr error
	// readsSaid are the node seat's manifest-reads-changed events said on the link (ADR 0618 §4).
	readsSaid []nodeengine.ManifestReadsChangedEvent
}

func (l *haveLink) SayEngine(event string, body any) error {
	if e, ok := body.(nodeengine.ManifestReadsChangedEvent); ok && event == nodeengine.EventManifestReadsChanged {
		l.hmu.Lock()
		l.readsSaid = append(l.readsSaid, e)
		l.hmu.Unlock()
	}
	return nil
}

func (l *haveLink) Stated(context.Context, string, string) (map[string]string, error) {
	l.hmu.Lock()
	defer l.hmu.Unlock()
	return l.stated, l.statedErr
}

type haveEntry struct {
	value []byte
	rev   uint64
}

func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} }

func (l *haveLink) Health(context.Context, string, []byte) error { return nil }

func (l *haveLink) HaveBind(ctx context.Context) (HaveBucket, error) {
	l.hmu.Lock()
	l.binds++
	stuck, refuse := l.bindStuck, l.bindFails
	reloading := l.bindNoResponders > 0
	if reloading {
		l.bindNoResponders--
	}
	l.hmu.Unlock()
	if reloading {
		return nil, fmt.Errorf("binding node-engine_have: %w", nats.ErrNoResponders)
	}
	if stuck {
		<-ctx.Done()
		return nil, ctx.Err()
	}
	if refuse {
		return nil, errors.New("no grant for the have bucket")
	}
	return l, nil
}

func (l *haveLink) Get(_ context.Context, key string) ([]byte, uint64, error) {
	l.hmu.Lock()
	defer l.hmu.Unlock()
	e, ok := l.kv[key]
	if !ok {
		return nil, 0, ErrHaveAbsent
	}
	return e.value, e.rev, nil
}

func (l *haveLink) Set(_ context.Context, key string, value []byte, revision uint64) error {
	l.hmu.Lock()
	defer l.hmu.Unlock()
	l.setTries++
	if l.failing {
		return errors.New("the bucket refused")
	}
	e, ok := l.kv[key]
	if (revision == 0) == ok || (ok && e.rev != revision) {
		return errors.New("wrong last sequence")
	}
	l.kv[key] = haveEntry{value: value, rev: e.rev + 1}
	l.sets = append(l.sets, key)
	l.events = append(l.events, "have "+key)
	return nil
}

func (l *haveLink) have(t *testing.T, key string) nodeengine.HaveByNodeModuleValue {
	t.Helper()
	l.hmu.Lock()
	defer l.hmu.Unlock()
	h, err := nodeengineclient.DecodeHave(l.kv[key].value)
	if err != nil {
		t.Fatalf("%s holds no have: %v", key, err)
	}
	return h
}

func (l *haveLink) setCount() int {
	l.hmu.Lock()
	defer l.hmu.Unlock()
	return len(l.sets)
}

func (l *haveLink) put(t *testing.T, key string, h nodeengine.HaveByNodeModuleValue) {
	t.Helper()
	raw, err := nodeengineclient.EncodeHave(h)
	if err != nil {
		t.Fatal(err)
	}
	l.hmu.Lock()
	defer l.hmu.Unlock()
	l.kv[key] = haveEntry{value: raw, rev: l.kv[key].rev + 1}
}

func healthOf(states map[string]string) *Health {
	h := &Health{Contract: LivenessContract, At: time.Now().UTC()}
	for module, state := range states {
		h.Resources = append(h.Resources, ResourceHealth{Module: module, Resource: module + ".server", State: state})
	}
	return h
}

func appliedModules() Report {
	return Report{Applied: []string{"a.x", "b.x"},
		Modules: []ModuleFact{{Module: "a", Build: "b-1"}, {Module: "b", Build: "b-2", Failed: true, Detail: "b.x failed: no space"}},
		Health:  healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy})}
}

func aHaveQueue(l *haveLink) (*Session, *[]string) {
	var said []string
	var mu sync.Mutex
	q := &Session{Membership: Membership{Node: "n1"}, Timeout: time.Second, Say: func(s string) {
		mu.Lock()
		said = append(said, s)
		mu.Unlock()
	}}
	q.Attach(context.Background(), l)
	return q, &said
}

// 1132/AC3, AC5: a module that applied is running with its build and a module that failed is failed with the detail;
// the key does not exist yet, so it is created.
func TestAnAppliedModuleIsWrittenRunningAndAFailedOneFailed(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	q.SayFacts(context.Background(), appliedModules())
	a, b := l.have(t, "n1.a"), l.have(t, "n1.b")
	if a.Phase != nodeengineclient.PhaseRunning || a.Build != "b-1" || a.Health != nodeengineclient.HealthOK {
		t.Errorf("a: %+v", a)
	}
	if b.Phase != nodeengineclient.PhaseFailed || b.Build != "b-2" || !strings.Contains(b.Detail, "no space") ||
		b.Health != nodeengineclient.HealthFailed {
		t.Errorf("b: %+v", b)
	}
	if l.kv["n1.a"].rev != 1 {
		t.Errorf("a was created at revision %d", l.kv["n1.a"].rev)
	}
}

// 1132/AC5: a have equal to the bucket's value is not written; a different one is written over the last revision.
func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) {
	l := newHaveLink()
	l.put(t, "n1.a", nodeengine.HaveByNodeModuleValue{V: sdkkeys.Version, Build: "b-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
	l.put(t, "n1.b", nodeengine.HaveByNodeModuleValue{V: sdkkeys.Version, Build: "b-0", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
	q, _ := aHaveQueue(l)
	q.SayFacts(context.Background(), appliedModules())
	if l.kv["n1.a"].rev != 1 {
		t.Errorf("a equal to the bucket was written again, to revision %d", l.kv["n1.a"].rev)
	}
	if l.kv["n1.b"].rev != 2 || l.have(t, "n1.b").Phase != nodeengineclient.PhaseFailed {
		t.Errorf("b was not updated over its last revision: %+v at %d", l.have(t, "n1.b"), l.kv["n1.b"].rev)
	}
	before := l.setCount()
	q.SayFacts(context.Background(), appliedModules())
	if l.setCount() != before {
		t.Errorf("an apply that changed nothing wrote %d keys", l.setCount()-before)
	}
}

// 1132/AC4: a look's health reaches the module's have: ok, failed, waits or unknown, and only a change is written.
func TestAModulesHealthFollowsTheLook(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	rep := Report{Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"},
		{Module: "c", Build: "b"}, {Module: "d", Build: "b"}, {Module: "e", Build: "b"}}}
	q.SayFacts(context.Background(), rep)
	for _, m := range []string{"a", "b", "c", "d", "e"} {
		if h := l.have(t, "n1."+m).Health; h != nodeengineclient.HealthUnknown {
			t.Errorf("%s with no look is %s, want unknown", m, h)
		}
	}
	h := healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy, "c": StateStarting})
	h.Resources[slicesIndex(h.Resources, "b.server")].Reason = ReasonRestarting
	// 1496: a directory used as found is not the module's health (e runs healthy beside one); a failed or waiting module's
	// have says the words of the resource that made it so.
	h.Resources = append(h.Resources, ResourceHealth{Module: "a", Resource: "a.two", State: StateWaiting},
		ResourceHealth{Module: "b", Resource: "b.two", State: StateHealthy}, ResourceHealth{Module: "d", Resource: "d.x", State: StateUnknown},
		ResourceHealth{Module: "e", Resource: "e.server", State: StateHealthy},
		ResourceHealth{Module: "e", Resource: "e.cache", Kind: KindDirectory, State: StateUnhealthy, Reason: ReasonUsedAsFound + " owned by 1:1"})
	if !q.SayHealth(context.Background(), *h) {
		t.Fatal("not said")
	}
	for m, want := range map[string]string{"a": nodeengineclient.HealthWaits, "b": nodeengineclient.HealthFailed,
		"c": nodeengineclient.HealthWaits, "d": nodeengineclient.HealthUnknown, "e": nodeengineclient.HealthOK} {
		if got := l.have(t, "n1."+m).Health; got != want {
			t.Errorf("%s is %s, want %s", m, got, want)
		}
	}
	for m, want := range map[string]string{"a": "a.two waiting", "b": "b.server unhealthy: restarting", "d": "", "e": ""} {
		if got := l.have(t, "n1."+m).Detail; got != want {
			t.Errorf("%s says %q, want %q", m, got, want)
		}
	}
	before := l.setCount()
	h2 := *h
	q.SayHealth(context.Background(), h2)
	if l.setCount() != before {
		t.Errorf("the same look wrote %d keys", l.setCount()-before)
	}
	h2.Resources = []ResourceHealth{{Module: "a", Resource: "a.server", State: StateHealthy}}
	q.SayHealth(context.Background(), h2)
	if got := l.have(t, "n1.a"); got.Health != nodeengineclient.HealthOK || got.Detail != "" {
		t.Errorf("a after a healthy look is %s, saying %q", got.Health, got.Detail)
	}
}

// 1132/AC6: the node's key holds its windows, busy, strays, filters and profile.
func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	rep := appliedModules()
	rep.Windows = []Window{{Step: "backup", Holds: []string{"db"}}}
	rep.Busy = []Busy{{ID: "x", Container: "x", State: "waiting"}}
	rep.Strays = []Stray{{Kind: "container", Name: "old"}}
	rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}}
	rep.Profile = map[string]any{"cores": 8}
	q.SayFacts(context.Background(), rep)
	got, err := nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value)
	if err != nil {
		t.Fatal(err)
	}
	enc := func(v any) []map[string]any {
		var out []map[string]any
		b, _ := json.Marshal(v)
		_ = json.Unmarshal(b, &out)
		return out
	}
	want := nodeengine.HaveByNodeNodeValue{V: sdkkeys.Version, Windows: enc(rep.Windows), Busy: enc(rep.Busy), Strays: enc(rep.Strays),
		Filters: enc(rep.Filters), Profile: rep.Profile}
	if !nodeengineclient.SameHaveNode(got, want) {
		t.Errorf("the node key holds %+v, want %+v", got, want)
	}

	// 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and
	// networks, the outward links, the health's contract and the public keys; a report that changes none of it writes
	// nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing.
	keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert",
		Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2)
	q.Keys = func() RecordKeys { return keys }
	l.stated = map[string]string{CapabilitiesKey: "ledger-calls, bus-state-calls"} // the tool runner's calls (ENG-2)
	rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true},
		map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}},
		"machine": map[string]any{"networks": []any{map[string]any{"link": "eth0", "network": "192.0.2.0/24"}}}}
	rep.Outward = []string{"eth0"}
	rep.Health = &Health{Contract: NodeKeyContract}
	q.SayFacts(context.Background(), rep)
	if got, err = nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); err != nil || got.Record == nil {
		t.Fatalf("the node key holds no record: %+v, %v", got, err)
	}
	wantRecord := nodeengine.NodeRecord{V: sdkkeys.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk",
		Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "bus-state-calls", "ledger-calls", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
		Networks: []nodeengine.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract,
		ManifestReads: catalogue.ManifestReads()} // what it reads of a manifest: its contract's (ADR 0614 §4)
	if !nodeengineclient.SameNodeRecord(*got.Record, wantRecord) {
		t.Errorf("record = %+v, want %+v", *got.Record, wantRecord)
	}
	// Among them, that it fills a seat's reach from the seat holder's listen (ADR 0632 §1).
	if !slices.Contains(got.Record.ManifestReads, catalogue.ReadForm+catalogue.ReferenceFormSeatReach) {
		t.Errorf("the engine does not state it reads a seat's reach from its holder: %v", got.Record.ManifestReads)
	}
	at := l.kv["n1.node"].rev
	q.SayFacts(context.Background(), rep)
	if l.kv["n1.node"].rev != at {
		t.Errorf("a report changing nothing wrote the node key again")
	}
	// A runner that does not answer states what it stated before; one answering without any states none (an older runner).
	l.statedErr = fmt.Errorf("%w: nothing answered", ErrNoAnswer)
	l.stated = nil
	q.SayFacts(context.Background(), rep)
	if l.kv["n1.node"].rev != at {
		t.Errorf("a runner that did not answer changed the record")
	}
	l.statedErr, l.stated = nil, map[string]string{}
	q.SayFacts(context.Background(), rep)
	if got, _ = nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); l.kv["n1.node"].rev != at+1 || !slices.Equal(got.Record.Capabilities, []string{"apt", "systemd"}) {
		t.Errorf("a runner stating no calls: revision %d (was %d), capabilities %v", l.kv["n1.node"].rev, at, got.Record.Capabilities)
	}
	at++
	l.stated = map[string]string{CapabilitiesKey: "ledger-calls,bus-state-calls"}
	q.SayFacts(context.Background(), rep)
	at++
	keys.Certificate = "cert-2"
	q.SayFacts(context.Background(), rep)
	if got, _ = nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); l.kv["n1.node"].rev != at+1 || got.Record.Certificate != "cert-2" {
		t.Errorf("a new certificate: revision %d (was %d), record %+v", l.kv["n1.node"].rev, at, got.Record)
	}
	bare := rep
	bare.Health = nil
	q.SayFacts(context.Background(), bare)
	if got, _ = nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); got.Record.Contract != NodeKeyContract {
		t.Errorf("a report without a health statement dropped the contract: %+v", got.Record)
	}

	// Issue 1966 (ADR 0618 §4): the seats the engine states are those of the seat list it holds now. A change of the list
	// writes the record once and says manifest-reads-changed with the new statement; a restatement that changes nothing
	// writes and says nothing; the verb answers the statement and when it was made.
	all := catalogue.Seats()
	defer catalogue.UseSeats(all)
	q.RestateReads(context.Background()) // the statement as the list stands, whatever an earlier test left
	_, before := q.StatedReads()
	at, saidBefore := l.kv["n1.node"].rev, len(l.readsSaid)
	catalogue.UseSeats(slices.DeleteFunc(slices.Clone(all), func(s catalogue.Seat) bool { return s.Name == "mesh-vault" }))
	q.RestateReads(context.Background())
	got, _ = nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value)
	if l.kv["n1.node"].rev != at+1 || slices.Contains(got.Record.ManifestReads, "seat:mesh-vault") ||
		!slices.Contains(got.Record.ManifestReads, "seat:mesh-catalog") || !slices.Contains(got.Record.ManifestReads, "word:claims") {
		t.Errorf("a seat list without mesh-vault: revision %d (was %d), the record states %v", l.kv["n1.node"].rev, at, got.Record.ManifestReads)
	}
	if said := l.readsSaid[saidBefore:]; len(said) != 1 || said[0].Why != ReadsStarted || !slices.Equal(said[0].ManifestReads, got.Record.ManifestReads) {
		t.Errorf("a changed statement said %+v, want one manifest-reads-changed carrying the record's", said)
	}
	if reads, made := q.StatedReads(); !slices.Equal(reads, got.Record.ManifestReads) || made.Before(before) {
		t.Errorf("the verb answers %v made at %v (was %v), want the record's", reads, made, before)
	}
	q.RestateReads(context.Background())
	if l.kv["n1.node"].rev != at+1 || len(l.readsSaid) != saidBefore+1 {
		t.Errorf("an unchanged statement wrote the record or said it again")
	}
	catalogue.UseSeats(all)
	q.RestateReads(context.Background())
	if got, _ = nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); !slices.Equal(got.Record.ManifestReads, catalogue.ManifestReads()) {
		t.Errorf("the list back whole: the record states %v, want its contract's", got.Record.ManifestReads)
	}
}

// 1132/AC7: a write the bucket refuses is said in the log, and the next apply writes again.
func TestAFailedHaveWriteIsSaidAndWrittenAgain(t *testing.T) {
	l := newHaveLink()
	l.failing = true
	q, said := aHaveQueue(l)
	if !q.SayFacts(context.Background(), appliedModules()) {
		t.Fatal("the facts were not said with a link open")
	}
	if !strings.Contains(strings.Join(*said, "\n"), "n1.a") || !strings.Contains(strings.Join(*said, "\n"), "refused") {
		t.Errorf("the failure is not in the log: %v", *said)
	}
	l.hmu.Lock()
	l.failing = false
	l.hmu.Unlock()
	q.SayFacts(context.Background(), appliedModules())
	if l.have(t, "n1.a").Phase != nodeengineclient.PhaseRunning {
		t.Error("the next apply did not write again")
	}
}

// A link without a bucket, and a report with no module facts (a refusal), write nothing and say nothing wrong.
func TestNoBucketAndNoFactsWriteNothing(t *testing.T) {
	plain := newQuietLink()
	q := &Session{Membership: Membership{Node: "n1"}, Timeout: time.Second}
	q.Attach(context.Background(), plain)
	if !q.SayFacts(context.Background(), appliedModules()) {
		t.Fatal("the report did not go out")
	}
	l := newHaveLink()
	q2, _ := aHaveQueue(l)
	q2.SayFacts(context.Background(), Report{Refused: "no"})
	if l.setCount() != 0 {
		t.Errorf("a refusal wrote %d keys", l.setCount())
	}
}

// 1132/AC7: a bind that fails, or never answers, writes nothing and returns within the queue's timeout; a write that
// fails ends the pass, and is said once.
func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) {
	for name, set := range map[string]func(*haveLink){
		"a bind refused":            func(l *haveLink) { l.bindFails = true },
		"a bind that never answers": func(l *haveLink) { l.bindStuck = true },
	} {
		l := newHaveLink()
		set(l)
		q, said := aHaveQueue(l)
		q.Timeout = 100 * time.Millisecond
		start := time.Now()
		if !q.SayFacts(context.Background(), appliedModules()) {
			t.Fatalf("%s: the report did not go out", name)
		}
		if took := time.Since(start); took > time.Second {
			t.Errorf("%s: the pass took %s", name, took)
		}
		if l.binds != 1 || l.setTries != 0 || len(*said) != 1 {
			t.Errorf("%s: binds %d, writes tried %d, said %v; want one bind, no write, one line", name, l.binds, l.setTries, *said)
		}
	}
	l := newHaveLink()
	l.failing = true
	q, said := aHaveQueue(l)
	q.SayFacts(context.Background(), appliedModules())
	if l.binds != 1 || l.setTries != 1 || len(*said) != 1 {
		t.Errorf("a failed write: binds %d, writes tried %d, said %v; want one of each and one line", l.binds, l.setTries, *said)
	}
	// A bind that meets no responders while the bus reloads its accounts (2026-10-25 23:04:24, four in one second on the
	// bus node) is written again after a backoff, with no apply or look to come.
	defer func(was func(int) time.Duration) { haveBackoff = was }(haveBackoff)
	haveBackoff = func(int) time.Duration { return time.Millisecond }
	l = newHaveLink()
	l.bindNoResponders = 1
	q, _ = aHaveQueue(l)
	q.SayFacts(context.Background(), appliedModules())
	deadline := time.Now().Add(3 * time.Second)
	for l.setCount() == 0 && time.Now().Before(deadline) {
		time.Sleep(5 * time.Millisecond)
	}
	if l.setCount() == 0 {
		t.Fatal("a bind that met no responders was not written again")
	}
	if l.have(t, "n1.a").Phase != nodeengineclient.PhaseRunning {
		t.Error("the have written again does not say running")
	}
}

func onlyA() Report {
	return Report{Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}},
		Health: healthOf(map[string]string{"a": StateHealthy})}
}

// 1369/AC1: after a declaration with a and b, one with a only writes n1.b with the phase removed, over the revision
// read, and leaves a as it was; the key is not deleted.
func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	q.SayFacts(context.Background(), appliedModules())
	revA, revB := l.kv["n1.a"].rev, l.kv["n1.b"].rev
	q.SayFacts(context.Background(), onlyA())
	b := l.have(t, "n1.b")
	if b.Phase != nodeengineclient.PhaseRemoved || b.Build != "b-2" {
		t.Errorf("b: %+v", b)
	}
	if l.kv["n1.b"].rev != revB+1 {
		t.Errorf("b was written at revision %d, want one over %d", l.kv["n1.b"].rev, revB)
	}
	if l.kv["n1.a"].rev != revA || l.have(t, "n1.a").Phase != nodeengineclient.PhaseRunning {
		t.Errorf("a changed: %+v at %d", l.have(t, "n1.a"), l.kv["n1.a"].rev)
	}
	before := l.setCount()
	q.SayFacts(context.Background(), onlyA())
	if l.setCount() != before {
		t.Errorf("a second apply without b wrote %d keys, removed is written once", l.setCount()-before)
	}
}

// 1369/AC2: a module removed and later added back is written with its new phase.
func TestARemovedModuleAddedBackIsWrittenWithItsNewPhase(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	q.SayFacts(context.Background(), appliedModules())
	q.SayFacts(context.Background(), onlyA())
	q.SayFacts(context.Background(), appliedModules())
	if b := l.have(t, "n1.b"); b.Phase != nodeengineclient.PhaseFailed || b.Build != "b-2" {
		t.Errorf("b after it came back: %+v", b)
	}
	q.SayFacts(context.Background(), onlyA())
	if b := l.have(t, "n1.b"); b.Phase != nodeengineclient.PhaseRemoved {
		t.Errorf("b removed again: %+v", b)
	}
}

// 1369/AC3: a module this node-engine never wrote a have key for is not written, and a key that is gone from the
// bucket is not created again.
func TestAModuleNeverWrittenIsNotWrittenRemoved(t *testing.T) {
	l := newHaveLink()
	l.put(t, "n1.c", nodeengine.HaveByNodeModuleValue{V: sdkkeys.Version, Build: "b-9", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
	q, _ := aHaveQueue(l)
	q.SayFacts(context.Background(), appliedModules())
	q.SayFacts(context.Background(), onlyA())
	if c := l.have(t, "n1.c"); c.Phase != nodeengineclient.PhaseRunning || l.kv["n1.c"].rev != 1 {
		t.Errorf("c, which this node-engine never wrote, changed: %+v at %d", c, l.kv["n1.c"].rev)
	}
	// a key that vanished from the bucket is not created by the removal
	l2 := newHaveLink()
	q2, _ := aHaveQueue(l2)
	q2.SayFacts(context.Background(), appliedModules())
	l2.hmu.Lock()
	delete(l2.kv, "n1.b")
	l2.hmu.Unlock()
	q2.SayFacts(context.Background(), onlyA())
	if _, there := l2.kv["n1.b"]; there {
		t.Error("the removal created a key that was not there")
	}
}

// A removal the bucket refused is written by the next pass.
func TestARefusedRemovalIsWrittenByTheNextPass(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	q.SayFacts(context.Background(), appliedModules())
	l.hmu.Lock()
	l.failing = true
	l.hmu.Unlock()
	q.SayFacts(context.Background(), onlyA())
	l.hmu.Lock()
	l.failing = false
	l.hmu.Unlock()
	q.SayFacts(context.Background(), onlyA())
	if b := l.have(t, "n1.b"); b.Phase != nodeengineclient.PhaseRemoved {
		t.Errorf("b: %+v", b)
	}
}

// 1369/F1: a node-engine that started since wrote nothing, and the report names the modules the kept declaration held
// before: one of them the new declaration lacks is written removed, one that is not in the bucket is not created.
func TestARestartedNodeEngineWritesTheRemovalFromTheReportsBefore(t *testing.T) {
	l := newHaveLink()
	l.put(t, "n1.a", nodeengine.HaveByNodeModuleValue{V: sdkkeys.Version, Build: "b-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
	l.put(t, "n1.b", nodeengine.HaveByNodeModuleValue{V: sdkkeys.Version, Build: "b-2", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
	q, _ := aHaveQueue(l) // fresh: nothing written by this process
	r := onlyA()
	r.Before = []string{"a", "b", "c"}
	q.SayFacts(context.Background(), r)
	if b := l.have(t, "n1.b"); b.Phase != nodeengineclient.PhaseRemoved {
		t.Errorf("b: %+v", b)
	}
	if _, there := l.kv["n1.c"]; there {
		t.Error("c, which has no key, was created")
	}
	if l.kv["n1.a"].rev != 1 {
		t.Errorf("a was written, revision %d", l.kv["n1.a"].rev)
	}
}

// R6-A AC7: the first have this node-engine writes — or finds already saying what it would write — is told to the
// proof, which proves a node-engine on trial to its launcher (novox/hq ADR 0539 §9); a write the bucket refuses is not.
func TestTheFirstHaveWrittenIsToldToTheProof(t *testing.T) {
	l := newHaveLink()
	l.failing = true
	q, _ := aHaveQueue(l)
	var told int
	q.HaveWritten = func() { told++ }
	q.SayFacts(context.Background(), appliedModules())
	if told != 0 {
		t.Fatalf("a refused write proved the build %d time(s)", told)
	}
	l.hmu.Lock()
	l.failing = false
	l.hmu.Unlock()
	q.SayFacts(context.Background(), appliedModules())
	if told == 0 {
		t.Fatal("a have written under this build was not told")
	}

	// The bucket already holds what this node-engine would write: a restart with nothing changed still proves it.
	same := newHaveLink()
	q2, _ := aHaveQueue(same)
	q2.SayFacts(context.Background(), appliedModules())
	again, _ := aHaveQueue(same)
	told = 0
	again.HaveWritten = func() { told++ }
	again.SayFacts(context.Background(), appliedModules())
	if told == 0 {
		t.Fatal("a have already saying the same was not told")
	}
}

// R6-A AC6: an applied module's have names its fingerprint and the digest of the composition applied; a health check that
// fails after the module's start window gives health failed, and the have keeps both.
func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	fp := "src2:" + strings.Repeat("a", 64)
	dig := "sha256:" + strings.Repeat("b", 64)
	q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig})
	h := l.have(t, "n1.web")
	if h.Fp != fp || h.Composed != dig || h.Phase != nodeengineclient.PhaseRunning || h.Health != nodeengineclient.HealthUnknown {
		t.Fatalf("the have after the apply: %+v", h)
	}
	// Starting: it waits. Past its start window the check fails: failed.
	q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateStarting}))
	if got := l.have(t, "n1.web"); got.Health != nodeengineclient.HealthWaits || got.Fp != fp || got.Composed != dig {
		t.Fatalf("while it starts: %+v", got)
	}
	q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateUnhealthy}))
	if got := l.have(t, "n1.web"); got.Health != nodeengineclient.HealthFailed || got.Fp != fp || got.Composed != dig {
		t.Fatalf("a failing check after the start window: %+v", got)
	}

	// The bus's user list written alone rides on the module's have, and an apply after it keeps it (ADR 0547 §8).
	w := nodeengine.UserListWritten{Source: "n1.web." + fp, Revision: 4, Wanted: dig, Digest: dig, Users: 3,
		At: time.Date(2026, 10, 25, 21, 0, 0, 0, time.UTC)}
	q.NoteUserList(context.Background(), "web", w)
	if got := l.have(t, "n1.web"); got.UserList == nil || got.UserList.Revision != 4 || got.Fp != fp {
		t.Fatalf("the user list is not in the have: %+v", got)
	}
	q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig})
	if got := l.have(t, "n1.web"); got.UserList == nil || got.UserList.Users != 3 {
		t.Fatalf("an apply lost the user list: %+v", got)
	}

	// A failed apply waiting on a store's holder carries the pull it waits on in its have (ENG-9, read by node_haves).
	wait := &nodeengine.PullWait{Image: "reg/x@sha256:1", Seat: "container-image-registry", Holder: "distribution",
		Began: time.Date(2026, 11, 4, 14, 0, 0, 0, time.UTC), Retried: "distribution settled"}
	q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Failed: true, Detail: "the pull failed", PullWait: wait})
	if got := l.have(t, "n1.web"); got.Phase != nodeengineclient.PhaseFailed || got.PullWait == nil || got.PullWait.Holder != "distribution" {
		t.Fatalf("the failed have does not say the pull it waits on: %+v", got)
	}

	// A module held for a failed backup says so and what it keeps running; removed is written once, over its key.
	q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseHeld), Detail: "the repository is locked"})
	if got := l.have(t, "n1.web"); got.Phase != nodeengineclient.PhaseHeld || got.Detail == "" {
		t.Fatalf("held: %+v", got)
	}
	q.NoteRemoved(context.Background(), "web", "")
	if got := l.have(t, "n1.web"); got.Phase != nodeengineclient.PhaseRemoved {
		t.Fatalf("removed: %+v", got)
	}
	q.NoteNode(context.Background(), dig, compositions.Seals{"core-box#sealed": "core"})
	if n, err := nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig || len(n.SealedAt) != 1 || n.SealedAt["core-box#sealed"] != "core" {
		t.Fatalf("the node key, with its own composition's sealed values by slot: %+v %v", n, err)
	}
	if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.SealedAt) != 1 {
		t.Fatalf("the node key read back: %+v %v", n, ok)
	}
	// The node's answers to assignment candidates ride in the same key (ADR 0559 §4), and go when no group is answered.
	answers := map[string]nodeengine.CandidateAnswer{"p-1": {Proposal: "p-1", Revision: 1, Verdict: nodeengine.CandidateAnswerVerdictSame, Same: []string{"web"}}}
	q.NoteCandidates(context.Background(), answers)
	if n, err := nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Candidates["p-1"].Proposal != "p-1" || n.Composed != dig {
		t.Fatalf("the node key with its answers: %+v %v", n, err)
	}
	q.NoteCandidates(context.Background(), nil)
	if n, _ := nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); n.Candidates != nil {
		t.Fatalf("an answer left after its group went: %+v", n.Candidates)
	}
}

// A module applied and running that has nothing long-running to judge is ok once the engine has looked: there is nothing
// of it that could be down. Unknown kept it from ever being healthy, so the assigner's canary rule held every other node
// of mesh-host back for good (issue 1945, live on 2026-10-25: ace's have of mesh-host said unknown, and novox and shanks
// never got the new node-engine). A failed one stays failed, and before any look it is unknown.
func TestAModuleWithNothingToJudgeIsOkOnceLookedAt(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	q.SayFacts(context.Background(), Report{Modules: []ModuleFact{{Module: "mesh-host", Build: "b"}, {Module: "web", Build: "b"},
		{Module: "broken", Build: "b", Failed: true, Detail: "x"}}})
	if h := l.have(t, "n1.mesh-host").Health; h != nodeengineclient.HealthUnknown {
		t.Fatalf("before any look: %s", h)
	}
	if !q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateHealthy})) {
		t.Fatal("not said")
	}
	if h := l.have(t, "n1.mesh-host").Health; h != nodeengineclient.HealthOK {
		t.Fatalf("a running module with nothing to judge after a look: %s", h)
	}
	if h := l.have(t, "n1.broken").Health; h == nodeengineclient.HealthOK {
		t.Fatalf("a failed module was said ok")
	}
}

// A have an older engine left unknown is written over with this engine's health once the module is said again, as the
// node agent says a module it finds running on its own have: a packages-only module (xorg, nothing long-running) is ok
// after a look, so the assigner no longer holds back what requires its provision (2026-10-25, g14 and shanks).
func TestAModuleSaidAgainOverAnOlderEnginesUnknownIsWrittenWithThisEnginesHealth(t *testing.T) {
	l := newHaveLink()
	fp := "src2:" + strings.Repeat("a", 64)
	dig := "sha256:" + strings.Repeat("b", 64)
	l.put(t, "n1.xorg", nodeengine.HaveByNodeModuleValue{V: sdkkeys.Version, Fp: fp, Composed: dig, Phase: nodeengineclient.PhaseRunning,
		Health: nodeengineclient.HealthUnknown})
	q, _ := aHaveQueue(l)
	if !q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateHealthy})) {
		t.Fatal("not said")
	}
	if h := l.have(t, "n1.xorg"); h.Health != nodeengineclient.HealthUnknown {
		t.Fatalf("a module the session does not hold was written: %+v", h)
	}
	q.NoteModule(context.Background(), ModuleFact{Module: "xorg", FP: fp, Composed: dig})
	if h := l.have(t, "n1.xorg"); h.Health != nodeengineclient.HealthOK || h.Fp != fp || h.Composed != dig || h.Phase != nodeengineclient.PhaseRunning {
		t.Fatalf("said again after a look: %+v", h)
	}
}

// A module's health is not yet judged (unknown) while its change is under way and until a look after it, never failed
// from a look at what was being replaced: on 2026-10-25 15:27:36 the controller's canary on novox was held on a have
// saying applying and failed, two seconds before its launch said healthy.
func TestAHealthIsNotJudgedWhileAModuleIsApplied(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	ctx := context.Background()
	old := ModuleFact{Module: "ctl", FP: "src2:" + strings.Repeat("a", 64), Composed: "sha256:" + strings.Repeat("1", 64)}
	q.NoteModule(ctx, old)
	q.SayHealth(ctx, *healthOf(map[string]string{"ctl": StateUnhealthy}))
	if h := l.have(t, "n1.ctl"); h.Health != nodeengineclient.HealthFailed {
		t.Fatalf("the running module's failed look was not said: %+v", h)
	}
	next := ModuleFact{Module: "ctl", FP: "src2:" + strings.Repeat("b", 64), Composed: "sha256:" + strings.Repeat("2", 64)}
	applying := next
	applying.Phase = string(nodeengineclient.PhaseApplying)
	q.NoteModule(ctx, applying)
	if h := l.have(t, "n1.ctl"); h.Phase != nodeengineclient.PhaseApplying || h.Health != nodeengineclient.HealthUnknown {
		t.Fatalf("applying carried the last look's health: %+v", h)
	}
	q.SayHealth(ctx, *healthOf(map[string]string{"ctl": StateUnhealthy}))
	if h := l.have(t, "n1.ctl"); h.Health != nodeengineclient.HealthUnknown {
		t.Fatalf("a look during the apply judged it: %+v", h)
	}
	q.NoteModule(ctx, next)
	if h := l.have(t, "n1.ctl"); h.Phase != nodeengineclient.PhaseRunning || h.Health != nodeengineclient.HealthUnknown {
		t.Fatalf("running before a look after the apply: %+v", h)
	}
	q.SayHealth(ctx, *healthOf(map[string]string{"ctl": StateHealthy}))
	if h := l.have(t, "n1.ctl"); h.Health != nodeengineclient.HealthOK {
		t.Fatalf("the look after the apply was not taken: %+v", h)
	}
}

// A fact naming another composition than the one looked at, with no apply phase between, is not judged by the old look.
func TestAHealthIsNotCarriedOverToAnotherComposition(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	ctx := context.Background()
	q.NoteModule(ctx, ModuleFact{Module: "ctl", FP: "src2:" + strings.Repeat("a", 64)})
	q.SayHealth(ctx, *healthOf(map[string]string{"ctl": StateUnhealthy}))
	q.NoteModule(ctx, ModuleFact{Module: "ctl", FP: "src2:" + strings.Repeat("b", 64)})
	if h := l.have(t, "n1.ctl"); h.Health != nodeengineclient.HealthUnknown {
		t.Fatalf("the old composition's look judged the new one: %+v", h)
	}
}

// A declaration's report naming another composition of a module, with no health of its own, does not carry the old
// look's health over; and a module removed leaves nothing marked unjudged behind.
func TestADeclarationReportMovesAHealthAndARemovalForgetsIt(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	ctx := context.Background()
	q.SayFacts(ctx, Report{Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}},
		Health: healthOf(map[string]string{"a": StateUnhealthy})})
	if h := l.have(t, "n1.a"); h.Health != nodeengineclient.HealthFailed {
		t.Fatalf("the failed look was not said: %+v", h)
	}
	q.SayFacts(ctx, Report{Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1", FP: "src2:" + strings.Repeat("c", 64)}}})
	if h := l.have(t, "n1.a"); h.Health != nodeengineclient.HealthUnknown {
		t.Fatalf("the old composition's look judged the new one: %+v", h)
	}
	select {
	case <-q.HealthOwed():
	default:
		t.Fatal("a module out of motion and unjudged owed no look")
	}
	q.NoteRemoved(ctx, "a", "")
	q.haves.mu.Lock()
	left := q.haves.unjudged["a"]
	q.haves.mu.Unlock()
	if left {
		t.Fatal("a removed module is still marked unjudged")
	}
}

// ADR 0547 §8, 2026-10-26 on the bus node: the bus's user list of a module this engine says no have of (no want, and no
// longer in the declaration) still reaches the module's key, merged into what the key says, every other field kept; a
// key that does not exist is not made up and is said once; a writer that drops the list again is written over.
func TestAUserListReachesTheHaveOfAModuleNoApplyHereSays(t *testing.T) {
	l := newHaveLink()
	older := nodeengine.HaveByNodeModuleValue{V: sdkkeys.Version, Build: "build-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
	l.put(t, "n1.nats", older)
	q, said := aHaveQueue(l)
	dig := "sha256:" + strings.Repeat("c", 64)
	w := nodeengine.UserListWritten{Source: "n1.nats", Revision: 7, Wanted: dig, Digest: dig, Users: 58,
		At: time.Date(2026, 10, 26, 12, 26, 51, 0, time.UTC)}
	q.NoteUserList(context.Background(), "nats", w)
	got := l.have(t, "n1.nats")
	if got.UserList == nil || got.UserList.Users != 58 || got.UserList.Digest != dig {
		t.Fatalf("the user list is not in the have: %+v", got)
	}
	if got.Build != "build-1" || got.Phase != nodeengineclient.PhaseRunning || got.Health != nodeengineclient.HealthOK {
		t.Fatalf("the merge lost the key's own fields: %+v", got)
	}
	rev := l.kv["n1.nats"].rev
	q.NoteUserList(context.Background(), "nats", w)
	if l.kv["n1.nats"].rev != rev {
		t.Errorf("the same list was written again, to revision %d", l.kv["n1.nats"].rev)
	}

	// Another writer replaces the value without the list: the next pass merges it back.
	l.put(t, "n1.nats", older)
	q.SayFacts(context.Background(), Report{})
	if got := l.have(t, "n1.nats"); got.UserList == nil || got.Build != "build-1" {
		t.Fatalf("a value written without the list kept it out: %+v", got)
	}

	// No key at all: nothing is created, said once; the module's first have carries the list.
	q.NoteUserList(context.Background(), "other", w)
	q.NoteUserList(context.Background(), "other", w)
	if _, ok := l.kv["n1.other"]; ok {
		t.Fatalf("a have was made up for a module no apply says")
	}
	n := 0
	for _, s := range *said {
		if strings.Contains(s, "is in no have") {
			n++
		}
	}
	if n != 1 {
		t.Errorf("the missing key was said %d times: %q", n, *said)
	}
	fp := "src2:" + strings.Repeat("a", 64)
	q.NoteModule(context.Background(), ModuleFact{Module: "other", FP: fp, Composed: dig})
	if got := l.have(t, "n1.other"); got.UserList == nil || got.Fp != fp {
		t.Fatalf("the first have of the module lost the list: %+v", got)
	}
}

// slicesIndex is the index of the resource with id in rs, or -1.
func slicesIndex(rs []ResourceHealth, id string) int {
	for i, r := range rs {
		if r.Resource == id {
			return i
		}
	}
	return -1
}