mesh-delivery/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"log"
	"maps"
	"slices"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"

	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
	mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
	"git.novox.be/novox/mesh-sdk/go/process"
	"git.novox.be/novox/mesh-sdk/go/views"

	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"

	"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
	"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
	"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
	"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
	"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
	sweep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshartifactsweep"

	"mesh-delivery-builder/internal/build"
	"mesh-delivery-builder/internal/versions"
)

// 1945 (a part exits at start for a bucket it works without): a refused bucket it cannot work without is an error naming the bucket and
// its grant. The gate over the seat's assignments, and its refusal, are mesh-sdk's (buildrun.PlacedNodes,
// issue 774).
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
	t.Run("a floor's wake is heard on the events consumer (design 51 The floor)", aFloorsWakeIsHeard)
	t.Run("a failed trunk build is a finding until a later done one (DLV-B7)", aFailedTrunkBuildIsAFinding)
	refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_modules")}
	if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
		!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
		t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
	}
	t.Run("ADR 0624: the counted nodes' engine reads", theCountedNodesEngineReads)
}

// theCountedNodesEngineReads: the builder asks the node-engine seat's verb of every node the assignments names; a node away
// and never asked has no statement and reads as the baseline's first forms, with no node record read in its place; an
// away node keeps its last statement; a node the assignments no longer names leaves the count, its key removed; and
// builder_engine_reads drops one node's key.
func theCountedNodesEngineReads(t *testing.T) {
	gate := buildrun.NewPlacedNodes(t.Logf)
	place := func(nodes ...string) {
		for _, node := range []string{"n1", "n2"} {
			pl := meshmoduleassigner.Assignments{Node: node}
			if slices.Contains(nodes, node) {
				pl.Modules = []meshmoduleassigner.PlacedModule{{Module: "web"}}
			}
			values, err := meshmoduleassignerclient.AssignmentsValues(pl)
			if err != nil {
				t.Fatal(err)
			}
			for key, raw := range values {
				gate.Assignments(key, raw, false, time.Now())
			}
			if !slices.Contains(nodes, node) {
				key, _ := meshmoduleassignerclient.ModuleKey(node, "web")
				gate.Assignments(key, nil, true, time.Now())
			}
		}
	}
	away := map[string]bool{"n2": true}
	asked := map[string]int{}
	r := newEngineReads(func(node, key string, _ any) (json.RawMessage, error) {
		asked[node]++
		if away[node] {
			return nil, nats.ErrNoResponders
		}
		return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)})
	}, gate, t.Logf)
	r.keeper.Sync = true
	kept := map[string][]byte{}
	r.keeper.Put = func(node string, v []byte) error { kept[node] = v; return nil }
	r.keeper.Delete = func(node string) error { delete(kept, node); return nil }
	place("n1", "n2")
	r.placed()
	present, counted := r.forms()
	if !slices.Equal(present["n1"], []string{"form:new"}) || len(present) != 1 || len(counted) != 2 || counted["n2"] != nil {
		t.Errorf("away and never asked: present %v, counted %v", present, counted)
	}
	if s := r.keeper.Statements()["n2"]; s.From != enginereads.FromNone || s.Answered || asked["n2"] != 1 {
		t.Errorf("n2, away and never asked, is %+v (asked %d)", s, asked["n2"])
	}
	// n2 back, stating {word:module}, then away again: it keeps that statement.
	away["n2"] = false
	r.keeper.Heard("n2", mustStatement(t, enginereads.Statement{Node: "n2", From: enginereads.FromVerb, Answered: true,
		AskAgain: &enginereads.Again{By: "n1.ada", Why: "back"}}), false)
	away["n2"] = true
	r.keeper.AskNode("n2", "its event")
	if _, counted = r.forms(); !slices.Equal(counted["n2"], []string{"form:new"}) || r.keeper.Statements()["n2"].Answered {
		t.Errorf("away again, n2 reads %v (%+v)", counted["n2"], r.keeper.Statements()["n2"])
	}
	// n2 retired: it leaves the assignments, and its key goes.
	place("n1")
	r.placed()
	if _, counted = r.forms(); len(counted) != 1 || kept["n2"] != nil {
		t.Errorf("retired n2: counted %v, its key %s", counted, kept["n2"])
	}
	// builder_engine_reads drops one key.
	st := &memReads{v: map[string][]byte{"n1": kept["n1"]}}
	tool := enginereads.Tool("builder_engine_reads", "the builder", st, true, time.Now)
	if _, err := tool.Run(map[string]any{"node": "n1", "drop": "true", "why": "a test"}); err != nil || st.v["n1"] != nil {
		t.Errorf("drop: %v, the key %s", err, st.v["n1"])
	}
	t.Run("ADR 0668 §3: a manifest-reads call unanswered past five minutes, its machine heard, is call-unanswered", func(t *testing.T) {
		// n2 answers nothing: a bound is kept at five minutes and the watchdogs seat's presence is asked only when it ends
		// (ADR 0596 §2), once; away, no finding; heard, the finding call-unanswered in the builder's findings at the
		// builder's clock; answered, it is cleared.
		place("n1", "n2")
		now := time.Unix(1000, 0)
		clock := func() time.Time { return now }
		down := true
		r2 := newEngineReads(func(node, _ string, _ any) (json.RawMessage, error) {
			if down {
				return nil, nats.ErrNoResponders
			}
			return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: now.Format(time.RFC3339Nano)})
		}, gate, t.Logf)
		r2.keeper.Sync, r2.keeper.Now = true, clock
		st := &memFindings{vals: map[string][]byte{}}
		armed := map[string]time.Time{}
		online, asked := false, []any{}
		heard := heardBy(func(key string, body any) (json.RawMessage, error) {
			asked = append(asked, key, body)
			return json.Marshal(meshwatchdogs.PresenceAnswer{Machine: "n2", Online: online})
		})
		calls := r2.callsOf(&findings{module: builds.BuilderModule, w: st, logf: t.Logf, now: clock}, func() jetstream.KeyValue { return nil }, heard, t.Logf)
		calls.Arm = func(_ context.Context, node string, ends time.Time) error { armed[node] = ends; return nil }
		key := meshconditionsclient.Key(meshconditionsclient.ScopeCall, "manifest-reads.n2", meshconditionsclient.KindCallUnanswered)
		r2.placed() // n2 asked, unanswered at 1000
		if len(st.vals) != 0 || len(asked) != 0 || !armed["n2"].Equal(time.Unix(1000, 0).Add(meshconditionsclient.CallUnansweredAfter)) {
			t.Fatalf("before five minutes: findings %v, presence asked %v, bounds %v", st.vals, asked, armed)
		}
		now = now.Add(meshconditionsclient.CallUnansweredAfter)
		r2.expiredCall("n2")
		if len(st.vals) != 0 || len(asked) != 2 || asked[0] != presenceInvoke || asked[1] != (meshwatchdogs.PresenceInput{Machine: "n2"}) {
			t.Fatalf("its machine away: findings %v, presence asked %v", st.vals, asked)
		}
		online = true
		r2.expiredCall("n2")
		f, err := decodeFinding(st.vals[key])
		if err != nil || f.Observer != builds.BuilderModule || f.Kind != meshconditionsclient.KindCallUnanswered || len(asked) != 4 {
			t.Fatalf("heard five minutes on: finding %+v, %v (keys %v), presence asked %v", f, err, slices.Collect(maps.Keys(st.vals)), asked)
		}
		r2.stepCalls()
		if len(st.vals) != 1 || len(asked) != 4 {
			t.Errorf("a step after it: findings %v, presence asked %v", st.vals, asked)
		}
		down = false
		r2.keeper.AskNode("n2", "answered now, its manifest-reads-changed heard")
		if len(st.vals) != 0 {
			t.Errorf("answered leaves %v", st.vals)
		}
	})
}

func mustStatement(t *testing.T, s enginereads.Statement) []byte {
	t.Helper()
	raw, err := enginereads.Encode(s)
	if err != nil {
		t.Fatal(err)
	}
	return raw
}

// memReads is the builder's state of engine reads in memory, as the tool reads it.
type memReads struct{ v map[string][]byte }

func (m *memReads) Keys() ([]string, error) { return slices.Collect(maps.Keys(m.v)), nil }
func (m *memReads) Get(key string) ([]byte, uint64, bool, error) {
	v, ok := m.v[key]
	return v, 1, ok, nil
}
func (m *memReads) Put(key string, value []byte) (uint64, error) { m.v[key] = value; return 2, nil }
func (m *memReads) Delete(key string) error                      { delete(m.v, key); return nil }
func (m *memReads) Await(string, uint64, func([]byte) bool, time.Duration) ([]byte, error) {
	return nil, errors.New("no part answers here")
}

// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
	refuseOpen bool
	err        error
}

func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
	if r.refuseOpen {
		return nil, r.err
	}
	return refusedWatch{err: r.err}, nil
}

type refusedWatch struct {
	jetstream.KeyValue
	err error
}

func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
	return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
	return nil, r.err
}

// wakeMsg is a message of the builder's events consumer: its subject and body, acknowledged.
type wakeMsg struct {
	jetstream.Msg
	subject string
	data    []byte
	acked   bool
	naked   bool
	delayed time.Duration
}

func (m *wakeMsg) Subject() string { return m.subject }
func (m *wakeMsg) Data() []byte    { return m.data }
func (m *wakeMsg) Ack() error      { m.acked = true; return nil }
func (m *wakeMsg) Nak() error      { m.naked = true; return nil }
func (m *wakeMsg) NakWithDelay(d time.Duration) error {
	m.delayed = d
	return nil
}
func (m *wakeMsg) Metadata() (*jetstream.MsgMetadata, error) {
	return nil, errors.New("no metadata")
}
func (m *wakeMsg) Headers() nats.Header {
	return nil
}

func aFloorsWakeIsHeard(t *testing.T) {
	var logs bytes.Buffer
	holders := map[string]string{"git": "mesh-delivery-forge-bridge", "mesh-catalog": "mesh-delivery-indexer"}
	waits := &versions.FloorWaits{Holder: func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }}
	waits.Wait("m/lib", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceRequirements}}, nil, time.Unix(0, 0))
	waits.Wait("m/img", versions.SeatImages, "", versions.Floor{Silent: []string{versions.SourceTags}}, nil, time.Unix(0, 0))
	f := &floorWake{b: &build.Builder{}, waits: waits, logf: log.New(&logs, "", 0).Printf}
	accepted := &wakeMsg{subject: manifestAcceptedSubject}
	if !f.heard(context.Background(), accepted) || !accepted.acked || !strings.Contains(logs.String(), "the floor of m/lib is asked again: manifest-accepted") {
		t.Fatalf("manifest-accepted: acked %v, log %q", accepted.acked, logs.String())
	}
	logs.Reset()
	cleared := &wakeMsg{subject: conditionClearedSubject, data: []byte(`{"subject":{"scope":"probe","id":"ace/mesh-delivery-forge-bridge"}}`)}
	if !f.heard(context.Background(), cleared) || !strings.Contains(logs.String(), "the floor of m/img is asked again: condition-cleared") {
		t.Fatalf("condition-cleared: log %q", logs.String())
	}
	if f.heard(context.Background(), &wakeMsg{subject: sweep.SubjectArtifactsGone}) {
		t.Fatal("artifacts-gone was taken as a floor's wake")
	}
	// DLV-B4: a registry seat's outcome on the same consumer moves a version; a published wakes a name whose seat was
	// silent; one the state cannot take is left for the bus to give again.
	logs.Reset()
	waits.Wait("m/pkg", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceSeat}}, nil, time.Unix(0, 0))
	h := &seatHearer{n: &versions.Numberer{Versions: versions.Versions{Store: noVersions{}}, Waits: waits}, b: &build.Builder{}, logf: log.New(&logs, "", 0).Printf}
	byDigest := &wakeMsg{subject: "mesh.seat.go-package-registry.event.published", data: []byte(`{"name":"m/other","digest":"sha256:aa","kind":"package","holder":"gitea"}`)}
	if !h.heard(context.Background(), byDigest) || !byDigest.acked || !strings.Contains(logs.String(), "the floor of m/pkg is asked again: go-package-registry said published") {
		t.Fatalf("published: acked %v, log %q", byDigest.acked, logs.String())
	}
	refused := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.refused", data: []byte(`{"name":"m/program","version":"0.1.3","digest":"sha256:bb","reason":"bytes differ"}`)}
	if !h.heard(context.Background(), refused) || refused.acked || !refused.naked {
		t.Fatalf("a refusal the state cannot take: acked %v, naked %v, log %q", refused.acked, refused.naked, logs.String())
	}
	if h.heard(context.Background(), &wakeMsg{subject: manifestAcceptedSubject}) {
		t.Fatal("manifest-accepted was taken as a seat's outcome")
	}
}

// noVersions is a state versions that holds nothing.
type noVersions struct{}

func (noVersions) Get(context.Context, string) ([]byte, uint64, bool, error) {
	return nil, 0, false, nil
}
func (noVersions) Create(context.Context, string, []byte) (uint64, error) {
	return 0, errors.New("read only")
}
func (noVersions) Update(context.Context, string, []byte, uint64) (uint64, error) {
	return 0, errors.New("read only")
}
func (noVersions) KeysOf(context.Context, string) ([]string, error) { return nil, nil }

// failingVersions is a state versions that cannot be read.
type failingVersions struct{ noVersions }

func (failingVersions) KeysOf(context.Context, string) ([]string, error) {
	return nil, errors.New("down")
}

func aFailedTrunkBuildIsAFinding(t *testing.T) {
	st := &memFindings{vals: map[string][]byte{}}
	f := &findings{module: builds.BuilderModule, w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
	ended := func(outcome, ref string) builds.BuildLogEntry {
		return builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "m", FP: "f1", Commit: "c", Ref: ref, Outcome: outcome}
	}
	key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m", BuildFailed)
	f.buildEnded(context.Background(), ended(builds.BuildFailed, "refs/pull/3/head"))
	if len(st.vals) != 0 {
		t.Fatalf("a candidate's failed build is a finding: %v", st.vals)
	}
	f.buildEnded(context.Background(), ended(builds.BuildFailed, ""))
	got, err := decodeFinding(st.vals[key])
	if err != nil || got.Observer != builds.BuilderModule || !strings.Contains(got.Summary, "trunk build of m at f1") {
		t.Fatalf("the finding = %+v, %v", got, err)
	}
	f.buildEnded(context.Background(), ended(builds.BuildDone, ""))
	if _, ok := st.vals[key]; ok {
		t.Error("a later done trunk build leaves build-failed")
	}
	// The floor finding: raised while a name waits on its floor, cleared once it is known.
	fw := &versions.FloorWaits{Said: f.floorSaid(context.Background())}
	fw.Wait("m/lib", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceTags}}, nil, time.Unix(0, 0))
	floorKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m/lib", versions.FloorSilent)
	if got, err := decodeFinding(st.vals[floorKey]); err != nil || !strings.Contains(got.Summary, "git.tags did not answer") {
		t.Fatalf("the floor finding = %+v, %v", got, err)
	}
	fw.Answered("m/lib")
	if _, ok := st.vals[floorKey]; ok {
		t.Error("a known floor leaves floor-silent")
	}
	// Design 51 *Health and conditions* (review round 1): a holder's refused is publish-refused with its reason, a version
	// no holder answered within its bound publish-unanswered, a seat's outcome that does not read seat-outcome-unread;
	// a later published of the name clears the first two, the seat's next outcome that reads the third.
	vs := versions.Versions{Store: memVersions{}}
	for _, v := range []string{"0.1.3", "0.1.4"} {
		raw, err := versions.Encode(versions.Entry{Name: "m/program", Kind: versions.KindBundle, Series: "0.1", Version: v, Seat: "oci-artifact-registry", State: versions.Numbered})
		if err != nil {
			t.Fatal(err)
		}
		vs.Store.(memVersions)[versions.Key("m/program", v)] = raw
	}
	pubs := &publishes{bounds: func() jetstream.KeyValue { return nil }, versions: vs, findings: f, bound: versions.PublishBound, logf: t.Logf}
	h := &seatHearer{n: &versions.Numberer{Versions: vs}, b: &build.Builder{}, logf: t.Logf, pub: pubs}
	refusedKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m/program", versions.PublishRefused)
	refused := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.refused", data: []byte(`{"name":"m/program","version":"0.1.3","digest":"sha256:bb","reason":"bytes differ"}`)}
	if !h.heard(context.Background(), refused) || !refused.acked {
		t.Fatalf("a refusal: acked %v", refused.acked)
	}
	if got, err := decodeFinding(st.vals[refusedKey]); err != nil || !strings.Contains(got.Summary, "refused to publish m/program@0.1.3") || !strings.Contains(got.Summary, "bytes differ") {
		t.Fatalf("a holder's refused: the finding = %+v, %v", got, err)
	}
	pubs.ended(context.Background(), purged{key: versions.PublishBoundKey("m/program", "0.1.4")})
	unansweredKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m/program", versions.PublishUnanswered)
	if got, err := decodeFinding(st.vals[unansweredKey]); err != nil || !strings.Contains(got.Summary, "neither published nor refused of m/program@0.1.4") {
		t.Fatalf("a bound ended unanswered: the finding = %+v, %v", got, err)
	}
	// Review round 2: a bound that cannot be written is publish-unanswered at once; a watch of bounds that cannot start
	// is family-not-heard naming the state.
	delete(st.vals, unansweredKey)
	pubs.arm(context.Background(), versions.Entry{Name: "m/program", Version: "0.1.4", Seat: "oci-artifact-registry"})
	if got, err := decodeFinding(st.vals[unansweredKey]); err != nil || !strings.Contains(got.Summary, "its bound could not be written") {
		t.Fatalf("a bound not written: the finding = %+v, %v", got, err)
	}
	watchBounds(context.Background(), refusedWatch{err: errors.New("permissions violation")}, func(jetstream.KeyValueEntry) {}, f.notHeard(context.Background()), nil, t.Logf)
	if got, err := decodeFinding(st.vals[heard.Key(builds.BuilderModule, versions.BoundsState)]); err != nil || !strings.Contains(got.Summary, "permissions violation") {
		t.Fatalf("bounds not watched: the finding = %+v, %v", got, err)
	}
	delete(st.vals, heard.Key(builds.BuilderModule, versions.BoundsState))
	malformed := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", data: []byte(`{"version":"0.1.4"}`)}
	unreadKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "oci-artifact-registry", versions.SeatOutcomeUnread)
	if !h.heard(context.Background(), malformed) || !malformed.acked || st.vals[unreadKey] == nil {
		t.Fatalf("an outcome that does not read: acked %v, findings %v", malformed.acked, st.vals)
	}
	published := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", data: []byte(`{"name":"m/program","version":"0.1.4","digest":"sha256:cc"}`)}
	if !h.heard(context.Background(), published) || st.vals[refusedKey] != nil || st.vals[unansweredKey] != nil || st.vals[unreadKey] != nil {
		t.Fatalf("a later published leaves the findings: %v", st.vals)
	}
	// Issue 1966 (ADR 0668 §3, §5; 1748 note 27): a key a look left building because its end could not be written arms its
	// bound; the bound's end while the key is still building and no look holds it is call-unanswered about the key; the key's
	// next written end deletes the bound and clears it; a key a look holds, or one no longer building, raises nothing; a
	// bound that cannot be written raises at once.
	buildKey := "m.1111111111111111111111111111111111111111111111111111111111111111"
	building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Step: "claimed"})
	done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "novox.builder", Step: "built"})
	keys := memBuilds{memVersions{buildKey: building}}
	looking := false
	unputMarks := &memBounds{keys: map[string]bool{}}
	u := &unputs{bounds: func() jetstream.KeyValue { return unputMarks }, store: keys, looking: func(string) bool { return looking },
		findings: f, bound: meshconditionsclient.CallUnansweredAfter, now: func() time.Time { return time.Unix(600, 0) }, logf: t.Logf}
	callKey := meshconditionsclient.Key(meshconditionsclient.ScopeCall, buildKey, meshconditionsclient.KindCallUnanswered)
	u.left(context.Background(), buildKey, "nats: timeout")
	if !unputMarks.keys[versions.UnputBoundKey(buildKey)] || st.vals[callKey] != nil {
		t.Fatalf("a key left building: bound %v, finding %v; want the bound armed, no finding yet", unputMarks.keys, st.vals[callKey] != nil)
	}
	for name, c := range map[string]struct {
		value []byte
		look  bool
	}{"a look holds it": {building, true}, "no longer building": {done, false}} {
		keys.memVersions[buildKey], looking = c.value, c.look
		u.expired(context.Background(), purged{key: versions.UnputBoundKey(buildKey)})
		if st.vals[callKey] != nil {
			t.Errorf("%s: the bound's end raised call-unanswered", name)
		}
	}
	keys.memVersions[buildKey], looking = building, false
	u.expired(context.Background(), purged{key: versions.UnputBoundKey(buildKey)})
	if got, err := decodeFinding(st.vals[callKey]); err != nil || got.Kind != meshconditionsclient.KindCallUnanswered || !strings.Contains(got.Summary, buildKey) || !strings.Contains(got.Needs, build.BuildAgainTool) {
		t.Fatalf("the bound's end of a key left building: the finding = %+v, %v", got, err)
	}
	u.ended(context.Background(), buildKey)
	if unputMarks.keys[versions.UnputBoundKey(buildKey)] || st.vals[callKey] != nil {
		t.Errorf("the key's end written: bound %v, finding left %v", unputMarks.keys, st.vals[callKey] != nil)
	}
	u.bounds = func() jetstream.KeyValue { return nil }
	u.left(context.Background(), buildKey, "nats: timeout")
	if got, err := decodeFinding(st.vals[callKey]); err != nil || !strings.Contains(got.Explanation, "nor its bound") {
		t.Fatalf("a bound not written: the finding = %+v, %v", got, err)
	}
	delete(st.vals, callKey)
	// ADR 0615 §5 (review round 1): a consumer not there yet is family-not-heard naming its stream, cleared when it binds.
	notHeard := f.notHeard(context.Background())
	unheardKey := heard.Key(builds.BuilderModule, "EVENTS")
	_ = notHeard.Bind("EVENTS", "a branch's end is not heard", func() error { return errors.New("consumer not found") })
	if got, err := decodeFinding(st.vals[unheardKey]); err != nil || got.Kind != heard.Kind || !strings.Contains(got.Summary, "does not hear EVENTS") {
		t.Fatalf("a consumer not there: the finding = %+v, %v", got, err)
	}
	_ = notHeard.Bind("EVENTS", "a branch's end is not heard", func() error { return nil })
	if _, ok := st.vals[unheardKey]; ok {
		t.Error("a consumer bound leaves family-not-heard")
	}
	// DLV-B9, the old form's window: one counted node reading only the first form holds it; seven days on it is
	// old-form-held about that node; the node stating the newest form closes it, said once, the finding cleared; a
	// retired node dropped by hand leaves the count.
	now := time.Unix(0, 0)
	fs := forms.ArtifactForms{{Name: "registry", First: true, Kinds: []string{"image"}}, {Name: "oci", Kinds: []string{"image"}}}
	var closings []versions.OldFormClosedBody
	var logged []builds.BuildLogEntry
	marks := &memBounds{keys: map[string]bool{}}
	window := &oldForm{module: builds.BuilderModule, forms: fs, findings: f, logf: t.Logf, now: func() time.Time { return now }, bounds: func() jetstream.KeyValue { return marks },
		said: func(b versions.OldFormClosedBody) { closings = append(closings, b) },
		log:  func(e builds.BuildLogEntry) { logged = append(logged, e) }}
	reads := &engineReads{module: builds.BuilderModule, keeper: &enginereads.Keeper{}, window: window, counted: []string{"a", "b"}}
	// Issue 1966 note 155, ADR 0668 §4: at the consumer's bind every counted node is called once again.
	var boundCalls []string
	reads.keeper.Sync = true
	reads.keeper.Call = func(node, _ string, _ any) (json.RawMessage, error) {
		boundCalls = append(boundCalls, node)
		return nil, errors.New("not served")
	}
	reads.atBind(now)
	if !slices.Equal(boundCalls, []string{"a", "b"}) {
		t.Fatalf("at the bind called %v, want every counted node once", boundCalls)
	}
	reads.keeper = &enginereads.Keeper{}
	newest := []string{forms.FormNeed("oci")}
	counted := map[string][]string{"a": newest, "b": nil}
	window.step(context.Background(), []string{"a", "b"}, counted)
	heldKey := meshconditionsclient.Key(meshconditionsclient.ScopeMachine, "b", versions.OldFormHeld)
	if !window.w.Open || !slices.Equal(window.w.Holding, []string{"b"}) || st.vals[heldKey] != nil {
		t.Fatalf("one node lacking the newest form: %+v", window.w)
	}
	// Review round 1: the node holding it has a bound in the state bounds ending at its seven days, so old-form-held is
	// raised then, not at the next change.
	if !marks.keys[versions.OldFormBoundKey("b")] || marks.keys[versions.OldFormBoundKey("a")] {
		t.Fatalf("the old form's bounds = %v, want b's alone", marks.keys)
	}
	// Security-upgrade F3 (review round 1): an empty node_haves read, naming no node, leaves the window open.
	window.step(context.Background(), nil, map[string][]string{})
	if !window.w.Open || !slices.Equal(window.w.Holding, []string{"b"}) || len(closings) != 0 {
		t.Fatalf("an empty read closed the window: %+v, closings %+v", window.w, closings)
	}
	now = now.Add(versions.FindingAfter)
	window.step(context.Background(), []string{"a", "b"}, counted)
	if got, err := decodeFinding(st.vals[heldKey]); err != nil || !strings.Contains(got.Summary, "the node-engine of b") {
		t.Fatalf("seven days held: the finding = %+v, %v", got, err)
	}
	counted["b"] = newest
	window.step(context.Background(), []string{"a", "b"}, counted)
	window.step(context.Background(), []string{"a", "b"}, counted)
	if marks.keys[versions.OldFormBoundKey("b")] {
		t.Error("a node no longer holding the window keeps its bound")
	}
	if window.w.Open || len(closings) != 1 || !slices.Equal(closings[0].LastHolding, []string{"b"}) || st.vals[heldKey] != nil {
		t.Errorf("the node back with the newest form: %+v, closings %+v, finding left %v", window.w, closings, st.vals[heldKey] != nil)
	}
	if len(logged) != 1 || logged[0].Event != builds.BuildSaidEvent || logged[0].Why != versions.SaidOldFormClosed ||
		!strings.Contains(logged[0].Line, "last held by b") {
		t.Errorf("the build log = %+v, want the close said once (build.said)", logged)
	} else if _, err := builds.EncodeBuildLogEntry(logged[0]); err != nil {
		t.Errorf("the close is no entry the build log takes: %v", err)
	}
	if _, err := window.drop(context.Background(), "b", "op@shanks", "retired", now); err != nil {
		t.Fatal(err)
	}
	if _, c := reads.forms(); len(c) != 1 || c["b"] != nil {
		t.Errorf("a node dropped by hand is still counted: %v", c)
	}
}

// memVersions is a state versions in memory.
type memVersions map[string][]byte

func (m memVersions) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
	v, ok := m[key]
	return v, 1, ok, nil
}
func (m memVersions) Create(_ context.Context, key string, v []byte) (uint64, error) {
	m[key] = v
	return 1, nil
}
func (m memVersions) Update(_ context.Context, key string, v []byte, _ uint64) (uint64, error) {
	m[key] = v
	return 2, nil
}
func (m memVersions) KeysOf(context.Context, string) ([]string, error) {
	return slices.Collect(maps.Keys(m)), nil
}

// memBuilds is the builder's builds in memory.
type memBuilds struct{ memVersions }

func (m memBuilds) Keys(context.Context) ([]string, error) {
	return slices.Collect(maps.Keys(m.memVersions)), nil
}

// memBounds is the state bounds in memory: which keys stand.
type memBounds struct {
	jetstream.KeyValue
	keys map[string]bool
}

func (m *memBounds) Create(_ context.Context, key string, _ []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
	if m.keys[key] {
		return 0, jetstream.ErrKeyExists
	}
	m.keys[key] = true
	return 1, nil
}
func (m *memBounds) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
	delete(m.keys, key)
	return nil
}

// purged is the bus's marker of a key whose time to live ended.
type purged struct {
	jetstream.KeyValueEntry
	key string
}

func (p purged) Key() string                     { return p.key }
func (p purged) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePurge }

// memFindings is a findings bucket in memory.
type memFindings struct{ vals map[string][]byte }

// Write keeps the finding under its key, as the held writer does in the state.
func (m *memFindings) Write(_ context.Context, f mcseat.Finding) error {
	v, err := json.Marshal(f)
	m.vals[views.FindingKey(f.Scope, f.ID, f.Kind)] = v
	return err
}

// Clear deletes the finding of scope, id and kind.
func (m *memFindings) Clear(_ context.Context, scope, id, kind, _ string) error {
	delete(m.vals, views.FindingKey(scope, id, kind))
	return nil
}

// decodeFinding reads a finding as the writer kept it.
func decodeFinding(v []byte) (mcseat.Finding, error) {
	var f mcseat.Finding
	return f, json.Unmarshal(v, &f)
}

// Design 47 G A-296: the builder's findings are observed by the module its credential names, never a constant of its name.
func TestTheBuildersFindingsAreObservedByItsCredentialsModule(t *testing.T) {
	st := &memFindings{vals: map[string][]byte{}}
	f := &findings{module: "acme-builder", w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
	f.buildEnded(context.Background(), builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "m", FP: "f1", Commit: "c", Outcome: builds.BuildFailed})
	got, err := decodeFinding(st.vals[meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m", BuildFailed)])
	if err != nil || got.Observer != "acme-builder" {
		t.Fatalf("the finding = %+v, %v; want the observer acme-builder", got, err)
	}
}