mesh-delivery/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go

//go:build integration

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"maps"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/nats-server/v2/server"
	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"

	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/announce"
	"git.novox.be/novox/mesh-sdk/go/asks"
	"git.novox.be/novox/mesh-sdk/go/buscompose"
	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/health"
	"git.novox.be/novox/mesh-sdk/go/process"
	"git.novox.be/novox/mesh-sdk/go/provisions"
	"mesh-delivery-assigner/internal/judge"

	"mesh-delivery-assigner/internal/assign"
)

// bed is an embedded bus with JetStream, every bucket the assigner reads and keeps, and the stream the mesh's events
// are kept in, as the controller's broker raises them.
type bed struct {
	srv     *server.Server
	js      jetstream.JetStream
	ctx     context.Context
	dir     string
	buckets map[string]jetstream.KeyValue
	// placed is, by node, the modules the bed's placement names on it (place).
	placed map[string][]string
}

func startBed(t *testing.T, leaveOut string) *bed {
	t.Helper()
	dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "assigner")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { os.RemoveAll(dir) })
	srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: filepath.Join(dir, "js"), NoLog: true, NoSigs: true})
	if err != nil {
		t.Fatal(err)
	}
	go srv.Start()
	if !srv.ReadyForConnections(10 * time.Second) {
		t.Fatal("the bus did not come up")
	}
	t.Cleanup(srv.Shutdown)
	nc, err := nats.Connect(srv.ClientURL())
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(nc.Close)
	js, err := jetstream.New(nc)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	t.Cleanup(cancel)
	b := &bed{srv: srv, js: js, ctx: ctx, dir: dir, buckets: map[string]jetstream.KeyValue{}}
	for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
		if name == leaveOut {
			continue
		}
		cfg := jetstream.KeyValueConfig{Bucket: name, History: 5}
		if name == delivery.PlacementPulsesBucket {
			cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl)
		}
		kv, err := js.CreateKeyValue(ctx, cfg)
		if err != nil {
			t.Fatal(err)
		}
		b.buckets[name] = kv
	}
	// The builder's module log, as the runtime raises it (ADR 0297): the running commits are read from it (ADR 0571 clause 1).
	if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog),
		Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
		t.Fatal(err)
	}
	if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
		t.Fatal(err)
	}
	// The operator-channel seat's stream of conditions, which the assigner says its conditions on.
	if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CONDITIONS", Subjects: []string{"mesh.seat.operator-channel.accept.condition.>"}}); err != nil {
		t.Fatal(err)
	}
	// The Go build seat's work queue, as the broker derives it from the seat's accepted kinds (delivery.AskStream).
	seat, _ := delivery.AskStream(delivery.AskGo)
	if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: seat, Subjects: []string{"mesh.seat.node-build-go.accept.>"},
		Retention: jetstream.WorkQueuePolicy}); err != nil {
		t.Fatal(err)
	}
	return b
}

// start runs the program as the node does, through run with its environment and a broker file; it answers the
// channel run's error arrives on.
func (b *bed) start(t *testing.T) <-chan error {
	t.Helper()
	done, _ := b.startStoppable(t)
	return done
}

// startStoppable is start, and answers the function that stops this one program (a replaced assigner is stopped, then
// another started).
func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
	t.Helper()
	broker := filepath.Join(b.dir, "broker")
	cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-assigner", User: "n1.mesh-delivery-assigner", Password: "pw"})
	if err := os.WriteFile(broker, cred, 0o600); err != nil {
		t.Fatal(err)
	}
	env := map[string]string{"MESH_BROKER_FILE": broker}
	ctx, stop := context.WithCancel(b.ctx)
	done := make(chan error, 1)
	go func() {
		done <- run(ctx, func(k string) string { return env[k] }, io.Discard)
		close(done) // a program that ended on its own (a refused start) is not waited for again at the cleanup
	}()
	stopped := func() {
		stop()
		select {
		case <-done:
		case <-time.After(10 * time.Second):
		}
	}
	t.Cleanup(stopped)
	return done, stopped
}

// 1297/B2b: started whole on its credential, the assigner says on the bus every state its manifest declares, each readable,
// having opened every bucket it names. The states are read from module.json, so a state added there is expected here too.
func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) {
	events, err := b.js.Stream(b.ctx, "EVENTS")
	if err != nil {
		t.Fatal(err)
	}
	var said announce.States
	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		select {
		case err := <-done:
			t.Fatalf("the assigner stopped: %v", err)
		default:
		}
		m, err := events.GetLastMsgForSubject(b.ctx, announce.Subject("mesh-delivery-assigner"))
		if err != nil {
			continue
		}
		if err := json.Unmarshal(m.Data, &said); err != nil {
			t.Fatal(err)
		}
		break
	}
	raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
	if err != nil {
		t.Fatal(err)
	}
	m, err := catalogue.Decode(raw)
	if err != nil {
		t.Fatal(err)
	}
	if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != len(m.State) {
		t.Fatalf("the assigner said %+v, want the %d states its manifest declares", said, len(m.State))
	}
	for _, s := range m.State {
		if !said.Readable(s.Name) {
			t.Errorf("the assigner said %+v: its state %s is not readable", said, s.Name)
		}
	}
}

// 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket.
func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
	for _, missing := range readBuckets {
		if startsWithout(missing) {
			continue // it starts without it (the placement and the grants' source: main_test.go; the puts: below)
		}
		b := startBed(t, missing)
		select {
		case err := <-b.start(t):
			if err == nil || !strings.Contains(err.Error(), missing) {
				t.Errorf("without %s the assigner stopped with %v, want the bucket named", missing, err)
			}
		case <-time.After(15 * time.Second):
			t.Errorf("without %s the assigner started", missing)
		}
	}
	// ADR 0546: without the puts' bucket it starts all the same, and once the bucket is made (by the controller at its start,
	// ADR 0201) a pass opens it, so a put is taken further without a restart.
	was := PutsReopenEvery
	PutsReopenEvery = 0
	defer func() { PutsReopenEvery = was }()
	b := startBed(t, delivery.AssignerPutsBucket)
	done := b.start(t)
	select {
	case err := <-done:
		t.Fatalf("without %s the assigner stopped: %v", delivery.AssignerPutsBucket, err)
	case <-time.After(2 * time.Second):
	}
	puts, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerPutsBucket, History: 5})
	if err != nil {
		t.Fatal(err)
	}
	pkey, _ := delivery.PutKey("n1", "web")
	asked := delivery.Put{FP: delivery.SourcePrefix2 + strings.Repeat("b", 64), Why: "a try", At: time.Now()}.Said(time.Now(), delivery.PutAsked, "asked")
	raw, _ := delivery.EncodePut(asked)
	if _, err := puts.Put(b.ctx, pkey, raw); err != nil {
		t.Fatal(err)
	}
	for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
		// A change of a bucket it watches starts a pass, which opens the puts' bucket.
		if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
			t.Fatal(err)
		}
		if e, err := puts.Get(b.ctx, pkey); err == nil {
			if got, err := delivery.DecodePut(e.Value()); err == nil && got.Step == delivery.PutRefused && strings.Contains(got.Words, "not assigned") {
				break
			}
		}
		if time.Now().After(deadline) {
			t.Fatal("the put was not taken further once its bucket was made")
		}
	}

	// Without the seat's placement state it starts all the same, and once the bucket is made (the controller makes
	// it from the manifest) it is read with no restart: a node it names gets its own want, the node composing its own.
	pb := startBed(t, delivery.PlacementBucket)
	pdone := pb.start(t)
	select {
	case err := <-pdone:
		t.Fatalf("without %s the assigner stopped: %v", delivery.PlacementBucket, err)
	case <-time.After(2 * time.Second):
	}
	placement, err := pb.js.CreateKeyValue(pb.ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: 5})
	if err != nil {
		t.Fatal(err)
	}
	pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "web"}}})
	plkey, _ := delivery.LegacyPlacementKey("n1") // the form before, read beside the new (research 072 row 37)
	if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
		t.Fatal(err)
	}
	if nw, err := delivery.DecodeNodeWant(pb.get(t, delivery.AssignerWantsBucket, "want.n1.node")); err != nil || !nw.Self {
		t.Fatalf("the placement state made after the start was not read: n1's own want = %+v, %v", nw, err)
	}
}

// putJSON writes a value into one of the bed's buckets.
func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
	t.Helper()
	if _, err := b.buckets[bucket].Put(b.ctx, key, value); err != nil {
		t.Fatalf("%s %s: %v", bucket, key, err)
	}
}

// place adds module to the seat's placement of node, written as the seat writes it: one node key and a key per module
// (research 072 row 37). The placement alone names where modules run (ADR 0567 §8).
func (b *bed) place(t *testing.T, node, module string) {
	t.Helper()
	if b.placed == nil {
		b.placed = map[string][]string{}
	}
	if !slices.Contains(b.placed[node], module) {
		b.placed[node] = append(b.placed[node], module)
	}
	pl := delivery.Placement{Node: node}
	for _, m := range b.placed[node] {
		pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
	}
	values, err := delivery.PlacementValues(pl)
	if err != nil {
		t.Fatal(err)
	}
	for key, raw := range values {
		b.putJSON(t, delivery.PlacementBucket, key, raw)
	}
}

// compose says the node's own composition is the digest d (a letter repeated), as its have names it (`<node>.node`): a
// placed node composes its own (ADR 0571 §6), and the controller's composition is read no more. It answers the digest.
func (b *bed) compose(t *testing.T, node string, d byte) string {
	t.Helper()
	key, err := delivery.HaveNodeKey(node)
	if err != nil {
		t.Fatal(err)
	}
	digest := "sha256:" + strings.Repeat(string(d), 64)
	v, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
	if err != nil {
		t.Fatal(err)
	}
	b.putJSON(t, delivery.HaveBucket, key, v)
	return digest
}

// manifest keeps a manifest in the indexer's bucket by its digest and answers the digest.
func (b *bed) manifest(t *testing.T, module, body string) string {
	t.Helper()
	digest := delivery.ManifestDigest([]byte(body))
	key, _ := delivery.ManifestsKey(digest)
	value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body})
	if err != nil {
		t.Fatal(err)
	}
	b.putJSON(t, delivery.ManifestsBucket, key, value)
	return digest
}

// get waits for a key of one of the bed's buckets and answers its value.
func (b *bed) get(t *testing.T, bucket, key string) []byte {
	t.Helper()
	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		if e, err := b.buckets[bucket].Get(b.ctx, key); err == nil {
			return e.Value()
		}
	}
	t.Fatalf("no %s %s after 10 s", bucket, key)
	return nil
}

// 1297/B4 on the bus, 1906/D5: the controller sends ace, g14 and shanks builds while the canary ace does not run F healthy yet.
// Started whole, the assigner wants F on the canary and writes no difference at all: there is no send to compare with (ADR 0539
// §8). 1945/AC19, AC21: shanks, which runs the older fingerprint (composed and granted) with no want, gets its first want at that
// fingerprint, never at F; g14, which runs a build of the old form, gets none, so web is on neither list there and the assigner
// raises the condition naming g14, web and the canary rule; once ace runs F healthy, g14's first want names F and the condition is
// cleared.
func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
	b := startBed(t, "")
	fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
	digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
	mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
	b.putJSON(t, delivery.ModulesBucket, "web", mod)
	built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
	if err != nil {
		t.Fatal(err)
	}
	bkey, _ := delivery.BuilderBuildKey("web", fNew)
	b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
	for _, fp := range []string{fNew, fOld} {
		gkey, _ := delivery.PartGrantKey("web", fp)
		grant, _ := delivery.EncodeGrant(delivery.Grant{})
		b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
	}
	runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
	for _, node := range []string{"ace", "g14", "shanks"} {
		b.compose(t, node, 'c')
		on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
		b.putJSON(t, delivery.PresenceBucket, node, on)
		have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
		b.putJSON(t, delivery.HaveBucket, node+".web", have)
		b.place(t, node, "web")
	}
	b.compose(t, "shanks", 'a')
	b.start(t)

	wkey, _ := delivery.WantKey("ace", "web")
	want, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey))
	if err != nil || want.FP != fNew {
		t.Fatalf("the canary ace's want = %+v, %v", want, err)
	}
	if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "" || !w.Self || w.Newest != fNew {
		t.Errorf("shanks' want = %+v, want the fingerprint it runs, the node composing it, held back from F", w)
	}
	var neither asks.Condition
	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
		for k, c := range b.conditionsSaid(t) {
			if strings.HasSuffix(k, ".neither") && c.State == asks.StateRaised {
				neither = c
			}
		}
	}
	if neither.Subject.Machine != "g14" || !strings.HasPrefix(neither.Subject.ID, "g14.web") || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
		t.Errorf("the condition of web on neither list on g14 = %+v", neither)
	}
	if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
		t.Errorf("differences were written: %v", keys)
	}
	if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
		t.Errorf("g14, which runs an old-form build and is held back by the canary, was given a want: %s", e.Value())
	}

	// 1945/AC21: once the canary ace runs F healthy, g14's first want names F by the rules as usual, and web leaves neither list
	// there by itself: the condition is cleared.
	healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
	if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "" || !w.Self || w.Newest != "" {
		t.Errorf("g14's want = %+v, want F, the node composing it", w)
	}
	cleared := false
	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !cleared; time.Sleep(100 * time.Millisecond) {
		c, ok := b.conditionsSaid(t)[neither.Key]
		cleared = ok && c.State == asks.StateCleared
	}
	if !cleared {
		t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
	}

	// Issue 1126 (ADR 0547 §2): the catalogue's placement reaches the indexer's state naming web on every node the
	// assignments name it on. Each node then composes its own: its want names the build alone (Self, no composition), and
	// its own want says it composes its own, which the node agent reads (mesh-host's TestNats...ComposesAndAppliesItsOwn).
	for _, node := range []string{"ace", "g14", "shanks"} {
		pl, err := delivery.EncodePlacement(delivery.Placement{Node: node, Modules: []delivery.PlacedModule{{Module: "web"}}})
		if err != nil {
			t.Fatal(err)
		}
		pkey, _ := delivery.LegacyPlacementKey(node)
		b.putJSON(t, delivery.PlacementBucket, pkey, pl)
	}
	if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w delivery.Want) bool { return w.Self }); w.Composed != "" || w.FP != fNew {
		t.Errorf("ace's want once placed = %+v", w)
	}
	nkey, _ := delivery.WantNodeKey("ace")
	var self bool
	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !self; time.Sleep(100 * time.Millisecond) {
		if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
			nw, err := delivery.DecodeNodeWant(e.Value())
			self = err == nil && nw.Self
		}
	}
	if !self {
		t.Error("ace's own want does not say it composes its own once placed")
	}

	// ADR 0603 (L3, D15): one source revision changes a provider (db), its new consumer (app) and a build module (tool, whose
	// recipe moved: ace runs its first build and the second, under the agreed toolchain, is the one the source revision names).
	// The keys are written as their writers write them: the indexer's modules and manifests, the builder's builds, the
	// controller's grants, the placement seat's placement and the provisioner's provisions. The build module's want moves at
	// once and the provider's with it; the consumer's waits for its provision, with the words naming it, and moves once the
	// provisioner provides it (rule 8, ADR 0570 §4): nothing waits for a turn or an order, only for a key.
	offer := `"provides":[{"name":"database","scope":"mesh","answered-by":"helper"}]`
	dbDigest := b.manifest(t, "db", `{"module":"db","version":"1",`+offer+`}`)
	appDigest := b.manifest(t, "app", `{"module":"app","version":"1","requires":["database"]}`)
	toolDigest := b.manifest(t, "tool", `{"module":"tool","version":"1"}`)
	asked := func(state, why string) {
		raw, err := provisions.Encode(provisions.Provision{V: provisions.Version, Provision: "database", Consumer: provisions.End{Module: "app", Node: "ace"},
			Provider: provisions.End{Module: "db", Node: "ace"}, State: state, Why: why, At: time.Now()})
		if err != nil {
			t.Fatal(err)
		}
		b.putJSON(t, provisions.ProvisionsBucket, "ace.app.database", raw)
	}
	// The provisioner keys it from the manifests, as soon as they are known and whether the provider is placed yet or not.
	asked(provisions.Asked, "the provider's build is not done")
	fpOf := func(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
	toolOld := fpOf('1')
	for module, row := range map[string]struct {
		fp, digest string
	}{"db": {fpOf('2'), dbDigest}, "app": {fpOf('3'), appDigest}, "tool": {fpOf('4'), toolDigest}} {
		mod, _ := delivery.EncodeModule(delivery.Module{FP2: row.fp, Manifest: row.digest, Repository: "novox/" + module, Commit: strings.Repeat("2", 40), Path: delivery.RootPath})
		b.putJSON(t, delivery.ModulesBucket, module, mod)
		fps := []string{row.fp}
		if module == "tool" {
			fps = append(fps, toolOld)
		}
		for _, fp := range fps {
			built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: row.digest})
			bkey, _ := delivery.BuilderBuildKey(module, fp)
			b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
			gkey, _ := delivery.PartGrantKey(module, fp)
			grant, _ := delivery.EncodeGrant(delivery.Grant{})
			b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
		}
		b.place(t, "ace", module)
	}
	runsOld, _ := delivery.EncodeHave(delivery.Have{FP: toolOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	b.putJSON(t, delivery.HaveBucket, "ace.tool", runsOld)
	if w := b.wantNow(t, "ace", "tool", "the build module's want at the second build", func(w delivery.Want) bool { return w.FP == fpOf('4') }); w.FP != fpOf('4') {
		t.Errorf("tool's want = %+v", w)
	}
	b.wantNow(t, "ace", "db", "the provider's want at its fingerprint", func(w delivery.Want) bool { return w.FP == fpOf('2') })
	if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.ace.app"); err == nil {
		t.Errorf("app was wanted while its provision was asked: %s", e.Value())
	}
	asked(provisions.Provided, "")
	if w := b.wantNow(t, "ace", "app", "the consumer's want once its provision is provided", func(w delivery.Want) bool { return w.FP == fpOf('3') }); w.FP != fpOf('3') {
		t.Errorf("app's want = %+v", w)
	}
}

// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check
// failed on a node and writes the ask about it in its own asks, which the view and the teller read; its asks bucket is
// the one its manifest declares (ownBuckets, held equal to the manifest by TestOpensWhatItsManifestNames).
func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
	b := startBed(t, "")
	fNew := delivery.SourcePrefix2 + strings.Repeat("c", 64)
	digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
	mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
	b.putJSON(t, delivery.ModulesBucket, "web", mod)
	on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
	b.putJSON(t, delivery.PresenceBucket, "ace", on)
	failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
	b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
	b.place(t, "ace", "web")
	b.start(t)
	hkey, _ := delivery.HeldKey("web", fNew)
	// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
	if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" ||
		!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
		t.Fatalf("held = %+v, %v", h, err)
	}
	akey, _ := delivery.HoldAskKey("web", fNew)
	a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
	if err != nil || a.Asking <= 0 || a.Ask.ID != delivery.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
		t.Fatalf("the ask = %+v, %v", a, err)
	}
}

// conditionsSaid reads what the assigner said of its conditions, by finding key: each a finding in its own bucket, which
// the condition keeper carries (novox/hq ADR 0590 §3, issue 1126), raised while it stands and cleared once deleted, each
// with the words its last finding said and the subject the finding names (`<module>[.<fingerprint>]`).
func (b *bed) conditionsSaid(t *testing.T) map[string]asks.Condition {
	t.Helper()
	w, err := b.buckets[assign.FindingsBucket].WatchAll(b.ctx, jetstream.IncludeHistory())
	if err != nil {
		t.Fatal(err)
	}
	defer w.Stop()
	out := map[string]asks.Condition{}
	for e := range w.Updates() {
		if e == nil {
			return out
		}
		c := out[e.Key()]
		c.Key, c.State = e.Key(), asks.StateCleared
		if e.Operation() == jetstream.KeyValuePut {
			f, err := health.DecodeFinding(e.Value())
			if err != nil {
				t.Fatal(err)
			}
			c = asks.Condition{Key: e.Key(), State: asks.StateRaised, Severity: string(f.Severity), Explanation: f.Explanation,
				Resolved: f.Resolved, Subject: asks.Scope{Scope: string(f.Subject.Scope), ID: f.Subject.ID, Machine: f.Subject.Machine}}
		}
		out[e.Key()] = c
	}
	return out
}

// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the
// canary is held, not set back, and asked about (restore it, or leave it stopped); the condition names the module, the node
// and the check's words.
func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
	b := startBed(t, "")
	fNew := delivery.SourcePrefix2 + strings.Repeat("d", 64)
	digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`)
	mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
	b.putJSON(t, delivery.ModulesBucket, "mail", mod)
	on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
	b.putJSON(t, delivery.PresenceBucket, "ace", on)
	failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
	b.putJSON(t, delivery.HaveBucket, "ace.mail", failed)
	wantKey, _ := delivery.WantKey("ace", "mail")
	want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
	b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
	b.place(t, "ace", "mail")
	b.start(t)

	akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
	a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
	if err != nil || len(a.Ask.Options) != 2 || a.Ask.Options[0].ID != "restore" || a.Ask.Options[1].ID != "leave-stopped" {
		t.Fatalf("the restore ask = %+v, %v", a, err)
	}
	if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wantKey)); err != nil || w.FP != fNew {
		t.Errorf("the data module was set back: want = %+v, %v", w, err)
	}
	key := ""
	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
		for k, c := range b.conditionsSaid(t) {
			if c.State == asks.StateRaised && strings.HasPrefix(c.Subject.ID, "mail.") && strings.HasSuffix(k, ".held") {
				if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
					t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
				}
				key = k
			}
		}
	}
	if key == "" {
		t.Fatal("no condition for the held fingerprint was said")
	}
}

// 1906/D8 (scenario D of acceptance test 18, ADR 0539 §19), whole on the bus: the assigner is stopped while it writes the wants
// of forty modules and another is started; every module's want is written, and each exactly once, by revision: nothing lost
// and nothing written again.
func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
	b := startBed(t, "")
	const modules = 40
	on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
	b.putJSON(t, delivery.PresenceBucket, "ace", on)
	names := make([]string, modules)
	for i := range names {
		name := fmt.Sprintf("mod%02d", i)
		names[i] = name
		fp := delivery.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16)
		digest := b.manifest(t, name, fmt.Sprintf(`{"module":%q,"version":"1"}`, name))
		mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
		b.putJSON(t, delivery.ModulesBucket, name, mod)
		built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
		if err != nil {
			t.Fatal(err)
		}
		bkey, _ := delivery.BuilderBuildKey(name, fp)
		b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
		gkey, _ := delivery.PartGrantKey(name, fp)
		grant, _ := delivery.EncodeGrant(delivery.Grant{})
		b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
		b.compose(t, "ace", 'c')
		b.place(t, "ace", name)
	}
	count := func() int {
		keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
		if err != nil {
			return 0
		}
		return len(keys)
	}
	_, stop := b.startStoppable(t)
	for deadline := time.Now().Add(20 * time.Second); count() == 0 && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
	}
	stop() // replaced, wherever its pass had got to
	b.start(t)
	for deadline := time.Now().Add(30 * time.Second); count() < modules && time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
	}
	time.Sleep(time.Second) // and no pass of the second writes any again
	for _, name := range names {
		key, _ := delivery.WantKey("ace", name)
		history, err := b.buckets[delivery.AssignerWantsBucket].History(b.ctx, key)
		if err != nil {
			t.Errorf("the want of %s was never written: %v", name, err)
			continue
		}
		if len(history) != 1 {
			t.Errorf("the want of %s was written %d times, want once", name, len(history))
		}
	}
}

// fixture is a module web (or the manifest given) at fp on the node ace, which is online: built, granted and assigned.
func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
	t.Helper()
	digest := b.manifest(t, name, manifest)
	mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
	b.putJSON(t, delivery.ModulesBucket, name, mod)
	built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
	bkey, _ := delivery.BuilderBuildKey(name, fp)
	b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
	gkey, _ := delivery.PartGrantKey(name, fp)
	grant, _ := delivery.EncodeGrant(delivery.Grant{})
	b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
	on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
	b.putJSON(t, delivery.PresenceBucket, "ace", on)
	b.place(t, "ace", name)
}

func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
	t.Helper()
	key, _ := delivery.WantKey(node, module)
	v, err := delivery.EncodeWant(w)
	if err != nil {
		t.Fatal(err)
	}
	b.putJSON(t, delivery.AssignerWantsBucket, key, v)
}

// wantNow waits until the want of module on node satisfies ok and answers it.
func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.Want) bool) delivery.Want {
	t.Helper()
	key, _ := delivery.WantKey(node, module)
	var last delivery.Want
	for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, key); err == nil {
			if w, err := delivery.DecodeWant(e.Value()); err == nil {
				last = w
				if ok(w) {
					return w
				}
			}
		}
	}
	t.Fatalf("never saw: %s; the want is %+v", what, last)
	return last
}

// answer says the teller wrote the operator's answer to the ask that stands at key (its warrant names that asking's id).
func (b *bed) answer(t *testing.T, key, option string) {
	t.Helper()
	a, err := delivery.DecodeAssignerAsk(key, b.get(t, delivery.AssignerAsksBucket, key))
	if err != nil {
		t.Fatal(err)
	}
	b.answerTo(t, key, option, a.Ask.ID)
}

// answerTo is answer to the ask with this id (the router's warrant names it).
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
	t.Helper()
	v, err := delivery.EncodeAnswer(delivery.Answer{Option: option, Warrant: askID + "@2026-10-24T12:00:00Z", Words: "the operator"})
	if err != nil {
		t.Fatal(err)
	}
	b.putJSON(t, delivery.AnswersBucket, key, v)
}

func (b *bed) hasKey(bucket, key string) bool {
	_, err := b.buckets[bucket].Get(b.ctx, key)
	return err == nil
}

// 1906/D4 (ADR 0539 §19), whole on the bus: the operator's release of a held fingerprint writes its want again, by the rules,
// and clears the condition.
func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
	b := startBed(t, "")
	fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
	b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
	b.compose(t, "ace", 'b')
	b.compose(t, "ace", 'a')
	healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
	failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
	b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
	b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
	b.start(t)
	w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w delivery.Want) bool { return w.FP == fOld })
	if w.Newest != fNew {
		t.Fatalf("the set-back want = %+v", w)
	}
	hold, _ := delivery.HoldAskKey("web", fNew)
	b.get(t, delivery.AssignerAsksBucket, hold)
	b.answer(t, hold, "release")
	w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
	if w.Composed != "" || !w.Self || w.Rule != 0 {
		t.Errorf("the want = %+v, want F by the rules, the node composing it", w)
	}
	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
		cleared := false
		for k, c := range b.conditionsSaid(t) {
			cleared = cleared || (strings.HasPrefix(k, "module.web.") && c.State == asks.StateCleared && strings.HasSuffix(k, ".held"))
		}
		if cleared {
			return
		}
	}
	t.Error("the condition of the held fingerprint was not cleared")
}

// askNow waits for the backup ask at key to be the one numbered asking and answers it.
func (b *bed) askNow(t *testing.T, key string, asking int) delivery.AssignerAsk {
	t.Helper()
	for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		if e, err := b.buckets[delivery.AssignerAsksBucket].Get(b.ctx, key); err == nil {
			if a, err := delivery.DecodeAssignerAsk(key, e.Value()); err == nil && a.Asking == asking {
				return a
			}
		}
	}
	t.Fatalf("the ask at %s never became asking %d", key, asking)
	return delivery.AssignerAsk{}
}

// 1906/D4 (ADR 0539 §16), whole on the bus: a module held on a node because its backup failed is asked about, and the answer
// is written into the node's want; an answer "again" followed by another failure is asked again, and its answer is another want
// value, so the node retries.
func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
	b := startBed(t, "")
	fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
	b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
	b.compose(t, "ace", 'b')
	b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Self: true}) // a placed node's want names the build alone
	held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
	b.putJSON(t, delivery.HaveBucket, "ace.web", held)
	b.start(t)
	akey, _ := delivery.BackupAskKey("ace", "web", fNew)
	a := b.askNow(t, akey, 0)
	if !strings.Contains(a.Ask.Explanation, "no space left") {
		t.Fatalf("the backup ask = %+v", a)
	}
	// an answer to another ask is no answer to this one
	b.answerTo(t, akey, delivery.BackupSkip, "someone-else")
	time.Sleep(time.Second)
	if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
		t.Fatalf("an answer to another ask was taken: %+v", w)
	}
	b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID)
	w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain })
	if w1.FP != fNew || w1.Composed != "" || !w1.Self {
		t.Errorf("the want = %+v", w1)
	}
	// the node retries and the backup fails again: a new have value, asked again under the next number
	again, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left, again"})
	b.putJSON(t, delivery.HaveBucket, "ace.web", again)
	a2 := b.askNow(t, akey, 2)
	if a2.Ask.ID == a.Ask.ID {
		t.Fatalf("asked again under the same id")
	}
	b.answerTo(t, akey, delivery.BackupAgain, a2.Ask.ID)
	w2 := b.wantNow(t, "ace", "web", "the second answer another want value", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain && w.Words != w1.Words })
	if w2.Words == w1.Words {
		t.Errorf("the second answer left the want as it was: %+v", w2)
	}
}

const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`

// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
// cleared saying why, and the want stays at the fingerprint.
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
	b := startBed(t, "")
	fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
	b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
	b.compose(t, "ace", 'b')
	failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
	b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
	b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
	b.start(t)
	hkey, _ := delivery.HeldKey("web", fNew)
	akey, _ := delivery.HoldAskKey("web", fNew)
	b.get(t, delivery.AssignerWantsBucket, hkey)
	b.get(t, delivery.AssignerAsksBucket, akey)
	healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
	for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
		if b.hasKey(delivery.AssignerWantsBucket, hkey) || b.hasKey(delivery.AssignerAsksBucket, akey) {
			continue
		}
		for k, c := range b.conditionsSaid(t) {
			if strings.HasPrefix(k, "module.web.") && c.State == asks.StateCleared && strings.HasSuffix(k, ".held") {
				if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
					t.Errorf("the want = %+v", w)
				}
				return
			}
		}
	}
	t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(delivery.AssignerWantsBucket, hkey), b.hasKey(delivery.AssignerAsksBucket, akey))
}

// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4).
func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
	b := startBed(t, "")
	fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
	b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
	b.compose(t, "ace", 'b')
	failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
	healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
	b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
	_, stop := b.startStoppable(t)
	hkey, _ := delivery.HeldKey("web", fNew)
	akey, _ := delivery.HoldAskKey("web", fNew)
	first, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
	if err != nil {
		t.Fatal(err)
	}
	b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
	for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
		if time.Now().After(deadline) {
			t.Fatal("the hold was not lifted")
		}
	}
	stop()
	time.Sleep(1100 * time.Millisecond)       // the next hold is written in another second
	b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
	b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
	b.start(t)
	again, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
	if err != nil {
		t.Fatal(err)
	}
	if again.Ask.ID == first.Ask.ID {
		t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
	}
	b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
	for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
		if time.Now().After(deadline) {
			t.Fatal("the stale \"keep\" of the first hold kept the second one held")
		}
	}
}

// bedStore is the runtime's states over the bed's buckets, as the seat's verbs read and write them: the assigner's own by
// their local names, another module's `<module>.<state>`.
type bedStore struct{ b *bed }

func (s bedStore) kv(state string) jetstream.KeyValue {
	if strings.Contains(state, ".") {
		return s.b.buckets[strings.Replace(state, ".", "_", 1)]
	}
	return s.b.buckets[delivery.AssignerModule+"_"+state]
}

func (s bedStore) get(state, key string) ([]byte, uint64, bool, error) {
	e, err := s.kv(state).Get(s.b.ctx, key)
	if err != nil {
		return nil, 0, false, nil
	}
	return e.Value(), e.Revision(), true, nil
}

func (s bedStore) put(state, key string, value []byte) error {
	_, err := s.kv(state).Put(s.b.ctx, key, value)
	return err
}

func (s bedStore) keys(state string) ([]string, error) {
	keys, err := s.kv(state).Keys(s.b.ctx)
	if err != nil && strings.Contains(err.Error(), "no keys") {
		return nil, nil
	}
	return keys, err
}

func (s bedStore) del(state, key string) error { return s.kv(state).Delete(s.b.ctx, key) }

func (s bedStore) history(string, string) ([]stdio.StateRevision, error) { return nil, nil }

// 1126 (ADR 0567 §2-§6), the seat mesh-placement whole on the bus: a proposal moving web from the anchor to the laptop is
// proposed through the verb, the assigner finds both nodes must answer, they answer the same in their haves, the
// operator is asked and approves, and each key is written at a new revision while placement-changed is heard on the
// assigner's stream naming both nodes. A second proposal laid over the old revision of a key moved since is laid again,
// and declined, writes nothing and is withdrawn.
func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
	b := startBed(t, "")
	if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: "MOD_" + delivery.AssignerModule + "_placement",
		Subjects: []string{process.ModuleStreamSubject(delivery.AssignerModule, ">")}}); err != nil {
		t.Fatal(err)
	}
	heard := make(chan delivery.PlacementEvent, 16)
	nc, err := nats.Connect(b.srv.ClientURL())
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(nc.Close)
	if _, err := nc.Subscribe(process.ModuleStreamSubject(delivery.AssignerModule, delivery.PlacementChangedEvent), func(m *nats.Msg) {
		var e delivery.PlacementEvent
		if json.Unmarshal(m.Data, &e) == nil {
			heard <- e
		}
	}); err != nil {
		t.Fatal(err)
	}
	for _, node := range []string{"anchor", "laptop"} {
		rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}})
		b.putJSON(t, delivery.HaveBucket, node+".node", rec)
	}
	b.putJSON(t, delivery.ModulesBucket, "web", []byte(`{"v":1}`))
	b.putJSON(t, delivery.ModulesBucket, "shell", []byte(`{"v":1}`))
	// place writes a node's placement as another writer would, in its node and module keys (research 072 row 37), and
	// answers its node key's revision; legacy writes it in the form before, placement.<node>, alone.
	place := func(node string, legacy bool, modules ...string) uint64 {
		p := delivery.Placement{Node: node, Modules: []delivery.PlacedModule{}}
		for _, m := range modules {
			p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
		}
		values, _ := delivery.PlacementValues(p)
		key, _ := delivery.PlacementNodeKey(node)
		if legacy {
			key, _ = delivery.LegacyPlacementKey(node)
			values = map[string][]byte{key: values[key]}
		}
		for _, k := range slices.Sorted(maps.Keys(values)) {
			if k == key {
				continue
			}
			if _, err := b.buckets[delivery.PlacementBucket].Put(b.ctx, k, values[k]); err != nil {
				t.Fatal(err)
			}
		}
		rev, err := b.buckets[delivery.PlacementBucket].Put(b.ctx, key, values[key])
		if err != nil {
			t.Fatal(err)
		}
		return rev
	}
	place("anchor", true, "web")
	laptopRev := place("laptop", false, "shell")
	bound := judge.AnswerBound
	t.Cleanup(func() { judge.AnswerBound = bound })
	judge.AnswerBound = 2 * time.Second
	saysWhatItKeeps(t, b, b.start(t))
	// The holder converts anchor, held in the form before alone, to its node and module keys once at its start.
	var anchorRev uint64
	for deadline := time.Now().Add(15 * time.Second); anchorRev == 0; time.Sleep(50 * time.Millisecond) {
		if e, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "anchor.node"); err == nil {
			anchorRev = e.Revision()
			if _, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "anchor.web"); err != nil {
				t.Fatalf("anchor converted without its module key: %v", err)
			}
		}
		if time.Now().After(deadline) {
			t.Fatal("anchor in the form before was not converted")
		}
	}
	verbs := newSeatTools(bedStore{b}, func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"ok":true}`), nil })
	propose := func() delivery.PlacementProposal {
		t.Helper()
		got, err := verbs.doPropose(map[string]any{"why": "move web", "changes": `[{"op":"take-off","node":"anchor","module":"web"},` +
			`{"op":"place","node":"laptop","module":"web"}]`}, "operator@laptop")
		if err != nil {
			t.Fatal(err)
		}
		return got.(proposed).Proposal
	}
	proposalNow := func(id, what string, ok func(delivery.PlacementProposal) bool) delivery.PlacementProposal {
		t.Helper()
		var last delivery.PlacementProposal
		for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
			if e, err := b.buckets[delivery.PlacementProposalsBucket].Get(b.ctx, id); err == nil {
				if p, err := delivery.DecodePlacementProposal(e.Value()); err == nil {
					if last = p; ok(p) {
						return p
					}
				}
			}
		}
		t.Fatalf("the proposal %s never was %s: %+v", id, what, last)
		return last
	}
	answerAll := func(p delivery.PlacementProposal) {
		t.Helper()
		for _, node := range p.Must {
			key, _ := delivery.PlacementCandidateKey(p.ID, node)
			e, err := b.buckets[delivery.PlacementCandidatesBucket].Get(b.ctx, key)
			if err != nil {
				t.Fatalf("no candidate %s: %v", key, err)
			}
			have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}, Candidates: map[string]delivery.CandidateAnswer{
				p.ID: {Proposal: p.ID, Revision: e.Revision(), Verdict: delivery.CandidateSame, Same: []string{"node"}}}})
			b.putJSON(t, delivery.HaveBucket, node+".node", have)
		}
	}

	p := propose()
	p = proposalNow(p.ID, "told who must answer", func(p delivery.PlacementProposal) bool { return len(p.Must) > 0 })
	if !slices.Equal(p.Must, []string{"anchor", "laptop"}) {
		t.Fatalf("must answer = %v, want both nodes", p.Must)
	}
	answerAll(p)
	p = proposalNow(p.ID, "asked", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
	akey, _ := delivery.PlacementAskKey(p.ID)
	b.answer(t, akey, assign.OptionYes)
	select {
	case e := <-heard:
		if e.Proposal.ID != p.ID || len(e.Nodes) != 2 || e.Nodes[0].Before != anchorRev || e.Nodes[1].Before != laptopRev ||
			e.Nodes[0].After <= anchorRev || e.Warrant == nil || !slices.Equal(e.Nodes[0].Removed, []string{"web"}) {
			t.Errorf("placement-changed = %+v", e)
		}
	case <-time.After(15 * time.Second):
		t.Fatal("no placement-changed heard")
	}
	for node, want := range map[string][]string{"anchor": {}, "laptop": {"shell", "web"}} {
		// The node key names the modules, each module key carries its own, and the mirror of the form before follows.
		values := map[string][]byte{}
		for _, key := range []string{node + ".node", "placement." + node, node + ".web", node + ".shell"} {
			if e, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, key); err == nil {
				values[key] = e.Value()
			}
		}
		got, err := delivery.PlacementsOf(values)
		if err != nil || !slices.Equal(got[node].Assigned(), want) {
			t.Errorf("%s = %v, %v, want %v", node, got[node].Assigned(), err, want)
		}
		mirror, _ := delivery.DecodePlacement(values["placement."+node])
		if !slices.Equal(mirror.Assigned(), want) {
			t.Errorf("the mirror placement.%s = %v, want %v", node, mirror.Assigned(), want)
		}
	}
	if b.hasKey(delivery.PlacementProposalsBucket, p.ID) {
		t.Error("an applied proposal stays in the state")
	}

	// A proposal whose key moves before its warrant is laid again; then declined, it writes nothing and ends.
	q, err := verbs.doPropose(map[string]any{"why": "back", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
	if err != nil {
		t.Fatal(err)
	}
	moved := q.(proposed).Proposal
	place("laptop", false, "shell", "web") // another writer moved the key
	relaid := proposalNow(moved.ID, "laid again", func(p delivery.PlacementProposal) bool {
		return strings.Contains(p.Words, "laid again") && len(p.Must) > 0
	})
	answerAll(relaid)
	proposalNow(moved.ID, "asked", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
	akey, _ = delivery.PlacementAskKey(moved.ID)
	b.answer(t, akey, assign.OptionNo)
	for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.PlacementProposalsBucket, moved.ID); time.Sleep(50 * time.Millisecond) {
		if time.Now().After(deadline) {
			t.Fatal("a declined proposal was not ended")
		}
	}
	key, _ := delivery.PlacementNodeKey("laptop")
	if e, _ := b.buckets[delivery.PlacementBucket].Get(b.ctx, key); e == nil || !strings.Contains(string(e.Value()), "web") {
		t.Error("a declined proposal wrote its key")
	}

	// ADR 0593 §8: a node that does not answer is named as giving no answer once its pulse key's time to live ends, heard on
	// the bus with no timer of the assigner's, and the proposal is decided on the answers it has: asked, not failed.
	q, err = verbs.doPropose(map[string]any{"why": "again", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
	if err != nil {
		t.Fatal(err)
	}
	silent := q.(proposed).Proposal
	decided := proposalNow(silent.ID, "decided without its silent nodes", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
	if decided.Verdict == nil || decided.Verdict.State != delivery.VerdictPass ||
		!strings.Contains(strings.Join(decided.Verdict.Notes, "; "), "no answer from") || !strings.Contains(strings.Join(decided.Verdict.Notes, "; "), "laptop (offline)") {
		t.Errorf("the silent node's verdict = %+v", decided.Verdict)
	}

	// 774/S59 (ADR 0587 §7): a joining node is written on the enrolment's warrant with no node to answer and no ask raised,
	// and placement-changed carries that warrant. The assigner reads no lease (ADR 0567 §8): the enrolment's own issue waits
	// for the lease write (ADR 0587 §4).
	for len(heard) > 0 {
		<-heard
	}
	joined, err := verbs.doPropose(map[string]any{"why": "enrolment of ace", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`,
		"changes": `[{"op":"add-node","node":"ace","decided":{"address":"10.9.0.2"},"keys":{"overlay":"dHVubmVs"}}]`}, "novox.mesh-enrolment")
	if err != nil {
		t.Fatal(err)
	}
	select {
	case e := <-heard:
		if e.Proposal.ID != joined.(proposed).Proposal.ID || e.Warrant == nil || e.Warrant.Warrant != "enrol-ace-1@2026-10-28T20:00:00Z" || len(e.Nodes) != 1 || e.Nodes[0].Node != "ace" {
			t.Errorf("placement-changed for the joining node = %+v", e)
		}
	case <-time.After(15 * time.Second):
		t.Fatal("no placement-changed heard for the joining node")
	}
	if ace, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "ace.node"); err != nil || !strings.Contains(string(ace.Value()), "10.9.0.2") {
		t.Errorf("the joining node's key = %v, %v", ace, err)
	}
	if akey, _ := delivery.PlacementAskKey(joined.(proposed).Proposal.ID); b.hasKey(delivery.AssignerAsksBucket, akey) || b.hasKey(delivery.PlacementProposalsBucket, joined.(proposed).Proposal.ID) {
		t.Error("an ask was raised for the joining node, or its proposal stays open")
	}
	// Its enrolment ends before it confirms: a remove-node on the same warrant deletes its key at once, with no ask, and
	// placement-changed carries the warrant (the operator's decision of 2026-11-02 01:05).
	for len(heard) > 0 {
		<-heard
	}
	if _, err := verbs.doPropose(map[string]any{"why": "enrolment of ace revoked", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`,
		"changes": `[{"op":"remove-node","node":"ace"}]`}, "novox.mesh-enrolment"); err != nil {
		t.Fatal(err)
	}
	select {
	case e := <-heard:
		if e.Warrant == nil || len(e.Nodes) != 1 || e.Nodes[0].Node != "ace" {
			t.Errorf("placement-changed for the removed node = %+v", e)
		}
	case <-time.After(15 * time.Second):
		t.Fatal("no placement-changed heard for the removed node")
	}
	if _, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "ace.node"); err == nil {
		t.Error("the removed node's key stays")
	}
}