mesh-delivery/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go

//go:build integration

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"slices"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/nats-io/nats-server/v2/server"
	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"

	gobuild "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
	"git.novox.be/novox/mesh-sdk/go/announce"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/process"

	"mesh-delivery-checker/internal/check"
)

// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
// controller's user and the checker's own principal <node>.<module>. It carries no per-principal permissions: the grants
// the controller composes from the manifest's state and reads (only the checker writes its buckets) are the controller's,
// and are proved by its own module check, not here.
const busConf = `
listen: 127.0.0.1:-1
jetstream { store_dir: %q }
accounts {
  MESH {
    jetstream: enabled
    users = [
      { user: "ctl", password: "pw" }
      { user: "n1.mesh-delivery-checker", password: "pw" }
    ]
  }
}
`

const (
	repo   = "novox/mesh-delivery"
	head1  = "1111111111111111111111111111111111111111"
	head2  = "2222222222222222222222222222222222222222"
	trunk  = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
	merged = "fedcba9876543210fedcba9876543210fedcba98" // the merge commit of the captured fetch answer
	digU   = "aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11"
	digI   = "bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22"
)

var captured = filepath.Join("..", "..", "internal", "check", "testdata", "captured")

func readCaptured(t *testing.T, name string) map[string]any {
	t.Helper()
	raw, err := os.ReadFile(filepath.Join(captured, name+".json"))
	if err != nil {
		t.Fatal(err)
	}
	var v map[string]any
	if err := json.Unmarshal(raw, &v); err != nil {
		t.Fatal(err)
	}
	return v
}

// fakeBuildModules serve the tools of mesh-build-go and mesh-build-node as the real ones do (ADR 0469 clause 2): a call on
// mesh.mod.<module>.call.<tool>.<node>.<caller> answers at once with the step and its key, and the step's end is written to
// the key <node>.<id> of the module's steps bucket. What the steps end with are the results the build modules' own code
// marshalled (internal/check/testdata/captured, README).
type fakeBuildModules struct {
	js       jetstream.JetStream
	ctx      context.Context
	conflict atomic.Bool
	// hold is a commit whose fetch never answers: its ask stays taken, as one lost in a bus reload (issue 734).
	hold     atomic.Value
	mu       sync.Mutex
	calls    map[string]int // by `<module>.<tool>`
	testTool *testing.T
	// beside is, by workspace, the siblings the fetch placed in it as `<owner/repo> <commit>`, sorted: a digest echoes them
	// (1929/AC2; mesh-build-go buildstep.besideOf), or the level runs whole and keeps nothing.
	beside map[string][]string
	// names are the nodes' names the last check ask carried, for the names check (ADR 0567 §12).
	names []string
}

func (f *fakeBuildModules) count(tool string) int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.calls[tool]
}

func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any) json.RawMessage {
	level, _ := args["level"].(string)
	var v any
	switch {
	case strings.HasSuffix(tool, "_fetch"):
		if h, _ := f.hold.Load().(string); h != "" && args["commit"] == h {
			<-f.ctx.Done()
		}
		name := "go-fetch-clean"
		if f.conflict.Load() {
			name = "go-fetch-conflict"
		}
		fetched := readCaptured(t, name)
		fetched["commit"], fetched["base"] = args["commit"], args["base"]
		var refs []struct {
			Repository string `json:"repository"`
			Commit     string `json:"commit"`
		}
		raw, _ := json.Marshal(args["beside"])
		_ = json.Unmarshal(raw, &refs)
		var lines []string
		for _, r := range refs {
			lines = append(lines, r.Repository+" "+r.Commit)
		}
		sort.Strings(lines)
		f.mu.Lock()
		if f.beside == nil {
			f.beside = map[string][]string{}
		}
		f.beside[fmt.Sprint(fetched["workspace"])] = lines
		f.mu.Unlock()
		v = fetched
	case strings.HasSuffix(tool, "_digest"):
		d := map[string]string{"unit": digU, "integration": digI}[level]
		f.mu.Lock()
		beside := f.beside[fmt.Sprint(args["workspace"])]
		f.mu.Unlock()
		v = map[string]any{"level": level, "packages": map[string]string{"fixture/slow": d}, "beside": beside}
	case strings.HasSuffix(tool, "_test"):
		r := readCaptured(t, "go-test-slow")
		r["level"] = level
		v = r
	}
	data, _ := json.Marshal(v)
	return data
}

// serve makes module's steps bucket and its build seat's stream and workers, and pulls its asks on n1 as the real module
// does since R4g (ADR 0525; issue 1887 A5): each ask's job runs with step bodies answering the captured results, and a
// check takes the kept verdicts of the checker's kept bucket by their digests.
func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.KeyValue) {
	kv, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
	if err != nil {
		t.Fatal(err)
	}
	language := strings.TrimPrefix(module, "mesh-build-")
	if err := gobuild.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
		t.Fatal(err)
	}
	prefix := map[string]string{"mesh-build-go": "build_go_", "mesh-build-node": "build_node_"}[module]
	call := func(name string) gobuild.StepFunc {
		return func(ctx context.Context, args map[string]any) (any, error) {
			tool := prefix + name
			f.mu.Lock()
			f.calls[module+"."+tool]++
			f.mu.Unlock()
			return f.result(t, tool, args), nil
		}
	}
	steps := gobuild.AskSteps{Fetch: call("fetch"), Test: call("test"),
		Kept: func(ctx context.Context, level, digest string) (int, bool, error) {
			key, err := delivery.KeptKey(level, digest)
			if err != nil {
				return 0, false, err
			}
			e, err := kept.Get(ctx, key)
			if errors.Is(err, jetstream.ErrKeyNotFound) {
				return 0, false, nil
			}
			if err != nil {
				return 0, false, err
			}
			v, err := delivery.DecodeKept(e.Value())
			return v.Tests, err == nil, err
		}}
	if module == "mesh-build-go" {
		steps.Digest = call("digest")
	}
	put := func(key string, value any) (uint64, error) {
		raw, _ := value.(json.RawMessage)
		return kv.Put(f.ctx, key, raw)
	}
	p := &gobuild.Puller{Source: &gobuild.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: "n1", Put: put,
		Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
		Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
			if a.Check != nil {
				f.mu.Lock()
				f.names = a.Check.NodeNames
				f.mu.Unlock()
			}
			return gobuild.RunAsk(ctx, steps, a)
		}}
	go func() { _ = p.Pull(f.ctx) }()
}

// serveForge answers the forge's two reads as its tools do, with the answers the forge gave (internal/check/testdata/captured):
// the go.mod of the captured file, whose mesh-sdk pin is cb54756c3865, and gitea_compare for the pin: on the trunk, unless the
// pin is the one the test set as stale.
func (b *bed) serveForge(t *testing.T, nc *nats.Conn) {
	reply := func(m *nats.Msg, name string) {
		var result map[string]any
		_ = json.Unmarshal(mustRead(t, name), &result)
		answer, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": "n0"})
		_ = m.Respond(answer)
	}
	for _, tool := range []string{"gitea_get_file", "gitea_compare"} {
		tool := tool
		sub, err := nc.Subscribe("mesh.mod.gitea.tool."+tool, func(m *nats.Msg) {
			var a map[string]string
			_ = json.Unmarshal(m.Data, &a)
			switch {
			case tool == "gitea_get_file" && b.stalePin.Load():
				content := strings.Replace(string(mustRead(t, "forge-get-file-go-mod")), "cb54756c3865", "9aaa5c80ac5c", 1)
				var result map[string]any
				_ = json.Unmarshal([]byte(content), &result)
				answer, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": "n0"})
				_ = m.Respond(answer)
			case tool == "gitea_get_file":
				reply(m, "forge-get-file-go-mod")
			case strings.HasPrefix(a["head"], "9aaa5c80ac5c"):
				reply(m, "forge-compare-branch")
			default:
				reply(m, "forge-compare-trunk")
			}
		})
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(func() { _ = sub.Unsubscribe() })
	}
}

func mustRead(t *testing.T, name string) []byte {
	t.Helper()
	raw, err := os.ReadFile(filepath.Join(captured, name+".json"))
	if err != nil {
		t.Fatal(err)
	}
	return raw
}

type bed struct {
	stalePin atomic.Bool
	ctx      context.Context
	js       jetstream.JetStream
	buckets  map[string]jetstream.KeyValue
	tools    map[string]jetstream.KeyValue
	fake     *fakeBuildModules
	srvURL   string
	announce []string
	mu       sync.Mutex
}

// engine places each node and serves compose-candidate for the first one, as its node-engine does on the bus (answered on
// the seat verb's caller-named and plain subjects); the others never answer. It answers what it was asked.
func (b *bed) engine(t *testing.T, nodes ...string) func() []delivery.ComposeCandidateAsk {
	t.Helper()
	kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: 1})
	if err != nil {
		t.Fatal(err)
	}
	for _, n := range nodes {
		pk, _ := delivery.PlacementNodeKey(n)
		raw, _ := json.Marshal(delivery.Placement{Node: n})
		if _, err := kv.Put(b.ctx, pk, raw); err != nil {
			t.Fatal(err)
		}
	}
	nc, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(nc.Close)
	var mu sync.Mutex
	var asked []delivery.ComposeCandidateAsk
	answer := func(m *nats.Msg) {
		ask, err := delivery.DecodeComposeCandidateAsk(m.Data)
		if err != nil {
			_ = m.Respond([]byte(`{"error":"` + err.Error() + `"}`))
			return
		}
		mu.Lock()
		asked = append(asked, ask)
		mu.Unlock()
		a := delivery.ComposeCandidateAnswer{Node: nodes[0], Candidate: ask.Candidate, Verdict: delivery.ComposeCandidateComposed,
			Own: delivery.ModuleComposed{Module: "node", State: delivery.ModuleComposedState, Digest: "sha256:" + strings.Repeat("b", 64)}}
		raw, err := delivery.EncodeComposeCandidateAnswer(a)
		if err != nil {
			t.Error(err)
		}
		out, _ := json.Marshal(map[string]any{"result": json.RawMessage(raw), "node": nodes[0]})
		_ = m.Respond(out)
	}
	for _, subject := range []string{"mesh.seat.node-engine.call.compose-candidate." + nodes[0] + ".>", "mesh.seat.node-engine.tool.compose-candidate." + nodes[0]} {
		if _, err := nc.Subscribe(subject, answer); err != nil {
			t.Fatal(err)
		}
	}
	return func() []delivery.ComposeCandidateAsk {
		mu.Lock()
		defer mu.Unlock()
		return slices.Clone(asked)
	}
}

// startBed starts the bus, the buckets the controller makes from the checker's and the other parts' manifests, the build
// modules' streams and the checker's durable consumers on them, one node's toolchains, the gate's source naming the trunk, and
// the fake build modules. It runs no checker.
func startBed(t *testing.T) *bed {
	dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "c")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { os.RemoveAll(dir) })
	conf := filepath.Join(dir, "bus.conf")
	if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
		t.Fatal(err)
	}
	opts, err := server.ProcessConfigFile(conf)
	if err != nil {
		t.Fatal(err)
	}
	opts.NoLog, opts.NoSigs = true, true
	srv, err := server.NewServer(opts)
	if err != nil {
		t.Fatal(err)
	}
	go srv.Start()
	if !srv.ReadyForConnections(10 * time.Second) {
		t.Fatal("the bus did not come up")
	}
	t.Cleanup(srv.Shutdown)
	ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(ctl.Close)
	js, err := jetstream.New(ctl)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
	t.Cleanup(cancel)
	b := &bed{ctx: ctx, js: js, buckets: map[string]jetstream.KeyValue{}, tools: map[string]jetstream.KeyValue{}, srvURL: srv.ClientURL()}
	for module, stream := range toolchainStreams {
		if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.stream." + module + ".event.toolchain.reported"}}); err != nil {
			t.Fatal(err)
		}
		if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-checker", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
			t.Fatal(err)
		}
		kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket(module), History: 1})
		if err != nil {
			t.Fatal(err)
		}
		b.tools[module] = kv
	}
	// What the controller keeps of the messages a consumer gave up on (issue 734).
	if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildrun.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}}); err != nil {
		t.Fatal(err)
	}
	// The checker's stream of events, which the indexer and the forge bridge hear.
	if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
		t.Fatal(err)
	}
	for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
		delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
		delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
		delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1,
		delivery.TurnsBucket: delivery.TurnsHistory} {
		kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
		if err != nil {
			t.Fatal(err)
		}
		b.buckets[name] = kv
	}
	// 1929/AC1: the indexer's value of the repository's module, with no TypeScript manifest, so the Go build module alone
	// judges the repository (an unknown repository is judged by both, issue 1887 A12).
	known, err := delivery.EncodeModule(delivery.Module{Repository: repo, Commit: trunk, Path: "modules/m", None2: "the fingerprint is computed with its chain"})
	if err != nil {
		t.Fatal(err)
	}
	if _, err := b.buckets[delivery.ModulesBucket].Put(ctx, "m", known); err != nil {
		t.Fatal(err)
	}
	b.setOnline(t, "n1", true)
	b.setNode(t, "n1")
	for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
		b.setHealth(t, "n1", module)
	}
	b.serveForge(t, ctl)
	b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
	b.fake.serve(t, "mesh-build-go", b.buckets[delivery.CheckerKeptBucket])
	b.fake.serve(t, "mesh-build-node", b.buckets[delivery.CheckerKeptBucket])
	b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
	b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
	b.setTrunk(t, map[string]string{repo: trunk})
	return b
}

func (b *bed) setTrunk(t *testing.T, heads map[string]string) {
	t.Helper()
	data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: heads})
	if err != nil {
		t.Fatal(err)
	}
	if _, err := b.buckets[delivery.SourceBucket].Put(b.ctx, delivery.SourceMainKey, data); err != nil {
		t.Fatal(err)
	}
}

func (b *bed) setOnline(t *testing.T, node string, online bool) {
	t.Helper()
	data, _ := delivery.EncodePresence(delivery.Presence{Online: online})
	if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, node, data); err != nil {
		t.Fatal(err)
	}
}

// setNode writes the node's record in the node-engine's have, as the node-engine does, so the checker names it in each
// check ask (ADR 0567 §12); with no record the checker asks no check.
func (b *bed) setNode(t *testing.T, node string) {
	t.Helper()
	data, err := delivery.EncodeNodeRecord(delivery.NodeRecord{V: 1, Node: node})
	if err != nil {
		t.Fatal(err)
	}
	key, _ := delivery.HaveNodeKey(node)
	if _, err := b.buckets[delivery.HaveBucket].Put(b.ctx, key, data); err != nil {
		t.Fatal(err)
	}
}

func (b *bed) setHealth(t *testing.T, node, module string) {
	t.Helper()
	data, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	if err != nil {
		t.Fatal(err)
	}
	key, _ := delivery.HaveKey(node, module)
	if _, err := b.buckets[delivery.HaveBucket].Put(b.ctx, key, data); err != nil {
		t.Fatal(err)
	}
}

func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
	t.Helper()
	data, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now().UTC()})
	if err != nil {
		t.Fatal(err)
	}
	key, _ := delivery.ToolchainKey(node, language)
	if _, err := b.tools[module].Put(b.ctx, key, data); err != nil {
		t.Fatal(err)
	}
	if _, err := b.js.Publish(b.ctx, "mesh.stream."+module+".event.toolchain.reported", []byte(`{}`)); err != nil {
		t.Fatal(err)
	}
}

// candidate writes candidates.<group> as the indexer does (the SDK's EncodeCandidate is the indexer's own encoder).
func (b *bed) candidate(t *testing.T, group string, pulls ...delivery.CandidatePull) string {
	t.Helper()
	heads := make([]delivery.Head, len(pulls))
	for i, p := range pulls {
		heads[i] = p.Head
	}
	hash, err := delivery.HeadsHash(heads)
	if err != nil {
		t.Fatal(err)
	}
	data, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: group, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
	if err != nil {
		t.Fatal(err)
	}
	key, _ := delivery.CandidatesKey(group)
	if _, err := b.buckets[delivery.CandidatesBucket].Put(b.ctx, key, data); err != nil {
		t.Fatal(err)
	}
	vk, _ := delivery.CheckerVerdictKey(group, hash)
	return vk
}

func pull(repository string, number int, head string) delivery.CandidatePull {
	return delivery.CandidatePull{Head: delivery.Head{Repository: repository, Number: number, Head: head}, Modules: []string{}}
}

// start runs the program as the checker of a node.
func (b *bed) start(t *testing.T, env map[string]string) {
	t.Helper()
	broker := filepath.Join(t.TempDir(), "broker")
	cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: "n1", Module: "mesh-delivery-checker", User: "n1.mesh-delivery-checker", Password: "pw"})
	if err := os.WriteFile(broker, cred, 0o600); err != nil {
		t.Fatal(err)
	}
	// The settle window of a group's heads (check.DefaultSettle) is cut short: each bed's candidate is written whole.
	e := map[string]string{"MESH_BROKER_FILE": broker, "MESH_CHECKER_SETTLE": "10ms"}
	for k, v := range env {
		e[k] = v
	}
	runCtx, stop := context.WithCancel(b.ctx)
	done := make(chan error, 1)
	go func() { done <- run(runCtx, func(k string) string { return e[k] }, io.Discard) }()
	t.Cleanup(func() { stop(); <-done })
}

// verdict reads the verdict key until pred holds of it, within 20 s.
func (b *bed) verdict(t *testing.T, key string, pred func(delivery.Verdict) bool) delivery.Verdict {
	t.Helper()
	var last delivery.Verdict
	for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		e, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, key)
		if err != nil {
			continue
		}
		if last, err = delivery.DecodeVerdict(e.Value()); err == nil && pred(last) {
			return last
		}
	}
	t.Fatalf("verdicts.%s = %+v within 20 s", key, last)
	return last
}

func ended(v delivery.Verdict) bool { return v.State != delivery.VerdictRunning }

// owed reads the status the checker owes on a head.
func (b *bed) owed(t *testing.T, number int, head, state string) delivery.Owed {
	t.Helper()
	key, err := delivery.CheckerOwedKey(repo, number, head)
	if err != nil {
		t.Fatal(err)
	}
	var last delivery.Owed
	for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		e, err := b.buckets[delivery.CheckerOwedBucket].Get(b.ctx, key)
		if err != nil {
			continue
		}
		if last, err = delivery.DecodeCheckerOwed(e.Value()); err == nil && last.State == state {
			return last
		}
	}
	t.Fatalf("owed.%s = %+v within 20 s, want the state %s", key, last, state)
	return last
}

// TestCheckerJudgesACandidateAndOwesItsStatus: a candidate the indexer wrote, one pull request of mesh-delivery, is fetched
// merged onto the trunk the gate names, digested and tested at both levels on the node, kept, previewed; the verdict passes
// with the counts of both levels and the composition not run, and the status mesh/check is owed on the head as success.
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1, 1929/AC1
	b := startBed(t)
	key := b.candidate(t, "feat/x", pull(repo, 7, head1))
	// ADR 0593 §7, §9 (D12): n1 and n2 are placed; n1's engine answers compose-candidate on the bus, n2's never does.
	asked := b.engine(t, "n1", "n2")
	b.start(t, nil)
	v := b.verdict(t, key, ended)
	if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 2 || v.Levels[delivery.LevelIntegration].Run != 2 || v.Trunk[repo] != trunk {
		t.Fatalf("verdict = %+v", v)
	}
	if c := v.Composition; c == nil || c.State != delivery.CompositionPass || !slices.Equal(c.Machines, []string{"n1"}) ||
		!strings.Contains(c.Words, "n2: "+delivery.ComposeNoAnswer) || len(v.Machines) != 1 || v.Machines[0] != "n1" {
		t.Errorf("composition %+v machines %v", v.Composition, v.Machines)
	}
	if got := asked(); len(got) == 0 || got[0].Candidate != key {
		t.Errorf("n1's engine was asked %+v, want the check's candidate %s", got, key)
	}
	b.fake.mu.Lock()
	names := b.fake.names
	b.fake.mu.Unlock()
	if len(names) != 1 || names[0] != "n1" {
		t.Errorf("the check ask named the nodes %v, want [n1] from the have's node record", names)
	}
	o := b.owed(t, 7, head1, "success")
	if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 2 run") {
		t.Errorf("owed = %+v", o)
	}
	for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_digest": 2, "mesh-build-go.build_go_test": 2} {
		if got := b.fake.count(tool); got != want {
			t.Errorf("%s called %d times, want %d", tool, got, want)
		}
	}
	// What passed is kept once under each level's digest.
	for _, k := range []string{"unit." + digU, "integration." + digI} {
		if _, err := b.buckets[delivery.CheckerKeptBucket].Get(b.ctx, k); err != nil {
			t.Errorf("kept.%s: %v", k, err)
		}
	}
	// The preview of the head merged onto the trunk, with the tree and the manifest digest the fetch named.
	pk, _ := delivery.PreviewKey(repo, head1, trunk)
	e, err := b.buckets[delivery.PreviewsBucket].Get(b.ctx, pk)
	if err != nil {
		t.Fatalf("previews.%s: %v", pk, err)
	}
	pv, err := delivery.DecodePreview(e.Value())
	if err != nil || pv.Merge != merged || pv.Modules["modules/m"].Tree != "1111111111111111111111111111111111111111" {
		t.Errorf("preview = %+v, %v", pv, err)
	}
}

// TestACheckOrACarryOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every
// package taken from the first's kept verdicts: no test step runs and no second kept key appears. Then, through the real bus
// (novox/hq ADR 0548 §3, §5): a third candidate is not claimed while the gate's merge turn on its repository stands, and is
// claimed when the turn's key is deleted; and a move of the trunk carries the first verdict on the recheck stream by its
// unchanged digests, running no test, the verdict naming the trunk it was carried from.
func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2, ADR 0548 §3, §5
	b := startBed(t)
	first := b.candidate(t, "feat/x", pull(repo, 7, head1))
	b.start(t, nil)
	b.verdict(t, first, ended)
	tests, kept := b.fake.count("mesh-build-go.build_go_test"), keptCount(t, b)
	second := b.candidate(t, "feat/y", pull(repo, 8, head2))
	v := b.verdict(t, second, ended)
	if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 0 || v.Levels[delivery.LevelUnit].Kept != 2 || v.Levels[delivery.LevelIntegration].Run != 0 {
		t.Fatalf("verdict = %+v, want everything taken from the kept verdicts", v)
	}
	if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
		t.Errorf("build_go_test called %d times, was %d: a test ran for equal inputs", got, tests)
	}
	if got := keptCount(t, b); got != kept {
		t.Errorf("%d kept keys, was %d", got, kept)
	}

	// §3: the gate's turn on the repository holds a new check off; its delete claims it.
	at := time.Now().UTC()
	turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: "fix/other", Heads: "0123456789ab", Approved: at, Since: at}}})
	if err != nil {
		t.Fatal(err)
	}
	tk, _ := delivery.TurnsKey(repo)
	if _, err := b.buckets[delivery.TurnsBucket].Put(b.ctx, tk, turn); err != nil {
		t.Fatal(err)
	}
	time.Sleep(300 * time.Millisecond) // the checker hears the turn
	third := b.candidate(t, "feat/z", pull(repo, 9, "3333333333333333333333333333333333333333"))
	time.Sleep(time.Second)
	if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, third); !errors.Is(err, jetstream.ErrKeyNotFound) {
		t.Fatalf("verdicts.%s exists (%v) while a merge turn holds its repository", third, err)
	}
	if err := b.buckets[delivery.TurnsBucket].Delete(b.ctx, tk); err != nil {
		t.Fatal(err)
	}
	b.verdict(t, third, ended)

	// §5: the trunk moves; the first verdict is carried by its digests, nothing runs.
	tests = b.fake.count("mesh-build-go.build_go_test")
	const moved = "abababababababababababababababababababab"
	b.setTrunk(t, map[string]string{repo: moved})
	c := b.verdict(t, first, func(v delivery.Verdict) bool { return ended(v) && v.Trunk[repo] == moved })
	if c.State != delivery.VerdictPass || c.CarriedFrom[repo] != trunk || c.Levels[delivery.LevelUnit].Run != 0 || c.Levels[delivery.LevelIntegration].Run != 0 {
		t.Fatalf("the carried verdict = %+v, want a pass carried from %s with no test run", c, trunk[:12])
	}
	if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
		t.Errorf("build_go_test called %d times on the carry, was %d: a carry of unchanged digests runs nothing", got, tests)
	}
	if o := b.owed(t, 7, head1, "success"); !strings.HasPrefix(o.Text, "pass 111111111111 on abababababab | carried from aaaaaaaaaaaa") {
		t.Errorf("the carried status says %q", o.Text)
	}
}

func keptCount(t *testing.T, b *bed) int {
	t.Helper()
	keys, err := b.buckets[delivery.CheckerKeptBucket].Keys(b.ctx)
	if err != nil {
		return 0
	}
	return len(keys)
}

// TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest: the fetch answers the files in conflict; the verdict fails naming
// them, no test or digest step is run, a preview of the conflicts alone is written and the status is owed as failure.
func TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest(t *testing.T) { // 1289/AC4
	b := startBed(t)
	b.fake.conflict.Store(true)
	key := b.candidate(t, "feat/x", pull(repo, 7, head1))
	b.start(t, nil)
	v := b.verdict(t, key, ended)
	if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "files in conflict: a.go, sub/b.go") {
		t.Fatalf("verdict = %+v", v)
	}
	if o := b.owed(t, 7, head1, "failure"); !strings.HasPrefix(o.Text, "failed: n1: ") || !strings.Contains(o.Text, "files in conflict: a.go, sub/b.go") {
		t.Errorf("owed = %+v", o)
	}
	if n := b.fake.count("mesh-build-go.build_go_digest") + b.fake.count("mesh-build-go.build_go_test"); n != 0 {
		t.Errorf("%d digest and test steps ran after a conflict", n)
	}
	pk, _ := delivery.PreviewKey(repo, head1, trunk)
	if e, err := b.buckets[delivery.PreviewsBucket].Get(b.ctx, pk); err != nil {
		t.Errorf("previews.%s: %v", pk, err)
	} else if pv, err := delivery.DecodePreview(e.Value()); err != nil || strings.Join(pv.Conflicts, ",") != "a.go,sub/b.go" || pv.Merge != "" {
		t.Errorf("preview = %+v, %v, want the files in conflict alone", pv, err)
	}
}

// TestAnHqPullRequestGetsNoVerdictAndAnotherDoes: hq's pull request is judged by no one (ADR 0484 clause 8), the other one in
// the same run is.
// 1887/A10, ADR 0527 §2: a pull request into hq gets a verdict and a mesh/check from the checker, as any repository's.
func TestAnHqPullRequestGetsAVerdictAndAStatus(t *testing.T) {
	b := startBed(t)
	b.setTrunk(t, map[string]string{repo: trunk, "novox/hq": trunk})
	hq := b.candidate(t, "feat/hq", pull("novox/hq", 3, head1))
	b.start(t, nil)
	if v := b.verdict(t, hq, ended); v.State != delivery.VerdictPass {
		t.Fatalf("hq's verdict = %+v", v)
	}
	key, _ := delivery.CheckerOwedKey("novox/hq", 3, head1)
	for deadline := time.Now().Add(20 * time.Second); ; time.Sleep(50 * time.Millisecond) {
		if e, err := b.buckets[delivery.CheckerOwedBucket].Get(b.ctx, key); err == nil {
			if o, err := delivery.DecodeCheckerOwed(e.Value()); err == nil && o.State == "success" {
				return
			}
		}
		if time.Now().After(deadline) {
			t.Fatalf("hq#3 owes no success within 20 s")
		}
	}
}

// TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts: the verdict key holds running, left by a checker that stopped; the
// started checker judges it again and ends it (ADR 0478 clause 6, as the builder's).
func TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts(t *testing.T) { // N6/AC2
	b := startBed(t)
	key := b.candidate(t, "feat/x", pull(repo, 7, head1))
	group, heads, _ := delivery.ParseCheckerVerdictKey(key)
	left, err := delivery.EncodeVerdict(delivery.Verdict{Group: group, Heads: heads, Trunk: map[string]string{repo: trunk}, State: delivery.VerdictRunning})
	if err != nil {
		t.Fatal(err)
	}
	if _, err := b.buckets[delivery.CheckerVerdictsBucket].Create(b.ctx, key, left); err != nil {
		t.Fatal(err)
	}
	b.start(t, nil)
	if v := b.verdict(t, key, ended); v.State != delivery.VerdictPass {
		t.Errorf("verdict = %+v", v)
	}
}

// TestAMemberNamingAMeshSDKCommitOffTheTrunkIsJudgedAndNotRefused: the go.mod the forge answers names a commit that main does
// not hold; the check is not refused for it, since the build takes the latest published mesh-sdk whatever go.mod names
// (novox/hq ADR 0573 §2, §4; issue 1585 S14, which retires issue 1624's pin check).
func TestAMemberNamingAMeshSDKCommitOffTheTrunkIsJudgedAndNotRefused(t *testing.T) { // 1624/AC2 as ADR 0573 §4 changes it
	b := startBed(t)
	b.stalePin.Store(true)
	key := b.candidate(t, "feat/x", pull(repo, 7, head1))
	b.start(t, nil)
	if v := b.verdict(t, key, ended); v.State != delivery.VerdictPass {
		t.Fatalf("verdict = %+v", v)
	}
}

// triesStore is the tries bucket as the tools write it.
type triesStore struct {
	kv  jetstream.KeyValue
	ctx context.Context
}

func (s triesStore) Keys() ([]string, error) {
	keys, err := s.kv.Keys(s.ctx)
	if err == jetstream.ErrNoKeysFound {
		return nil, nil
	}
	return keys, err
}
func (s triesStore) Get(key string) (json.RawMessage, error) {
	e, err := s.kv.Get(s.ctx, key)
	if err == jetstream.ErrKeyNotFound {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	return e.Value(), nil
}
func (s triesStore) Create(key string, value []byte) error {
	_, err := s.kv.Create(s.ctx, key, value)
	return err
}

// TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed: a try written as checker_try writes it is taken by the running checker,
// judged through the build modules as a pull request's head merged onto its trunk, and ended pass in its key, with no
// verdict and no status owed (ADR 0501 §5).
func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
	b := startBed(t)
	b.start(t, nil)
	store := triesStore{b.buckets[check.TriesBucket], b.ctx}
	id, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head1}, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	var last check.Try
	for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		raw, _ := store.Get(id)
		if last, err = check.DecodeTry(raw); err == nil && last.Over() {
			break
		}
	}
	if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" {
		t.Fatalf("try = %+v, want passed on the trunk", last)
	}
	if b.fake.count("mesh-build-go.build_go_fetch") == 0 {
		t.Errorf("the try fetched nothing")
	}
	for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket, delivery.PreviewsBucket} {
		if keys, _ := b.buckets[name].Keys(b.ctx); len(keys) != 0 {
			t.Errorf("a try wrote %s: %v", name, keys)
		}
	}
	// Issue 734: a try whose ask the build seat gives up on (the controller keeps its copy in DEAD_LETTERS with the ask's
	// message id) ends as error saying so, and its head asked again is a new try.
	b.fake.hold.Store(head2)
	lost, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	steps, err := b.js.KeyValue(b.ctx, delivery.StepsBucket("mesh-build-go"))
	if err != nil {
		t.Fatal(err)
	}
	askID := ""
	for deadline := time.Now().Add(20 * time.Second); askID == "" && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		keys, _ := steps.Keys(b.ctx)
		for _, k := range keys {
			if e, err := steps.Get(b.ctx, k); err == nil && strings.Contains(string(e.Value()), `"running"`) {
				askID = strings.TrimPrefix(k, "n1.")
			}
		}
	}
	if askID == "" {
		t.Fatal("the held try's ask was never taken")
	}
	dead := &nats.Msg{Subject: "mesh.events.dead.SEAT_NODE_BUILD_GO.SEAT_NODE_BUILD_GO_CHECK_worker", Header: nats.Header{}}
	dead.Header.Set(buildrun.DeadHeaderMsgID, askID)
	dead.Header.Set(buildrun.DeadHeaderDeliveries, "5")
	dead.Header.Set(buildrun.DeadHeaderStream, "SEAT_NODE_BUILD_GO")
	dead.Header.Set(buildrun.DeadHeaderGaveUp, time.Now().UTC().Format(time.RFC3339Nano))
	if _, err := b.js.PublishMsg(b.ctx, dead); err != nil {
		t.Fatal(err)
	}
	for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		raw, _ := store.Get(lost)
		if last, err = check.DecodeTry(raw); err == nil && last.Over() {
			break
		}
	}
	if last.State != check.TryError || !strings.Contains(last.Said, buildrun.LostWords+" after 5 deliveries") {
		t.Fatalf("the try whose ask was given up = %+v, want error saying so", last)
	}
	if again, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now()); err != nil || again == lost {
		t.Errorf("its head asked again answered %q, %v, want a new try", again, err)
	}
}

// TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among
// the readable ones, which is what grants the forge module its read of owed (ADR 0495 §2-§3, ADR 0521; R4e finding R2-1).
func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
	b := startBed(t)
	if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
		t.Fatal(err)
	}
	b.start(t, nil)
	events, err := b.js.Stream(b.ctx, "EVENTS")
	if err != nil {
		t.Fatal(err)
	}
	var said announce.States
	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		m, err := events.GetLastMsgForSubject(b.ctx, announce.Subject("mesh-delivery-checker"))
		if err != nil {
			continue
		}
		if err := json.Unmarshal(m.Data, &said); err != nil {
			t.Fatal(err)
		}
		break
	}
	if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
		!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") ||
		!said.Keeps("asks") || said.Readable("asks") || len(said.States) != 7 {
		t.Fatalf("the checker said %+v, want its seven states, asks and hearing its own, with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
	}
}