mesh-controller/cmd/mesh-controller/composed_served_integration_test.go

//go:build integration

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/overlay"
	"github.com/nats-io/nats-server/v2/server"
	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"
	"golang.org/x/crypto/bcrypt"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/builderkeys"
	"github.com/novox/mesh-controller/internal/chain"
	"github.com/novox/mesh-controller/internal/identity"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// The compositions through the controller whole (novox/hq ADR 0539; the review's G1): the real store, the controller's
// signing key, and a bus that runs the user list the controller composes from that store, so a subject release 6 uses
// and the list does not grant is refused as it would be on the mesh. serve() runs whole on it; the builder and the
// indexer are their own accounts writing their own keys; a rotation is the operator's verb.
type servedMesh struct {
	open       *stores
	controller jetstream.JetStream
	builder    jetstream.JetStream
	indexer    jetstream.JetStream
}

// keyOfManifest is a done key of a module at fp with no artifacts, its manifest the given one, and the indexer's trunk
// value naming it in the catalogue's repository at commit.
func keyOfManifest(t *testing.T, m catalogue.Manifest, fp, commit string) (raw []byte, k sdkdelivery.BuilderBuild, trunk sdkdelivery.Module) {
	t.Helper()
	raw, err := json.Marshal(m)
	if err != nil {
		t.Fatal(err)
	}
	digest := sdkdelivery.ManifestDigest(raw)
	return raw, sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "anchor", Manifest: digest},
		sdkdelivery.Module{FP2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: commit, Path: "modules/" + m.Module}
}

func aServedMesh(t *testing.T) *servedMesh {
	t.Helper()
	open := theCatalogue(t)
	ctx := t.Context()
	inv := open.inventory
	withTrunkList(t, true)
	ident := identity.ForTest(t)
	if _, err := ident.Establish(ctx); err != nil {
		t.Fatal(err)
	}

	// The bus, running the user list the controller composes from the store, with the builder and the indexer as the
	// accounts their own manifests make (each writes its own state).
	records, err := inv.BusRecords(ctx)
	if err != nil {
		t.Fatal(err)
	}
	users, err := broker.Users(records)
	if err != nil {
		t.Fatal(err)
	}
	var controller broker.Principal
	for _, p := range users {
		if p.Kind == broker.KindController {
			controller = p
		}
	}
	password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: controller.Username(), Kind: inventory.BusController})
	if err != nil {
		t.Fatal(err)
	}
	builder := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: sdkdelivery.BuilderModule, State: []string{"builds"}}
	indexer := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: sdkdelivery.IndexerModule, State: []string{"modules", "manifests"}}
	kept, err := inv.BusUsers(ctx)
	if err != nil {
		t.Fatal(err)
	}
	hashes := map[string]string{}
	for name, u := range kept {
		hashes[name] = u.PasswordHash
	}
	fixture, _ := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
	hashes[builder.Username()], hashes[indexer.Username()] = string(fixture), string(fixture)
	filled, missing := broker.WithPasswords([]broker.Principal{controller, builder, indexer}, hashes)
	if len(missing) > 0 {
		t.Fatalf("no password for %v", missing)
	}
	accounts, err := broker.ComposeAccounts(filled)
	if err != nil {
		t.Fatal(err)
	}
	conf := filepath.Join(t.TempDir(), "bus.conf")
	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
		t.Fatal(err)
	}
	opts, err := server.ProcessConfigFile(conf)
	if err != nil {
		t.Fatalf("the composed user list does not parse: %v", err)
	}
	opts.NoLog, opts.NoSigs = true, true
	s, err := server.NewServer(opts)
	if err != nil {
		t.Fatal(err)
	}
	go s.Start()
	if !s.ReadyForConnections(10 * time.Second) {
		t.Fatal("the bus did not come up")
	}
	t.Cleanup(s.Shutdown)
	u, _ := url.Parse(s.ClientURL())
	u.User = url.UserPassword(controller.Username(), password)
	t.Setenv(broker.NATSVar, u.String())

	connect := func(user, pw string) jetstream.JetStream {
		t.Helper()
		nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, pw), nats.CustomInboxPrefix("_INBOX."+user))
		if err != nil {
			t.Fatalf("%s cannot connect: %v", user, err)
		}
		t.Cleanup(nc.Close)
		js, err := jetstream.New(nc)
		if err != nil {
			t.Fatal(err)
		}
		return js
	}
	m := &servedMesh{open: open, controller: connect(controller.Username(), password),
		builder: connect(builder.Username(), "pw"), indexer: connect(indexer.Username(), "pw")}

	// The test's own inventory on the bus's chain, as the serving controller's will be: what is registered and assigned
	// below is in the chain it reads (assignments and builds live on the bus, ADR 0408).
	chainConn, err := nats.Connect(s.ClientURL(), nats.UserInfo(controller.Username(), password),
		nats.CustomInboxPrefix("_INBOX."+controller.Username()))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(chainConn.Close)
	// The controller's own buckets, the chain's among them, as it asserts them at its start.
	asserter, err := broker.Dial(u.String())
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(asserter.Close)
	if err := asserter.EnsureControllerBuckets(); err != nil {
		t.Fatalf("the controller's buckets: %v", err)
	}
	c, err := chain.OnTheBus(ctx, chainConn, func(context.Context) error { return nil })
	if err != nil {
		t.Fatal(err)
	}
	inv.UseChain(c)

	// A provider and its consumer (bindings_test.go's store and board), registered from done keys of the catalogue's
	// repository, so each has a fingerprint of the new form; the store holds the mesh's seat on anchor, the board on
	// laptop is bound to it.
	shelf := map[string]catalogue.Manifest{}
	for _, m := range storeManifests() {
		shelf[m.Module] = m
	}
	// The board is given its credential in a file, as a consumer of a database is.
	board := shelf["board"]
	board.Secrets = map[string]string{"postgres-database": "/var/lib/board/database-password"}
	shelf["board"] = board
	for i, module := range []string{"store", "board"} {
		raw, k, trunk := keyOfManifest(t, shelf[module], fpOf(byte('a'+i)), "c0ffee0123456789")
		keys := trunkKeys{keyBuckets: keyBuckets{keys: map[string]builderkeys.Key{module: aBuilderKey(module, trunk.FP2, k)},
			manifests: map[string]string{trunk.Manifest: string(raw)}}, trunk: trunk}
		if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil {
			t.Fatalf("%s is not registered from its key: %v", module, err)
		}
	}
	// The private network on both machines, as aMesh assigns it, now on the chain the controller reads.
	for _, node := range []string{"anchor", "laptop"} {
		if _, err := inv.Assign(ctx, node, overlay.Name); err != nil {
			t.Fatal(err)
		}
	}
	if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.Assign(ctx, "anchor", "store"); err != nil {
		t.Fatal(err)
	}
	if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.Assign(ctx, "laptop", "board"); err != nil {
		t.Fatal(err)
	}
	if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
		t.Fatal(err)
	}

	// The builder's and the indexer's buckets as their own parts make them, there before the controller starts (its
	// watch of their keys begins at its start); and the plans' buckets a controller of release 5 left.
	for _, b := range []string{sdkdelivery.BuilderBuildsBucket, sdkdelivery.ModulesBucket, sdkdelivery.ManifestsBucket} {
		if _, err := m.controller.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil {
			t.Fatalf("the bucket %s: %v", b, err)
		}
	}
	for _, b := range broker.PlanStateBuckets {
		if _, err := m.controller.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
			t.Fatal(err)
		}
	}

	keepWhatServeSets(t)
	serving, stop := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- serve(serving) }()
	t.Cleanup(func() {
		stop()
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			t.Log("the controller did not stop within 30 s")
		}
		// A stopping controller refuses every verb from then on as a handover (issue 289), which its process exits
		// after; this one is the test process, so a later test's verb would be refused without running (issue 1945:
		// the conditions verb's own process was never started). Its stop marks it once the serving context ends.
		for deadline := time.Now().Add(5 * time.Second); !handingOver.Load() && time.Now().Before(deadline); {
			time.Sleep(10 * time.Millisecond)
		}
		handingOver.Store(false)
	})
	// Served once both nodes have their own composition and their module's: the start's pass wrote them all. A node's
	// compositions are written in key order, so anchor's own comes before its store's (issue 1945: a test that reads the
	// revisions at once must not catch the pass midway).
	for _, k := range []struct{ node, module, fp string }{{"anchor", "", ""}, {"laptop", "", ""}, {"anchor", "store", fpOf('a')},
		{"laptop", "board", fpOf('b')}} {
		key, _ := sdkdelivery.ComposedNodeKey(k.node)
		if k.module != "" {
			key, _ = sdkdelivery.ComposedKey(k.node, k.module, k.fp)
		}
		m.waitFor(t, done, k.node, key, 0)
	}
	return m
}

// aBuilderKey is the builder's done key of module at fp, as the take-in reads it.
func aBuilderKey(module, fp string, b sdkdelivery.BuilderBuild) builderkeys.Key {
	return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, Build: b}
}

// waitFor waits until the node's composition bucket holds key at a revision above after, failing when the controller
// stops or a minute passes: a test's wait for what an event made, never a loop of the code under test.
func (m *servedMesh) waitFor(t *testing.T, done chan error, node, key string, after uint64) uint64 {
	t.Helper()
	bucket, _ := sdkdelivery.ComposedBucketFor(node)
	deadline := time.Now().Add(time.Minute)
	for time.Now().Before(deadline) {
		select {
		case err := <-done:
			t.Fatalf("the controller stopped: %v", err)
		default:
		}
		if kv, err := m.controller.KeyValue(t.Context(), bucket); err == nil {
			if e, err := kv.Get(t.Context(), key); err == nil && e.Revision() > after {
				return e.Revision()
			}
		}
		time.Sleep(200 * time.Millisecond)
	}
	t.Fatalf("%s in %s was not written within a minute; %s", key, bucket, m.whatIsThere(t, node))
	return 0
}

// whatIsThere says, for a failure, what the node's bucket holds, the ids its own composition carries and the
// fingerprints the controller has registered.
func (m *servedMesh) whatIsThere(t *testing.T, node string) string {
	t.Helper()
	bucket, _ := sdkdelivery.ComposedBucketFor(node)
	out := "keys:"
	kv, err := m.controller.KeyValue(context.Background(), bucket)
	if err != nil {
		return out + " " + err.Error()
	}
	lister, err := kv.ListKeys(context.Background())
	if err == nil {
		for k := range lister.Keys() {
			out += " " + k
		}
	}
	if lister, err := kv.ListKeys(context.Background()); err == nil {
		for k := range lister.Keys() {
			if e, err := kv.Get(context.Background(), k); err == nil {
				if c, err := sdkdelivery.DecodeComposed(e.Value()); err == nil {
					out += fmt.Sprintf("; %s at revision %d carries", k, e.Revision())
					for _, r := range c.Resources {
						var head struct{ ID string }
						_ = json.Unmarshal(r, &head)
						out += " " + head.ID
					}
				}
			}
		}
	}
	holders, herr := m.open.inventory.HoldersOf(context.Background(), "postgres-database", "")
	out += fmt.Sprintf("; holders of postgres-database: %+v (%v)", holders, herr)
	own, _ := sdkdelivery.ComposedNodeKey(node)
	if e, err := kv.Get(context.Background(), own); err == nil {
		if c, err := sdkdelivery.DecodeComposed(e.Value()); err == nil {
			out += "; its own carries:"
			for _, r := range c.Resources {
				var head struct{ ID string }
				_ = json.Unmarshal(r, &head)
				out += " " + head.ID
			}
		}
	}
	fps, err := currentFingerprints(context.Background(), m.open.inventory)
	out += fmt.Sprintf("; fingerprints registered: %v (%v)", fps, err)
	// And what composing the node says now, in this process, under the serving controller's lease.
	_, cerr := composeModules(context.Background(), m.open, []string{node})
	return out + fmt.Sprintf("; composing %s now: %v", node, cerr)
}

// revisions are a node's composition keys and their revisions.
func (m *servedMesh) revisions(t *testing.T, node string) map[string]uint64 {
	t.Helper()
	bucket, _ := sdkdelivery.ComposedBucketFor(node)
	kv, err := m.controller.KeyValue(t.Context(), bucket)
	if err != nil {
		t.Fatal(err)
	}
	lister, err := kv.ListKeys(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	out := map[string]uint64{}
	for k := range lister.Keys() {
		if e, err := kv.Get(t.Context(), k); err == nil {
			out[k] = e.Revision()
		}
	}
	return out
}

// declarationsSent is how many declarations the bus's stream of them holds: none is ever sent in release 6.
func (m *servedMesh) declarationsSent(t *testing.T) uint64 {
	t.Helper()
	s, err := m.controller.Stream(t.Context(), broker.DeclarationsStream)
	if errors.Is(err, jetstream.ErrStreamNotFound) {
		return 0
	}
	if err != nil {
		t.Fatal(err)
	}
	info, err := s.Info(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	return info.State.Msgs
}

// 1945/AC6, AC14: started whole on a bus running its own user list, the controller composes every node and every
// assigned module, deletes the plans' buckets and says each as bucket-deleted, and sends no declaration.
func TestServedTheStartComposesEveryNodeAndDeletesThePlansBuckets(t *testing.T) {
	m := aServedMesh(t)
	for _, k := range []struct{ node, module string }{{"anchor", "store"}, {"laptop", "board"}} {
		fp := map[string]string{"store": fpOf('a'), "board": fpOf('b')}[k.module]
		key, _ := sdkdelivery.ComposedKey(k.node, k.module, fp)
		m.waitFor(t, nil, k.node, key, 0)
	}
	for _, b := range broker.PlanStateBuckets {
		if _, err := m.controller.KeyValue(t.Context(), b); !errors.Is(err, jetstream.ErrBucketNotFound) {
			t.Errorf("%s is still there: %v", b, err)
		}
	}
	if n := m.declarationsSent(t); n != 0 {
		t.Errorf("%d declarations were sent", n)
	}
}

// 1945/AC3: the builder writes the done key of a new fingerprint of an assigned module and the indexer its trunk
// value: the controller, hearing the keys, registers it, writes its composition on the node it is assigned to and
// `grants.<module>.<fp>`; nothing is sent.
func TestServedADoneKeyIsComposedAndGranted(t *testing.T) {
	m := aServedMesh(t)
	ctx := t.Context()
	shelf := map[string]catalogue.Manifest{}
	for _, mf := range storeManifests() {
		shelf[mf.Module] = mf
	}
	next := shelf["board"]
	next.Secrets = map[string]string{"postgres-database": "/var/lib/board/database-password"}
	next.Version = "2"
	fp := fpOf('c')
	raw, k, trunk := keyOfManifest(t, next, fp, "d00d0123456789ab")
	manifests, err := m.indexer.KeyValue(ctx, sdkdelivery.ManifestsBucket)
	if err != nil {
		t.Fatal(err)
	}
	value, _ := sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: "board", JSON: string(raw)})
	if _, err := manifests.Put(ctx, trunk.Manifest, value); err != nil {
		t.Fatalf("the indexer cannot write its manifest: %v", err)
	}
	builds, err := m.builder.KeyValue(ctx, sdkdelivery.BuilderBuildsBucket)
	if err != nil {
		t.Fatal(err)
	}
	key, _ := sdkdelivery.BuilderBuildKey("board", fp)
	value, _ = sdkdelivery.EncodeBuilderBuild(k)
	if _, err := builds.Put(ctx, key, value); err != nil {
		t.Fatalf("the builder cannot write its key: %v", err)
	}
	modules, err := m.indexer.KeyValue(ctx, sdkdelivery.ModulesBucket)
	if err != nil {
		t.Fatal(err)
	}
	mk, _ := sdkdelivery.ModulesKey("board")
	value, _ = sdkdelivery.EncodeModule(trunk)
	if _, err := modules.Put(ctx, mk, value); err != nil {
		t.Fatalf("the indexer cannot write its trunk value: %v", err)
	}
	composed, _ := sdkdelivery.ComposedKey("laptop", "board", fp)
	m.waitFor(t, nil, "laptop", composed, 0)
	grants, err := m.controller.KeyValue(ctx, broker.GrantsBucket)
	if err != nil {
		t.Fatal(err)
	}
	gk, _ := sdkdelivery.ControllerGrantKey("board", fp)
	// The take-in writes the grant after the compositions (composed_keys.go), so it is waited for, not read in the instant
	// the composition shows: a loaded build node read it a moment early (issue 1945).
	becomes(t, 10*time.Second, "grants."+gk+" written", func() bool {
		_, err := grants.Get(ctx, gk)
		return err == nil
	})
	if n := m.declarationsSent(t); n != 0 {
		t.Errorf("%d declarations were sent", n)
	}
}

// 1945/AC2: the operator rotates the store's credential, the real verb: the compositions that carry it (the store's
// grants on anchor, and the board's on laptop when it carries the credential) get a new revision, every other
// composition keeps its own, and nothing is sent.
func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) {
	m := aServedMesh(t)
	// Everything that writes or discards a credential row, and what composing each node saw and decided, in order, with the
	// callers: one failure shows the cause (issue 1945).
	var traceMu sync.Mutex
	var trace []string
	note := func(line string) {
		traceMu.Lock()
		defer traceMu.Unlock()
		trace = append(trace, time.Now().Format("15:04:05.000")+" "+line)
	}
	inventory.SecretTrace = func(op, what string) { note("secret " + op + ": " + what) }
	composeTrace = func(line string) { note("compose " + line) }
	t.Cleanup(func() { inventory.SecretTrace, composeTrace = nil, nil })
	traced := func() string {
		traceMu.Lock()
		defer traceMu.Unlock()
		return strings.Join(trace, "\n    ")
	}
	defer func() {
		if t.Failed() {
			t.Logf("TRACE\n    %s", traced())
		}
	}()
	before := map[string]map[string]uint64{"anchor": m.revisions(t, "anchor"), "laptop": m.revisions(t, "laptop")}
	holders, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "")
	var waited uint64
	// The verb runs here as its child process runs it: its inventory takes the chain from the bus the served controller keeps it
	// on (the registrations and assignments are there), not the one in memory the fixture's store was made with.
	inventory.OnTheBusChainForTest(t)
	said, err := capturedOutput(func() error {
		if err := rotateCommand(t.Context(), []string{"postgres-database"}); err != nil {
			return err
		}
		// The serving controller's own lines while the rotation lands are captured too.
		boardKey, _ := sdkdelivery.ComposedKey("laptop", "board", fpOf('b'))
		waited = m.waitForOrSay(t, "laptop", boardKey, before["laptop"][boardKey])
		return nil
	})
	if err != nil {
		t.Fatalf("rotate: %v\n%s", err, said)
	}
	t.Logf("rotate said (holders before: %+v):\n%s", holders, said)
	defer func() {
		if t.Failed() {
			t.Logf("rotate said:\n%s", said)
		}
	}()
	boardKey, _ := sdkdelivery.ComposedKey("laptop", "board", fpOf('b'))
	storeKey, _ := sdkdelivery.ComposedKey("anchor", "store", fpOf('a'))
	// The board's sealed credential file is new: its composition moves with the rotation.
	if rev := waited; rev == 0 {
		var kept []string
		for _, line := range strings.Split(said, "\n") {
			for _, w := range []string{"board", "postgres", "secret", "credential", "laptop", "withdr", "retir", "error", "refused"} {
				if strings.Contains(line, w) {
					kept = append(kept, line)
					break
				}
			}
		}
		said = strings.Join(kept, " | ")
		after, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "")
		in, ierr := inputFor(t.Context(), m.open)
		var credentials string
		if ierr == nil {
			credentials = fmt.Sprintf("would make %+v; credentials %+v", in.wouldMake, in.Credentials)
		}
		state, serr := m.open.inventory.PairCredentialState(t.Context(), inventory.PairCredential{Name: "postgres-database",
			Consumer: "laptop", ConsumerModule: "board", Provider: "anchor"})
		d, derr := composeSend(t.Context(), m.open, "laptop")
		var sealed []string
		for _, r := range d.Resources {
			if r["id"] == "board.secret-postgres-database" {
				sealed = append(sealed, fmt.Sprint(r["sealed"])[:min(24, len(fmt.Sprint(r["sealed"])))])
			}
		}
		diag := fmt.Sprintf("state %+v %v; compose now %v sealed %v; %s %v", state, serr, derr, sealed, credentials, ierr)
		// One line: the verdict keeps only a failure's first line (issue 1921).
		t.Fatalf("the board's composition did not move within a minute after the rotation (revision before %d); TRACE %s; LINES %s; "+
			"holders before %+v, after %+v; DIAG %s; %s", before["laptop"][boardKey],
			strings.Join(strings.Fields(strings.ReplaceAll(traced(), "\n", " | ")), " "), strings.Join(strings.Fields(said), " "),
			holders, after, diag, strings.Join(strings.Fields(m.whatIsThere(t, "laptop")), " "))
	}
	moved := map[string]bool{}
	for node, was := range before {
		for k, rev := range m.revisions(t, node) {
			if rev != was[k] {
				moved[k] = true
			}
		}
	}
	for k := range moved {
		if k != boardKey && k != storeKey {
			t.Errorf("%s moved: only the compositions carrying the credential do", k)
		}
	}
	if !moved[boardKey] {
		t.Errorf("the board's composition did not move: %v", moved)
	}
	if n := m.declarationsSent(t); n != 0 {
		t.Errorf("%d declarations were sent", n)
	}
}

// waitForOrSay is waitFor that answers 0 instead of failing, for a caller that says more.
func (m *servedMesh) waitForOrSay(t *testing.T, node, key string, after uint64) uint64 {
	t.Helper()
	bucket, _ := sdkdelivery.ComposedBucketFor(node)
	deadline := time.Now().Add(time.Minute)
	for time.Now().Before(deadline) {
		if kv, err := m.controller.KeyValue(t.Context(), bucket); err == nil {
			if e, err := kv.Get(t.Context(), key); err == nil && e.Revision() > after {
				return e.Revision()
			}
		}
		time.Sleep(200 * time.Millisecond)
	}
	return 0
}

// capturedOutput runs act with this process's standard output captured, and answers what it printed.
func capturedOutput(act func() error) (string, error) {
	r, w, err := os.Pipe()
	if err != nil {
		return "", act()
	}
	was := os.Stdout
	os.Stdout = w
	done := make(chan string)
	go func() {
		var b strings.Builder
		buf := make([]byte, 4096)
		for {
			n, err := r.Read(buf)
			b.Write(buf[:n])
			if err != nil {
				break
			}
		}
		done <- b.String()
	}()
	actErr := act()
	os.Stdout = was
	w.Close()
	return <-done, actErr
}

// keepWhatServeSets gives back, once the served controller stopped, what serve sets for the rest of its process, which
// a controller's process exits after and the test process does not (issue 1945): the served mark in the environment, the
// hand-act log's connection, the keeper, the status and the event publishers. Registered before the controller starts, so
// it runs after the cleanup that stops it.
func keepWhatServeSets(t *testing.T) {
	t.Helper()
	t.Setenv(servedVar, "")
	unheld, status, given, check, follow, keeper, hand := logUnheldChanges, statusFrom, givenEvents, checkEvents,
		link.Calls.Follow, conditionsFrom, handActConn
	t.Cleanup(func() {
		logUnheldChanges, statusFrom, givenEvents, checkEvents, link.Calls.Follow, conditionsFrom, handActConn =
			unheld, status, given, check, follow, keeper, hand
	})
}