Let the mesh's resolver seat have several holders on record

musl takes the first reply from any listed nameserver, so a public fallback
beside the mesh's resolver answered NXDOMAIN for mesh names in every Alpine
container (hq ADR 0223). The fix is two mesh resolvers and no public one, which
needs mesh-dns-resolver held on two machines: a seat can now be replicated,
each holder recorded by 'seat <name> --add', checkClaims accepts every holder
on record and still refuses a second holder of any other mesh seat, a holder
answers its own requirement, and a roster fact gives each replicated seat's
holders, this machine first, so resolv-conf can list them. Migration 0062 keys
a holding by seat and assignment.

cmd/mesh-controller/adopting_test.go

@@ -85,7 +85,7 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
+	if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{

cmd/mesh-controller/held_back.go

@@ -0,0 +1,241 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
+// issue 259, ADR 0221).
+//
+// A named push ends by sending every other machine whose declaration differs from what it was last
+// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
+// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
+// out makes every machine running it differ from the merge on, and so did a module whose plan was
+// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
+// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
+// everywhere at once.
+//
+// What tells the two apart is which build of each module the machine was last sent, kept with every
+// send. A machine any of whose modules would move to a build its policy or an open plan holds back
+// is not sent by a push that did not name it; the push says which, and why, and how to send it.
+
+// heldBack is why a push that did not name a machine must not send it, empty when it may.
+//
+// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
+// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
+// machine was last sent — including a module the machine was never sent at all. A move is held when
+// the module's policy records rather than rolls out, or when an open plan has not yet sent this
+// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
+// is not known, so a held upgrade cannot be told from anything else.
+func heldBack(node string, modules []string, sent map[string]string, known bool,
+	current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
+	if !known {
+		return []string{"which builds it was last sent is not known — it was last sent before the " +
+			"mesh kept them, or sent a declaration by hand"}
+	}
+	var why []string
+	for _, m := range modules {
+		now := current[m]
+		was, carried := sent[m]
+		if carried && was == now.Commit {
+			continue
+		}
+		move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
+		if !now.RollOut {
+			why = append(why, move+", which its upgrade policy records rather than rolls out")
+			continue
+		}
+		if id := planStillToSend(plans, m, node); id != "" {
+			why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
+		}
+	}
+	sort.Strings(why)
+	return why
+}
+
+// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
+// one holding the module that has neither finished sending it nor sent it here first, and has not
+// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
+// per machine.
+func planStillToSend(plans []inventory.Plan, module, node string) string {
+	for _, p := range plans {
+		s, holds := p.Modules[module]
+		if !p.Open() || !holds {
+			continue
+		}
+		if s == nil {
+			return p.ID
+		}
+		if s.SentAt != nil || s.State == "failed" {
+			continue
+		}
+		first := false
+		for _, n := range s.First {
+			if n == node {
+				first = true
+			}
+		}
+		if !first {
+			return p.ID
+		}
+	}
+	return ""
+}
+
+func fromBuild(was string, carried bool) string {
+	if !carried {
+		return "(never sent it) "
+	}
+	return "from " + buildName(was) + " "
+}
+
+func buildName(commit string) string {
+	if commit == "" {
+		return "a build with no source"
+	}
+	return shortCommit(commit)
+}
+
+// heldMachines reads, for each machine named, why a push that did not name it must not send it
+// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
+// to the send, which says why.
+func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
+	out := map[string][]string{}
+	if len(names) == 0 {
+		return out, nil
+	}
+	inv := open.inventory
+	current, err := inv.CurrentBuilds(ctx)
+	if err != nil {
+		return nil, err
+	}
+	plans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for _, node := range names {
+		plan, _, err := planFor(ctx, open, node)
+		if err != nil {
+			continue
+		}
+		modules := make([]string, 0, len(plan.Modules))
+		for _, m := range plan.Modules {
+			modules = append(modules, m.Module)
+		}
+		sent, known, err := inv.SentBuilds(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 {
+			out[node] = why
+		}
+	}
+	return out, nil
+}
+
+// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
+// was not sent, that whatever else it is owed waits with it, and the command that sends it.
+func sayHeld(w io.Writer, node string, why []string) {
+	fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
+		"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
+		"grant from this push among it — waits with it. `push %s` sends it\n",
+		node, strings.Join(why, "; "), node)
+}
+
+// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
+// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
+// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
+//
+// `handled` is every machine already sent or already said; it is not considered again. Answers the
+// machines that could not be composed, as refusals.
+func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
+	compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
+	w io.Writer) ([]string, error) {
+	inv := open.inventory
+	var refusals []string
+	// Bounded by the node count: a node is marked handled the round it is considered and is never
+	// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
+	// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
+	// cascade legitimately still converging, so it is said rather than passed over in silence.
+	rounds := 0
+	for {
+		would, err := wouldSend(ctx, open, nodes)
+		if err != nil {
+			return refusals, err
+		}
+		behind, err := inv.Waiting(ctx, would)
+		if err != nil {
+			return refusals, err
+		}
+		var also []string
+		for _, m := range behind {
+			if !handled[m.Node] {
+				also = append(also, m.Node)
+			}
+		}
+		if len(also) == 0 {
+			return refusals, nil
+		}
+		if rounds++; rounds > len(nodes) {
+			fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
+				"should not happen; run `push --behind` to finish\n",
+				rounds-1, strings.Join(also, ", "))
+			return refusals, nil
+		}
+		sort.Strings(also)
+		held, err := heldMachines(ctx, open, also)
+		if err != nil {
+			return refusals, err
+		}
+		var sending []string
+		for _, name := range also {
+			// Every candidate this round is marked handled — the sent ones so they are not
+			// re-listed, the held ones because they stay held, and the refused ones so a machine
+			// that cannot be composed does not make the loop spin on it for ever.
+			handled[name] = true
+			if why, isHeld := held[name]; isHeld {
+				sayHeld(w, name, why)
+				continue
+			}
+			sending = append(sending, name)
+		}
+		if len(sending) == 0 {
+			continue
+		}
+		fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
+			"declaration since changed; sending it too\n", strings.Join(sending, ", "))
+		// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
+		// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
+		// Held for this round only, and after the last round's were given back, so two pushes
+		// cascading into each other's machines never each wait on the other.
+		refused, err := sendRound(ctx, open, sending, compose, d, holder)
+		refusals = append(refusals, refused...)
+		if err != nil {
+			return refusals, err
+		}
+	}
+}
+
+// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
+// is left out of it said, and its declaration allocated.
+func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
+	return func(held context.Context, node string) (sendable, error) {
+		plan, settings, err := planFor(held, open, node)
+		if err != nil {
+			return sendable{}, err
+		}
+		reportUnhostable(node, plan)
+		declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
+		if err == nil {
+			reportLeftOut(node, declared)
+		}
+		return declared, err
+	}
+}

cmd/mesh-controller/held_back_test.go

@@ -0,0 +1,335 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"reflect"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/overlay"
+)
+
+// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
+// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
+// else that moved is still a consequence the push sends (ADR 0083).
+func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
+	current := map[string]inventory.CurrentBuild{
+		"resolver": {Commit: "c2c2c2c2c2"},
+		"agent":    {Commit: "a2", RollOut: true},
+		"network":  {},
+	}
+	modules := []string{"network", "resolver", "agent"}
+	sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
+
+	// A module whose policy records moved: held, naming it, both builds and why.
+	why := heldBack("laptop", modules, sent, true, current, nil)
+	if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
+		!strings.Contains(why[0], "upgrade policy records") {
+		t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
+	}
+
+	// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
+	sent["resolver"] = "c2c2c2c2c2"
+	if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
+		t.Fatalf("a machine whose builds are all current was held: %v", why)
+	}
+
+	// A module whose policy rolls out moved, and no plan holds it: sent, as before.
+	sent["agent"] = "a1"
+	if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
+		t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
+	}
+
+	// The last send's builds are not known: held whole.
+	if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
+		!strings.Contains(why[0], "not known") {
+		t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
+	}
+
+	// A module the machine was never sent, under a recording policy: held, and said so.
+	delete(sent, "resolver")
+	sent["agent"] = "a2"
+	if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
+		!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
+		t.Fatalf("a module never sent under a recording policy: %v", why)
+	}
+}
+
+// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
+// naming some other machine must not send them for it.
+func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
+	at := time.Now()
+	current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
+	sent := map[string]string{"agent": "a1"}
+	waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
+		"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
+
+	why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
+	if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
+		t.Fatalf("a machine the plan has not reached was not held: %v", why)
+	}
+	// The first machine itself was sent by the plan: not held by it.
+	if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
+		t.Fatalf("the plan's first machine was held: %v", why)
+	}
+	// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
+	for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
+		plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
+			Modules: map[string]*inventory.PlanModule{"agent": s}}}
+		if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
+			t.Errorf("%s: not held: %v", what, why)
+		}
+	}
+	// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
+	for what, plans := range map[string][]inventory.Plan{
+		"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
+			"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
+		"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
+			"agent": {State: "failed"}}}},
+		"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
+			"agent": {State: "built"}}}},
+	} {
+		if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
+			t.Errorf("%s: held: %v", what, why)
+		}
+	}
+}
+
+// A send records the build of each module it carried; a module left out of it keeps the build it
+// was last sent, since the machine keeps that one.
+func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
+	current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
+	got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
+		current, map[string]string{"a": "a1", "b": "b1"})
+	if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("carried %v, wanted %v", got, want)
+	}
+	// Not known before: the left-out module is not recorded at all, so it reads as never sent.
+	got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
+	if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("carried %v, wanted %v", got, want)
+	}
+}
+
+// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
+// reads the machine as current — and writes down which machines it declared.
+type recordedDelivery struct {
+	inv      *inventory.Inventory
+	declared []string
+}
+
+func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
+
+func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
+	r.declared = append(r.declared, s.node)
+	return recordSent(ctx, r.inv, s.node, body, s.declared.Builds)
+}
+
+// aResolver is a module built from a repository, at a commit, with something on the machine that
+// says which build it is.
+func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
+	t.Helper()
+	m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
+		{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
+	}}
+	if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
+		Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// A third machine on the private network: once it is sent, every other machine's peers change with
+// it, which is a consequence a push must still send — no build moved.
+func aThirdMachine(t *testing.T, open *stores) {
+	t.Helper()
+	ctx := t.Context()
+	record, err := open.inventory.AddNode(ctx, "spare")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
+		t.Fatal(err)
+	}
+	reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
+		{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
+		{"name": "systemd", "present": true}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var profile map[string]any
+	if err := json.Unmarshal(reported, &profile); err != nil {
+		t.Fatal(err)
+	}
+	if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
+		t.Fatal(err)
+	}
+	if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
+		t.Fatal(err)
+	}
+	if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
+// naming the anchor, and the laptop — running the same module — left with what it had, by name.
+func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	asked := time.Now().Add(-time.Hour)
+	aResolver(t, open, "c1c1c1c1c1", asked)
+	for _, node := range []string{"anchor", "laptop"} {
+		if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	gens, err := generators(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	compose := composeForPush(open, gens)
+	d := &recordedDelivery{inv: inv}
+	if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
+		t.Fatal(err)
+	}
+	if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
+		t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
+	}
+	digestOfLaptop := func() string {
+		sent, err := inv.Outstanding(ctx, "laptop")
+		if err != nil {
+			t.Fatal(err)
+		}
+		return sent
+	}
+	before := digestOfLaptop()
+
+	// The change merges; the policy is the default, record. `push anchor` sends the anchor...
+	aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
+	d.declared = nil
+	if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
+		t.Fatal(err)
+	}
+	// ...and its cascade leaves the laptop, saying so.
+	var said bytes.Buffer
+	d.declared = nil
+	refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
+	if err != nil || len(refused) != 0 {
+		t.Fatalf("the cascade failed: %v %v", refused, err)
+	}
+	if len(d.declared) != 0 {
+		t.Fatalf("the cascade sent %v a build its policy records", d.declared)
+	}
+	if digestOfLaptop() != before {
+		t.Fatal("the laptop's last send moved: it was sent the held build")
+	}
+	for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
+		"upgrade policy records", "`push laptop` sends it"} {
+		if !strings.Contains(said.String(), want) {
+			t.Errorf("the push did not say %q:\n%s", want, said.String())
+		}
+	}
+
+	// Held and owed something else at once — a peer joined: still not sent, and both said: why it
+	// is held, and that what else it is owed waits with it.
+	aThirdMachine(t, open)
+	d.declared = nil
+	if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
+		t.Fatal(err)
+	}
+	d.declared = nil
+	said.Reset()
+	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
+		compose, d, "", &said); err != nil {
+		t.Fatal(err)
+	}
+	if len(d.declared) != 0 || digestOfLaptop() != before {
+		t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
+	}
+	if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
+		t.Fatalf("the push did not say both:\n%s", said.String())
+	}
+
+	// A policy that rolls out: the laptop is a consequence like any other, and sent.
+	if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
+		t.Fatal(err)
+	}
+	said.Reset()
+	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
+		compose, d, "", &said); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
+		t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
+	}
+	if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
+		t.Fatalf("the new send did not record the new build: %v", builds)
+	}
+}
+
+// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
+// for another reason is sent by a push that names someone else.
+func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
+	if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	gens, err := generators(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	compose := composeForPush(open, gens)
+	d := &recordedDelivery{inv: inv}
+	if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
+		t.Fatal(err)
+	}
+
+	// `push spare`, the machine just placed: the others' peers change with it.
+	aThirdMachine(t, open)
+	if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
+		t.Fatal(err)
+	}
+	d.declared = nil
+	var said bytes.Buffer
+	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
+		t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
+	}
+	if strings.Contains(said.String(), "was not sent") {
+		t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
+	}
+
+	// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
+	record, err := inv.NodeByName(ctx, "laptop")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
+		t.Fatal(err)
+	}
+	d.declared = nil
+	said.Reset()
+	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
+		t.Fatal(err)
+	}
+	// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
+	// question.
+	if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
+		t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
+	}
+}

cmd/mesh-controller/holdings.go

@@ -6,6 +6,8 @@ import (
 	"sort"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/overlay"
 )
 
 // recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
@@ -90,3 +92,72 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
 	}
 	return said, nil
 }
+
+// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
+// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
+// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
+//
+// **The holders on record, and only the sole claimant when there are none** — the same answer the
+// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
+// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
+// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
+// listed here. A holder off the private network is left out: a resolver named at an address nothing
+// answers is a lookup that waits out its timeout on every name.
+func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
+	shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
+	var replicated []catalogue.Seat
+	for _, s := range catalogue.Seats() {
+		if s.Replicated && s.Scope == catalogue.ScopeMesh {
+			replicated = append(replicated, s)
+		}
+	}
+	if len(replicated) == 0 {
+		return nil, nil
+	}
+	recorded, err := inv.Holdings(ctx)
+	if err != nil {
+		return nil, err
+	}
+	places, err := onTheNetwork(ctx, inv, shelf)
+	if err != nil {
+		return nil, err
+	}
+	address := map[string]string{}
+	for _, p := range places {
+		address[p.Name] = p.Address
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return nil, err
+	}
+	out := map[string]map[string]string{}
+	for _, seat := range replicated {
+		var nodes []string
+		for _, h := range recorded {
+			if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
+				nodes = append(nodes, h.Node)
+			}
+		}
+		if len(nodes) == 0 {
+			var derived []string
+			for _, e := range entries {
+				for _, c := range e.Manifest.Claims {
+					if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
+						derived = append(derived, e.On...)
+					}
+				}
+			}
+			if len(derived) == 1 {
+				nodes = derived
+			}
+		}
+		at := map[string]string{}
+		for _, n := range nodes {
+			if address[n] != "" {
+				at[overlay.InternalName(n)] = address[n]
+			}
+		}
+		out[seat.Name] = at
+	}
+	return out, nil
+}

cmd/mesh-controller/issue_204_test.go

@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
 	}
 	cancel() // the sender is going away: its context is cancelled between the send and the record
 	body := []byte(`{"declaration":1,"resources":[]}`)
-	digest, err := recordSent(ctx, inv, "anchor", body)
+	digest, err := recordSent(ctx, inv, "anchor", body, nil)
 	if err != nil {
 		// NodeByName on the cancelled context may itself refuse; the record must still be possible
 		// through the detached context, so look the node up again on a live one.

cmd/mesh-controller/main.go

@@ -194,6 +194,7 @@ func usage() {
   seats [--json]                 every seat this mesh defines, what it delivers, and who holds it
   seat rename <from> <to>        rename a seat; its former name still resolves (ADR 0122)
   seat <name> --to <node>/<module>   hand a seat to that assignment as one act; never empty in between (ADR 0131)
+  seat <name> --add <node>/<module>  add a holder beside the others, for a replicated seat (ADR 0223)
   board [--listen ADDR]          the same three questions, as a page that holds nothing
   api --issuer URL [--listen A]  assign and unassign over http, for a surface that is not here
   assign <node> <module>...      put modules on a node, judged together (ADR 0207)

cmd/mesh-controller/network.go

@@ -275,25 +275,9 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
 	if err != nil {
 		return nil, err
 	}
+	// No registry trust is composed here any more: the container runtime's module states it, told
+	// where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR 0222, issue 190).
 	g, err := overlay.From(nodes, cidr, "")
-	if g != nil {
-		// The artifact store, as this network reaches it. Found rather than configured: the
-		// provider is whichever module offers it, on whichever machine holds that module — and if
-		// nothing does yet (genesis raises the registry before the catalogue knows it), there is
-		// no trust to write and nothing is written (novox/hq ADR 0082).
-		//
-		// Refused rather than composed without it when the question could not be answered: a
-		// declaration missing the trust because a lookup failed is a machine that cannot pull,
-		// delivered by a push that reported success — and nothing recomposes it until the next
-		// push (the shape of novox/hq issues 042/048, reappearing as a race).
-		at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
-		if storeErr != nil {
-			return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
-		}
-		if found {
-			g.TrustRegistry(overlay.InternalName(at) + ":" + port)
-		}
-	}
 	if err != nil && len(refused) > 0 {
 		// The network is missing something, and some machines could not be resolved at all. Those
 		// are almost always the same fact: a node that does not resolve contributes nothing, so

cmd/mesh-controller/plan.go

@@ -397,9 +397,49 @@ func declarationWith(ctx context.Context, open *stores, node string,
 	if err != nil {
 		return sendable{}, err
 	}
-	return sendable{Resources: composed.Resources, Adoption: adoption,
+	out := sendable{Resources: composed.Resources, Adoption: adoption,
 		Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
-		LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
+		LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}
+	// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
+	// 0221). Read only on the send path: a question about what would be sent records nothing.
+	if choosing == Allocating {
+		current, err := open.inventory.CurrentBuilds(ctx)
+		if err != nil {
+			return sendable{}, err
+		}
+		before, known, err := open.inventory.SentBuilds(ctx, node)
+		if err != nil {
+			return sendable{}, err
+		}
+		if !known {
+			before = nil
+		}
+		names := make([]string, 0, len(plan.Modules))
+		for _, m := range plan.Modules {
+			names = append(names, m.Module)
+		}
+		out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
+	}
+	return out, nil
+}
+
+// carriedBuilds is the build of each module a declaration carries, as a send records it (novox/hq
+// issue 259): the module's current build for each module in it, and for a module left out of it
+// (ADR 0163, rule 6) the build it was last sent, since the machine keeps that one — or nothing, when
+// that is not known. Never nil, so a send through here always records what it knows.
+func carriedBuilds(modules []string, leftOut map[string]string, current map[string]inventory.CurrentBuild,
+	before map[string]string) map[string]string {
+	out := map[string]string{}
+	for _, m := range modules {
+		if _, left := leftOut[m]; left {
+			if was, kept := before[m]; kept {
+				out[m] = was
+			}
+			continue
+		}
+		out[m] = current[m].Commit
+	}
+	return out
 }
 
 // sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
@@ -667,6 +707,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
 		return catalogue.Rendering{}, inventory.Node{}, err
 	}
 
+	// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
+	// lists every holder of the mesh's resolver.
+	replicas, err := replicatedHolders(ctx, inv, shelf)
+	if err != nil {
+		return catalogue.Rendering{}, inventory.Node{}, err
+	}
+
 	// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
 	// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
 	// before it had an address on the private network. A machine joins through the tunnel now, and
@@ -732,14 +779,21 @@ func renderingFor(ctx context.Context, open *stores, node string,
 	if err != nil {
 		return catalogue.Rendering{}, inventory.Node{}, err
 	}
+	// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
+	// 0222): the artifact store as this network reaches it, which the container runtime is told to
+	// trust (ADR 0082). The same address composed into every reference the mesh built.
+	var reach map[string]string
+	if artifactStore != "" {
+		reach = map[string]string{"mesh-artifact-store": artifactStore}
+	}
 	return catalogue.Rendering{
 		BusMembership: memberships[node],
 		Settings:      settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
 		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
-		Machines: machines, Zones: zones,
+		Machines: machines, Zones: zones, Holders: replicas,
 		Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
 		Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
-		Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
+		Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
 		BusUsers: busUsers,
 	}, record, nil
 }

cmd/mesh-controller/push.go

@@ -217,8 +217,9 @@ func declare(ctx context.Context, args []string) error {
 	}
 	// Written down like every other send (novox/hq issue 204): a declaration a person sent by hand
 	// is still what the machine was last told, and status must not read it as current for the one
-	// the mesh would compose.
-	if _, err := recordSent(ctx, inv, node, raw); err != nil {
+	// the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259):
+	// the mesh did not compose it, so a push that does not name this machine treats it as held.
+	if _, err := recordSent(ctx, inv, node, raw, nil); err != nil {
 		return err
 	}
 	fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
@@ -367,6 +368,23 @@ func pushCommand(ctx context.Context, args []string) error {
 	if err != nil {
 		return err
 	}
+	// **Not when its own modules are held back** (novox/hq issue 259, ADR 0221): added rather than
+	// named, it is sent its whole declaration, and a build its policy records or a plan has not sent
+	// it yet would go with the user list. Named and left, with what that costs.
+	saidHeld := map[string]bool{}
+	if holderBehind && len(args) == 1 {
+		held, err := heldMachines(ctx, open, []string{holder})
+		if err != nil {
+			return err
+		}
+		if why, isHeld := held[holder]; isHeld {
+			sayHeld(os.Stdout, holder, why)
+			fmt.Printf("%s holds the bus, and the user list it would carry has changed: until it is "+
+				"sent, the bus may refuse what this push's machines were newly granted\n", holder)
+			holderBehind = false
+			saidHeld[holder] = true
+		}
+	}
 	asked = brokerFirst(asked, holder, holderBehind)
 
 	// Held from composing to sending, so a converge on one of them cannot send between the two
@@ -418,76 +436,21 @@ func pushCommand(ctx context.Context, args []string) error {
 	//
 	// Compared against what each machine was last SENT, not against a before/after of this push:
 	// the mint usually happened at `assign` or `module issue`, before this command ran, so the
-	// only durable signal is "what it should be" versus "what it last received". A machine behind
-	// for an unrelated reason is caught here too, which is not a cost — a named push that knew a
-	// machine was behind and left it so would be the very silence this removes. Bounded: a
+	// only durable signal is "what it should be" versus "what it last received". Bounded: a
 	// flushed send may itself mint, so this converges over a few rounds.
+	//
+	// **Except a machine a policy or a plan holds back** (novox/hq issue 259, ADR 0221): one whose
+	// modules would move to a build their upgrade policy records rather than rolls out, or that an
+	// open plan has not sent it yet. It is named, with why, and left for a push that names it.
 	if len(args) == 1 {
-		flushed := map[string]bool{args[0]: true}
-		// Bounded by the node count: a node is marked flushed the round it is handled and is
-		// never handled twice, so the loop cannot run more than len(nodes) rounds. The bound is
-		// a guard against a logic error, not a real limit — if it were ever hit, that is a bug
-		// rather than a cascade legitimately still converging, so it is said rather than passed
-		// over in silence, unlike the earlier fixed cap that could stop a real cascade short.
-		rounds := 0
-		for {
-			would, err := wouldSend(ctx, open, nodes)
-			if err != nil {
-				return err
-			}
-			behind, err := inv.Waiting(ctx, would)
-			if err != nil {
-				return err
-			}
-			var also []string
-			for _, m := range behind {
-				if !flushed[m.Node] {
-					also = append(also, m.Node)
-				}
-			}
-			if len(also) == 0 {
-				break
-			}
-			if rounds++; rounds > len(nodes) {
-				fmt.Printf("\nstopped cascading after %d rounds with %s still behind — this "+
-					"should not happen; run `push --behind` to finish\n",
-					rounds-1, strings.Join(also, ", "))
-				break
-			}
-			sort.Strings(also)
-			fmt.Printf("\nthis push left %s behind — a provision granted from there, or a "+
-				"declaration since changed; sending it too\n", strings.Join(also, ", "))
-			// Tolerantly, exactly as the named send above: a machine that cannot be composed is
-			// collected as a refusal and reported at the end, and the others are still sent
-			// (novox/hq ADR 0066). The earlier cut routed these through sendTo, which is
-			// all-or-nothing — so one swept machine's compose error failed the operator's named
-			// push and skipped its --wait, the very intolerance the main path exists to avoid.
-			// Held for this round only, and after the last round's were given back, so two pushes
-			// cascading into each other's machines never each wait on the other.
-			refused, err := sendRound(ctx, open, also,
-				func(held context.Context, node string) (sendable, error) {
-					plan, settings, err := planFor(held, open, node)
-					if err != nil {
-						return sendable{}, err
-					}
-					reportUnhostable(node, plan)
-					declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
-					if err == nil {
-						reportLeftOut(node, declared)
-					}
-					return declared, err
-				},
-				bus, holder)
-			refusals = append(refusals, refused...)
-			if err != nil {
-				return err
-			}
-			// Every candidate this round is marked handled — the sent ones so they are not
-			// re-listed, and the refused ones so a machine that cannot be composed does not make
-			// the loop spin on it for ever. Its refusal is already in the report.
-			for _, name := range also {
-				flushed[name] = true
-			}
+		handled := map[string]bool{args[0]: true}
+		for n := range saidHeld {
+			handled[n] = true
+		}
+		refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, os.Stdout)
+		refusals = append(refusals, refused...)
+		if err != nil {
+			return err
 		}
 	}
 
@@ -762,7 +725,7 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
 	}
 	// After it is away, not before. A digest recorded for something that failed to send would make
 	// the machine look current for a declaration it never received.
-	digest, err := recordSent(ctx, b.open.inventory, s.node, body)
+	digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds)
 	if err != nil {
 		return "", err
 	}
@@ -1244,7 +1207,11 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, e
 // never wrote it down: status read "applied, current" over a machine that had just been sent
 // something else. What was sent was sent; the record of it must not depend on the sender living
 // another second. Bounded, so a store that is away does not hold a dying process open for ever.
-func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte) (string, error) {
+//
+// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
+// what tells a machine held back by a policy or a plan from one a push left behind.
+func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
+	builds map[string]string) (string, error) {
 	kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
 	defer cancel()
 	record, err := inv.NodeByName(kept, node)
@@ -1252,7 +1219,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
 		return "", err
 	}
 	digest := digestOf(body)
-	if err := inv.RecordSent(kept, record.ID, digest); err != nil {
+	if err := inv.RecordSent(kept, record.ID, digest, builds); err != nil {
 		return "", err
 	}
 	return digest, nil

cmd/mesh-controller/seats.go

@@ -29,11 +29,13 @@ type seatHolder struct {
 
 // seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
 type seatRow struct {
-	Seat     string       `json:"seat"`
-	Scope    string       `json:"scope"`
-	Delivers string       `json:"delivers,omitempty"`
-	Decision string       `json:"decision"`
-	Holders  []seatHolder `json:"holders"`
+	Seat     string `json:"seat"`
+	Scope    string `json:"scope"`
+	Delivers string `json:"delivers,omitempty"`
+	Decision string `json:"decision"`
+	// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
+	Replicated bool         `json:"replicated,omitempty"`
+	Holders    []seatHolder `json:"holders"`
 }
 
 // seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
@@ -47,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
 	rows := make([]seatRow, 0, len(seats))
 	for _, s := range seats {
 		row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
-			Holders: []seatHolder{}}
+			Replicated: s.Replicated, Holders: []seatHolder{}}
 		seen := map[seatHolder]bool{}
 		for _, h := range held {
 			// Resolve the held claim to a seat rather than comparing names, so a record naming a
@@ -104,9 +106,13 @@ func seatCommand(ctx context.Context, args []string) error {
 		return nil
 	}
 	if len(args) == 3 && args[1] == "--to" {
-		return handOver(ctx, args[0], args[2])
+		return handOver(ctx, args[0], args[2], false)
 	}
-	return fmt.Errorf("seat rename <from> <to>  |  seat <name> --to <node>/<module>")
+	if len(args) == 3 && args[1] == "--add" {
+		return handOver(ctx, args[0], args[2], true)
+	}
+	return fmt.Errorf("seat rename <from> <to>  |  seat <name> --to <node>/<module>  |  " +
+		"seat <name> --add <node>/<module>")
 }
 
 // handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
@@ -119,7 +125,12 @@ func seatCommand(ctx context.Context, args []string) error {
 // **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
 // and refusing to record a handover to a module the node has not started would make the handover
 // impossible to do before the switch instead of as the switch.
-func handOver(ctx context.Context, seatName, to string) error {
+//
+// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
+// records the named assignment as a further holder and leaves every holder on record as it is. A
+// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
+// the one named, as it always did.
+func handOver(ctx context.Context, seatName, to string, adding bool) error {
 	nodeName, module, ok := strings.Cut(to, "/")
 	if !ok || nodeName == "" || module == "" {
 		return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
@@ -135,6 +146,10 @@ func handOver(ctx context.Context, seatName, to string) error {
 	if !known {
 		return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
 	}
+	if adding && !seat.Replicated {
+		return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
+			"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
+	}
 	assigned, err := inv.Assigned(ctx, nodeName)
 	if err != nil {
 		return err
@@ -157,6 +172,7 @@ func handOver(ctx context.Context, seatName, to string) error {
 		return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
 	}
 	var was string
+	var held []string
 	holdings, err := inv.Holdings(ctx)
 	if err != nil {
 		return err
@@ -164,6 +180,7 @@ func handOver(ctx context.Context, seatName, to string) error {
 	for _, h := range holdings {
 		if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
 			was = h.Node
+			held = append(held, h.Node)
 		}
 	}
 
@@ -187,6 +204,15 @@ func handOver(ctx context.Context, seatName, to string) error {
 	} else if err := catalogue.CanHold(*m, seat); err != nil {
 		return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
 	}
+	if adding {
+		if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
+			return err
+		}
+		fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
+			strings.Join(held, ", "))
+		fmt.Printf("  `push --behind` re-declares every machine that reads the seat's holders\n")
+		return nil
+	}
 	if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
 		return err
 	}

cmd/mesh-controller/sendable.go

@@ -43,6 +43,10 @@ type sendable struct {
 	LeftOut []string
 	// leftOutWhy is why each was, for push and plan to say; never on the wire.
 	leftOutWhy map[string]string
+	// Builds is the build of each module this declaration carries — module to the commit its build
+	// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
+	// Composed only on the send path; nil records that it is not known.
+	Builds map[string]string
 }
 
 // adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on

internal/catalogue/computed_test.go

@@ -159,7 +159,8 @@ func TestTwoWaysToBeOnAPrivateNetworkRefuseAndNameBoth(t *testing.T) {
 	}
 }
 
-// reloading answers the runtime's trust as the networking module does (novox/hq ADR 0102): a file
+// reloading answers the runtime's trust as the networking module once did (novox/hq ADR 0102; it no
+// longer does, ADR 0222 — and beside the runtime's module it is refused, generated_collision_test): a file
 // written into, and the runtime reloaded on it.
 type reloading struct{}
 

internal/catalogue/declaration.go

@@ -159,6 +159,11 @@ type Rendering struct {
 	// 0199): the mesh's resolver forwards each one there.
 	Zones []ZoneAt
 
+	// Holders is, for each replicated mesh seat, every machine holding it, by internal name and
+	// private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver
+	// file lists: every holder of the mesh's resolver, this machine first if it is one.
+	Holders map[string]map[string]string
+
 	Settings   SettingsBy
 	Generators map[string]Generator
 	// Grants are the credentials this node must create, for the provisions it offers. Passed in
@@ -201,6 +206,11 @@ type Rendering struct {
 	// stored (novox/hq 04-ISSUES/102).
 	ArtifactStore string
 
+	// SeatReach is where this machine reaches the holder of each mesh-scoped seat it may be asked
+	// about (host:port), by seat: what ${seat:<seat>:reach} answers with (novox/hq ADR 0222). Absent
+	// when no holder is reachable yet; see seat_into.go for which seats are answered.
+	SeatReach map[string]string
+
 	// Built is every `<module>/<artifact>` the mesh has built. What tells a reference recorded
 	// with an address — before references were kept without one — from an image a module runs
 	// straight from a public registry.
@@ -426,6 +436,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 		return nil, err
 	}
 
+	generated, err := r.generatedHere(with)
+	if err != nil {
+		return nil, err
+	}
+
 	var out []map[string]any
 	for _, m := range r.Modules {
 		if with.Adopted && m.Filtering != nil {
@@ -692,23 +707,14 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 			first = append(first, file)
 		}
 		if m.Computed != "" {
-			generator, known := with.Generators[m.Computed]
-			if !known {
-				return nil, fmt.Errorf(
-					"%s says its resources are computed by %q, and this control plane has no %q",
-					m.Module, m.Computed, m.Computed)
-			}
-			generated, part, err := generator.Resources(r.Node)
-			if err != nil {
-				return nil, err
-			}
+			mine, part := generated[m.Module]
 			if !part {
 				// Assigned, and not yet part of what this generates. Nothing to put on the
 				// machine, which is different from an error: a node given the network module
 				// before it has an address is in exactly that state, briefly.
 				continue
 			}
-			resources = generated
+			resources = mine
 		}
 
 		// Now, and not before: a module whose resources are computed replaces them wholesale, and
@@ -979,7 +985,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 		// plane's; making a name resolve is the module's software. Emitted as ordinary files under
 		// this module's name, so they are applied, reported and removed exactly as anything else
 		// it declares.
-		given, err := FactsWithZonesInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix, with.Zones)
+		given, err := FactsFrom(m, r, with)
 		if err != nil {
 			return nil, err
 		}
@@ -2348,3 +2354,49 @@ func accountsFirst(resources []map[string]any) (accounts, rest []map[string]any)
 	}
 	return accounts, rest
 }
+
+// generatedHere is what each computed module's generator answers for this machine, by module —
+// absent for a module whose machine is not yet part of what it generates — held to the rule every
+// module is held to: no two modules on a machine declare one path, unit, name or package (novox/hq
+// issue 190, step 5; ADR 0222).
+//
+// Resolution checks the catalogue's manifests, and a computed module's manifest has none of the
+// resources it will declare — they exist only once its generator has answered for this machine,
+// here — so a generated resource writing into another module's file was never seen. That is how
+// the private network came to declare the container runtime's daemon file and service beside the
+// runtime's own module. Asked once, so what is checked is exactly what is declared.
+func (r Resolution) generatedHere(with Rendering) (map[string][]map[string]any, error) {
+	generated := map[string][]map[string]any{}
+	placed := make([]Manifest, 0, len(r.Modules))
+	for _, m := range r.Modules {
+		if m.Computed == "" {
+			placed = append(placed, m)
+			continue
+		}
+		generator, known := with.Generators[m.Computed]
+		if !known {
+			return nil, fmt.Errorf(
+				"%s says its resources are computed by %q, and this control plane has no %q",
+				m.Module, m.Computed, m.Computed)
+		}
+		resources, part, err := generator.Resources(r.Node)
+		if err != nil {
+			return nil, err
+		}
+		computed := m
+		computed.Resources = nil
+		if part {
+			generated[m.Module] = resources
+			computed.Resources = resources
+		}
+		placed = append(placed, computed)
+	}
+	if len(generated) == 0 {
+		return generated, nil // nothing resolution has not already judged
+	}
+	if problems := checkResources(placed); len(problems) > 0 {
+		return nil, fmt.Errorf("what the mesh computes for this machine collides with a module's own: %s",
+			strings.Join(problems, "; "))
+	}
+	return generated, nil
+}

internal/catalogue/generated_collision_test.go

@@ -0,0 +1,84 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// novox/hq issue 190, step 5 (ADR 0222): what the mesh computes for a module is held to the rule
+// every module is — no two modules on a machine declare one path, unit, name or package. The
+// private network once declared the container runtime's file and service beside the runtime's own
+// module, and nothing refused it, because the collision check only ever saw catalogue manifests.
+
+func runtimesOwn() Manifest {
+	return Manifest{Module: "docker", Resources: []map[string]any{
+		{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json", "into": "json",
+			"content": `{"live-restore": true}`},
+		{"id": "runtime", "type": "service", "unit": "docker.service", "state": "running",
+			"reload-on": []any{"daemon"}},
+	}}
+}
+
+func TestAGeneratedResourceCollidingWithAModulesIsRefused(t *testing.T) {
+	r := Resolution{Node: "workstation", Modules: []Manifest{
+		{Module: "mesh-network", Computed: "mesh-network", Provides: Offers("private-network")},
+		runtimesOwn(),
+	}}
+	_, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": reloading{}}})
+	if err == nil {
+		t.Fatal("a generated resource declaring the runtime's file beside the runtime's module was accepted")
+	}
+	for _, want := range []string{"mesh-network", "docker", "/etc/docker/daemon.json", "docker.service"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the refusal does not name %s: %v", want, err)
+		}
+	}
+}
+
+func TestAGeneratedResourceBesideAModulesOwnIsComposed(t *testing.T) {
+	r := Resolution{Node: "workstation", Modules: []Manifest{
+		{Module: "mesh-network", Computed: "mesh-network", Provides: Offers("private-network")},
+		runtimesOwn(),
+	}}
+	gen := &fake{on: map[string]bool{"workstation": true}}
+	out, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": gen}})
+	if err != nil {
+		t.Fatalf("disjoint resources were refused: %v", err)
+	}
+	if fileNamed(out, "docker.daemon") == nil {
+		t.Fatalf("the runtime's own file is missing: %v", out)
+	}
+	// And a machine not on the network yet generates nothing, which collides with nothing.
+	if _, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": &fake{}}}); err != nil {
+		t.Fatalf("a machine off the network was refused: %v", err)
+	}
+}
+
+// The catalogue's container runtime module states the mesh's registry itself (ADR 0222).
+func TestTheRuntimesModuleTrustsTheMeshsRegistry(t *testing.T) {
+	docker := catalogueManifest(t, "docker")
+	r := Resolution{Node: "workstation", Modules: []Manifest{docker}}
+	out, err := r.Declaration(Rendering{
+		SeatReach: map[string]string{"mesh-artifact-store": "anchor.internal:5100"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	daemon := fileNamed(out, "docker.daemon")
+	if daemon == nil || daemon["into"] != "json" {
+		t.Fatalf("the runtime's file is not written into: %v", daemon)
+	}
+	content, _ := daemon["content"].(string)
+	if !strings.Contains(content, `"insecure-registries": ["anchor.internal:5100"]`) ||
+		!strings.Contains(content, `"live-restore": true`) {
+		t.Fatalf("the runtime's file says %q", content)
+	}
+
+	out, err = r.Declaration(Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if content, _ := fileNamed(out, "docker.daemon")["content"].(string); strings.Contains(content, "insecure-registries") {
+		t.Fatalf("with no store on the network, the runtime is told %q", content)
+	}
+}

internal/catalogue/resolve.go

@@ -767,16 +767,27 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
 	var problems []string
 	var held []Held
 
-	// onRecord is the recorded holder of a seat, if a handover ever named one.
-	onRecord := func(claim, scope string) (Held, bool) {
+	// onRecord is every recorded holder of a seat, if a handover ever named one: one for most seats,
+	// and as many as were added for a replicated one (novox/hq ADR 0223).
+	onRecord := func(claim, scope string) []Held {
+		var out []Held
 		for _, h := range holdings {
 			hs, ok := SeatNamed(h.Claim)
 			cs, cok := SeatNamed(claim)
 			if ok && cok && hs.Name == cs.Name && h.Scope == scope {
-				return h, true
+				out = append(out, h)
 			}
 		}
-		return Held{}, false
+		return out
+	}
+	// recordedHere says this node's module is one of a seat's holders on record.
+	recordedHere := func(claim, scope, module string) bool {
+		for _, rec := range onRecord(claim, scope) {
+			if rec.Node == node.Name && rec.Module == module {
+				return true
+			}
+		}
+		return false
 	}
 
 	byScope := map[string]map[string]string{} // scope → claim → module
@@ -787,8 +798,18 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
 			// the seat but is not the one on record is eligible, and that is all: it is not a second
 			// holder, so it is not refused, and it does not hold (novox/hq ADR 0131). This is what
 			// lets the next holder stand beside the current one until the seat is handed over.
-			if rec, recorded := onRecord(c.Name, scope); recorded {
-				if rec.Node != node.Name || rec.Module != m.Module {
+			if recs := onRecord(c.Name, scope); len(recs) > 0 {
+				// **A seat held once is on record once** (novox/hq ADR 0223). Only a replicated seat
+				// may have several holders on record; several for any other seat is a store that
+				// disagrees with the mesh's definition of the role, and it is refused, named, rather
+				// than letting two machines answer for what the mesh has one of.
+				if s, known := SeatNamed(c.Name); known && !s.Replicated && len(recs) > 1 {
+					problems = append(problems, fmt.Sprintf(
+						"%q is on record as held by %d assignments, and it is held once per %s — "+
+							"`seat %s --to <node>/<module>` records one", c.Name, len(recs), scope, c.Name))
+					continue
+				}
+				if !recordedHere(c.Name, scope, m.Module) {
 					continue
 				}
 			}
@@ -815,6 +836,12 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
 			}
 			switch h.Scope {
 			case ScopeMesh:
+				// **A holder on record is never a second claimant** (novox/hq ADR 0223). Records are
+				// the mesh's settled answer: one for most seats, several only for a replicated seat,
+				// each added by an act. Two holders here are two records, and both hold.
+				if recordedHere(h.Claim, h.Scope, h.Module) {
+					continue
+				}
 				// Both claim and nobody is on record, or this refusal could not have happened.
 				// The remedy is the handover that records the holder (novox/hq ADR 0131,
 				// 04-ISSUES/170), so it is named here rather than left to be found.
@@ -1156,6 +1183,18 @@ func answeredElsewhere(want string, node Node, world World, brokered map[string]
 	if c, pinned := world.Pinned[want]; pinned {
 		return c.Node != node.Name
 	}
+	// **A machine holding the seat answers itself** (novox/hq ADR 0223). With a replicated seat
+	// another machine holds it too, and the first holder in the providers' order may be that one; a
+	// holder is still where this machine's own requirement is answered, so its resolver file lists
+	// itself first.
+	if seat, delivered := SeatDelivering(want); delivered {
+		for _, h := range append(append([]Held(nil), world.Holdings...), world.Held...) {
+			if hs, ok := SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope &&
+				h.Node == node.Name {
+				return false
+			}
+		}
+	}
 	holder, held := HolderAmong(want, world.Offered[want], world.Held)
 	return held && holder.Node != node.Name
 }

internal/catalogue/resolver_manifests_test.go

@@ -84,25 +84,24 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
 			t.Errorf("the mesh's resolver still reads or listens on %q", never)
 		}
 	}
-	// And the file that decides what the machine asks names the mesh's resolver first, by address,
-	// and a public one second, asked only when the first is silent (ADR 0196).
-	var resolv string
-	for _, r := range catalogueManifest(t, "resolv-conf").Resources {
-		if r["path"] == "/etc/resolv.conf" {
-			resolv, _ = r["content"].(string)
+	// And the file that decides what the machine asks names every one of the mesh's resolvers, by
+	// address, from the seat's holders, and no public one (ADR 0223): a resolver library that asks
+	// every listed server at once takes the first reply, and a public "no such name" for a mesh name
+	// won it.
+	fact, ok := catalogueManifest(t, "resolv-conf").Facts["resolvers"]
+	if !ok || fact.Path != "/etc/resolv.conf" {
+		t.Fatalf("the machine's resolver file is not rendered from the roster: %+v", fact)
+	}
+	if !strings.Contains(fact.Template, `{{range index .Holders "mesh-dns-resolver"}}nameserver {{.Address}}`) {
+		t.Errorf("resolv.conf does not list every holder of the mesh's resolver:\n%s", fact.Template)
+	}
+	for _, line := range strings.Split(fact.Template, "\n") {
+		if strings.HasPrefix(line, "nameserver ") && !strings.Contains(line, "{{") {
+			t.Errorf("resolv.conf names a resolver of its own beside the mesh's: %s", line)
 		}
 	}
-	var nameservers []string
-	for _, line := range strings.Split(resolv, "\n") {
-		if strings.HasPrefix(line, "nameserver ") {
-			nameservers = append(nameservers, strings.TrimPrefix(line, "nameserver "))
-		}
-	}
-	if len(nameservers) != 2 || nameservers[0] != "${bound:wildcard-resolution:address}" || nameservers[1] != "1.1.1.1" {
-		t.Errorf("resolv.conf names %v; the mesh's resolver by address first, a public one second", nameservers)
-	}
-	if !strings.Contains(resolv, "\noptions timeout:1 attempts:1") {
-		t.Errorf("the fallback is not reached after one short attempt:\n%s", resolv)
+	if !strings.Contains(fact.Template, "options timeout:1 attempts:2 edns0\n") {
+		t.Errorf("a silent resolver is not passed over after one short wait:\n%s", fact.Template)
 	}
 }
 
@@ -126,6 +125,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
 		// happen to be the same map, since nothing routed is part of it.
 		Names: twoMachines, Machines: twoMachines, Suffix: "internal",
 		Zones:    []ZoneAt{{Zone: "incus", Address: "10.42.0.2", Port: 5353}},
+		Holders:  map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1"}},
 		Needed:   map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
 		Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
 	})
@@ -205,9 +205,9 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
 		t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
 	}
 
-	resolv := ids["resolv-conf.resolv"]
-	if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\nnameserver 1.1.1.1\n") {
-		t.Fatalf("the machine is not pointed at the resolver by address, with the public fallback: %v", resolv)
+	resolv := ids["resolv-conf.fact-resolvers"]
+	if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\noptions ") {
+		t.Fatalf("the machine is not pointed at the resolver by address, and only at it: %v", resolv)
 	}
 }
 

internal/catalogue/roster.go

@@ -64,6 +64,12 @@ type rosterView struct {
 	// Zones is every zone a module in the mesh answers itself, with where its answerer is (novox/hq
 	// ADR 0199) — what the mesh's resolver forwards. Ordered by zone.
 	Zones []rosterZone
+	// Holders is the machines holding each replicated mesh seat, by seat (novox/hq ADR 0223) — what
+	// a machine's resolver file lists for `mesh-dns-resolver`. **This machine first when it is one
+	// of them**, then the rest by name: the nearest holder is asked first, and two renderings of
+	// one mesh on one machine are one file. A template reads one seat's with
+	// `index .Holders "<seat>"`; a seat nobody holds ranges over nothing.
+	Holders map[string][]rosterEntry
 }
 
 // rosterZone is one zone as a template sees it: the zone, and the address and port answering it.
@@ -96,6 +102,13 @@ func FactsInto(m Manifest, r Resolution, every, machines, accounts map[string]st
 // FactsWithZonesInto is FactsInto with the mesh's zones in the view, for a template that ranges them.
 func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[string]string, suffix string,
 	zones []ZoneAt) ([]map[string]any, error) {
+	return FactsFrom(m, r, Rendering{Names: every, Machines: machines, Accounts: accounts, Suffix: suffix,
+		Zones: zones})
+}
+
+// FactsFrom renders the roster files a module asked for from everything the mesh composed for this
+// machine: its machines, zones and the holders of each replicated seat.
+func FactsFrom(m Manifest, r Resolution, with Rendering) ([]map[string]any, error) {
 	if len(m.Facts) == 0 {
 		return nil, nil
 	}
@@ -107,10 +120,11 @@ func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[
 
 	view := rosterView{
 		Node:     r.Node,
-		Suffix:   strings.TrimPrefix(suffixOr(suffix), "."),
-		Names:    entriesFrom(every, accounts, suffix),
-		Machines: entriesFrom(machines, accounts, suffix),
-		Zones:    zonesFrom(zones),
+		Suffix:   strings.TrimPrefix(suffixOr(with.Suffix), "."),
+		Names:    entriesFrom(with.Names, with.Accounts, with.Suffix),
+		Machines: entriesFrom(with.Machines, with.Accounts, with.Suffix),
+		Zones:    zonesFrom(with.Zones),
+		Holders:  holdersFrom(with.Holders, with.Accounts, with.Suffix, r.Node),
 	}
 
 	out := make([]map[string]any, 0, len(names))
@@ -250,3 +264,18 @@ func zonesFrom(zones []ZoneAt) []rosterZone {
 	sort.Slice(out, func(i, j int) bool { return out[i].Zone < out[j].Zone })
 	return out
 }
+
+// holdersFrom is each replicated seat's holders as a template ranges them: this machine first when
+// it holds the seat, then the others by name (novox/hq ADR 0223). A machine with no address is left
+// out, as everywhere in the roster — a resolver named at nothing is a lookup that hangs.
+func holdersFrom(holders map[string]map[string]string, accounts map[string]string, suffix, node string) map[string][]rosterEntry {
+	out := make(map[string][]rosterEntry, len(holders))
+	for seat, at := range holders {
+		entries := entriesFrom(at, accounts, suffix)
+		sort.SliceStable(entries, func(i, j int) bool {
+			return entries[i].Name == node && entries[j].Name != node
+		})
+		out[seat] = entries
+	}
+	return out
+}

internal/catalogue/seat_into.go

@@ -1,6 +1,7 @@
 package catalogue
 
 import (
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"sort"
@@ -42,6 +43,27 @@ import (
 // ofSeat is where a module asks about a seat: ${seat:<seat>:<the port its holder's software uses>}.
 var ofSeat = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):([0-9]+)\}`)
 
+// **Where this machine reaches a mesh seat's holder** (novox/hq ADR 0222, issue 190).
+//
+// `${seat:<mesh-seat>:reach}` is host:port — the address this machine dials to reach whatever holds a
+// seat the mesh holds once. The same reasoning as the port above, one step further: nothing is
+// required, nothing is granted, no credential is minted, and the answer is an address the mesh
+// already holds in the clear and composes into every reference it built. What it is for is a module
+// that must *state* where a mesh service is to software it owns — the container runtime trusting
+// the mesh's registry is the case — without becoming that service's consumer.
+//
+// Answered for the seats in reachedSeats only. Another seat is refused by name rather than answered
+// with nothing: a module asking where something is that the mesh does not say would otherwise be
+// given an empty value and never know the question was not understood.
+//
+// The answer may be empty: no machine on the private network holds the seat yet (genesis raises
+// the store before the network). In a file written into as JSON, an empty member is dropped from
+// its list, and a list left with none is dropped, so the runtime is never told to trust "".
+var ofSeatReach = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):reach\}`)
+
+// reachedSeats is every seat ${seat:…:reach} answers for.
+var reachedSeats = map[string]bool{"mesh-artifact-store": true}
+
 // seatInto replaces a resource's ${seat:…} placeholders with where this machine put each seat's
 // holder — in a file's content, and in a value of a container's or a process's environment. The
 // same places portInto fills, for the same reason: they are where a program reads a number from.
@@ -49,13 +71,24 @@ func seatInto(resource map[string]any, module string, with Rendering) error {
 	switch fmt.Sprint(resource["type"]) {
 	case "file":
 		content, ok := resource["content"].(string)
-		if !ok || !ofSeat.MatchString(content) {
+		if !ok || (!ofSeat.MatchString(content) && !ofSeatReach.MatchString(content)) {
 			return nil
 		}
-		filled, err := seatsFilledInto(content, fmt.Sprintf("%s has a file that", module), with)
+		where := fmt.Sprintf("%s has a file that", module)
+		filled, err := seatsFilledInto(content, where, with)
 		if err != nil {
 			return err
 		}
+		if ofSeatReach.MatchString(filled) {
+			if filled, err = reachFilledInto(filled, where, with); err != nil {
+				return err
+			}
+			if fmt.Sprint(resource["into"]) == "json" {
+				if filled, err = withoutEmptyMembers(filled, where); err != nil {
+					return err
+				}
+			}
+		}
 		resource["content"] = filled
 
 	case "container", "process":
@@ -74,15 +107,18 @@ func seatInto(resource map[string]any, module string, with Rendering) error {
 		var filled map[string]any
 		for _, key := range named {
 			written, ok := env[key].(string)
-			if !ok || !ofSeat.MatchString(written) {
+			if !ok || (!ofSeat.MatchString(written) && !ofSeatReach.MatchString(written)) {
 				continue
 			}
-			value, err := seatsFilledInto(written,
-				fmt.Sprintf("%s's %s %s sets %s to something that",
-					module, resource["type"], resource["name"], key), with)
+			where := fmt.Sprintf("%s's %s %s sets %s to something that",
+				module, resource["type"], resource["name"], key)
+			value, err := seatsFilledInto(written, where, with)
 			if err != nil {
 				return err
 			}
+			if value, err = reachFilledInto(value, where, with); err != nil {
+				return err
+			}
 			if filled == nil {
 				filled = map[string]any{}
 				for k, v := range env {
@@ -118,3 +154,69 @@ func seatsFilledInto(written, where string, with Rendering) (string, error) {
 	}
 	return written, nil
 }
+
+// reachFilledInto answers every ${seat:…:reach} in one written value with where this machine
+// reaches the seat's holder, or with nothing when no holder is reachable yet. A seat the mesh does
+// not answer this for is refused by name.
+func reachFilledInto(written, where string, with Rendering) (string, error) {
+	for _, m := range ofSeatReach.FindAllStringSubmatch(written, -1) {
+		seat := m[1]
+		if !reachedSeats[seat] {
+			known := make([]string, 0, len(reachedSeats))
+			for s := range reachedSeats {
+				known = append(known, s)
+			}
+			sort.Strings(known)
+			return "", fmt.Errorf("%s says ${seat:%s:reach}, and the mesh says where a seat's holder is "+
+				"reached only for %s (novox/hq ADR 0222)", where, seat, strings.Join(known, ", "))
+		}
+		written = strings.ReplaceAll(written, m[0], with.SeatReach[seat])
+	}
+	return written, nil
+}
+
+// withoutEmptyMembers drops every empty string from the lists at the top of a JSON object written
+// into a machine's file, and a list left with no members, so an unanswered ${seat:…:reach} adds
+// nothing to the machine's list rather than adding "".
+func withoutEmptyMembers(content, where string) (string, error) {
+	var object map[string]json.RawMessage
+	if err := json.Unmarshal([]byte(content), &object); err != nil {
+		return "", fmt.Errorf("%s is written into as JSON and is not a JSON object: %w", where, err)
+	}
+	changed := false
+	for key, raw := range object {
+		var members []json.RawMessage
+		if err := json.Unmarshal(raw, &members); err != nil {
+			continue // not a list
+		}
+		kept := make([]json.RawMessage, 0, len(members))
+		for _, member := range members {
+			var s string
+			if json.Unmarshal(member, &s) == nil && s == "" {
+				continue
+			}
+			kept = append(kept, member)
+		}
+		if len(kept) == len(members) {
+			continue
+		}
+		changed = true
+		if len(kept) == 0 {
+			delete(object, key)
+			continue
+		}
+		list, err := json.Marshal(kept)
+		if err != nil {
+			return "", err
+		}
+		object[key] = list
+	}
+	if !changed {
+		return content, nil
+	}
+	out, err := json.Marshal(object)
+	if err != nil {
+		return "", err
+	}
+	return string(out) + "\n", nil
+}

internal/catalogue/seat_into_test.go

@@ -1,6 +1,7 @@
 package catalogue
 
 import (
+	"fmt"
 	"os"
 	"strings"
 	"testing"
@@ -211,3 +212,83 @@ func withSeatPorts(m Manifest) Manifest {
 	}
 	return out
 }
+
+// Where this machine reaches a mesh seat's holder (novox/hq ADR 0222, issue 190): the container
+// runtime's module tells the runtime to trust the mesh's registry without binding the store.
+
+func runtimeTrust() map[string]any {
+	return map[string]any{
+		"type": "file", "id": "daemon", "path": "/etc/docker/daemon.json", "into": "json",
+		"content": `{"live-restore": true, "insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n",
+	}
+}
+
+func TestASeatsReachIsWhereThisMachineReachesItsHolder(t *testing.T) {
+	file := runtimeTrust()
+	with := Rendering{SeatReach: map[string]string{"mesh-artifact-store": "anchor.internal:5100"}}
+	if err := seatInto(file, "docker", with); err != nil {
+		t.Fatal(err)
+	}
+	want := `{"live-restore": true, "insecure-registries": ["anchor.internal:5100"]}` + "\n"
+	if file["content"] != want {
+		t.Fatalf("content = %q, want %q", file["content"], want)
+	}
+
+	process := map[string]any{"type": "process", "name": "p",
+		"env": map[string]any{"REGISTRY": "${seat:mesh-artifact-store:reach}"}}
+	if err := seatInto(process, "x", with); err != nil {
+		t.Fatal(err)
+	}
+	if got := process["env"].(map[string]any)["REGISTRY"]; got != "anchor.internal:5100" {
+		t.Fatalf("an environment value was filled with %q", got)
+	}
+}
+
+// With no holder reachable, the runtime is not told to trust "": the member is dropped, and the
+// list with it when nothing else is in it.
+func TestAnUnansweredReachAddsNothingToAList(t *testing.T) {
+	file := runtimeTrust()
+	if err := seatInto(file, "docker", Rendering{}); err != nil {
+		t.Fatal(err)
+	}
+	if got := file["content"]; got != `{"live-restore":true}`+"\n" {
+		t.Fatalf("with no store reachable, the runtime's keys are %q", got)
+	}
+
+	kept := map[string]any{"type": "file", "into": "json",
+		"content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}", "192.0.2.7:5000"]}`}
+	if err := seatInto(kept, "docker", Rendering{}); err != nil {
+		t.Fatal(err)
+	}
+	if got := kept["content"]; got != `{"insecure-registries":["192.0.2.7:5000"]}`+"\n" {
+		t.Fatalf("a list's other members were not kept: %q", got)
+	}
+}
+
+func TestAReachTheMeshDoesNotAnswerIsRefused(t *testing.T) {
+	file := map[string]any{"type": "file", "content": "${seat:mesh-store:reach}"}
+	err := seatInto(file, "x", Rendering{SeatReach: map[string]string{"mesh-store": "anchor.internal:5432"}})
+	if err == nil || !strings.Contains(err.Error(), "mesh-artifact-store") {
+		t.Fatalf("a reach the mesh does not answer was not refused by name: %v", err)
+	}
+}
+
+// Through the whole composition, as the container runtime's module declares it.
+func TestTheRuntimesTrustIsComposedFromTheSeatsReach(t *testing.T) {
+	r := Resolution{Node: "anchor", Modules: []Manifest{{
+		Module: "docker", Resources: []map[string]any{runtimeTrust()},
+	}}}
+	out, err := r.Declaration(Rendering{
+		SeatReach: map[string]string{"mesh-artifact-store": "anchor.internal:5100"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	file := fileNamed(out, "docker.daemon")
+	if file == nil {
+		t.Fatalf("no file in %v", out)
+	}
+	if got := file["content"]; !strings.Contains(fmt.Sprint(got), `["anchor.internal:5100"]`) {
+		t.Fatalf("the runtime's file says %q", got)
+	}
+}

internal/catalogue/seats.go

@@ -21,8 +21,16 @@ import (
 type Seat struct {
 	// Name is what a manifest claims.
 	Name string
-	// Scope is where there may be only one holder.
+	// Scope is where there may be only one holder — unless the seat is Replicated.
 	Scope string
+	// Replicated says a mesh seat may be held on several machines at once, each holder answering the
+	// same thing (novox/hq ADR 0223): the mesh's resolver, held on the anchor and the home server so a
+	// machine's resolver file lists two that give one answer. Each holder is on record, added by an
+	// act (`seat <name> --add <node>/<module>`), never by being assigned: two claimants with nothing on
+	// record are refused exactly as for any mesh seat. One per machine still — two modules on one
+	// node claiming it are refused. Compiled, never stored, like Needs: it is the mesh's definition of
+	// the role, and the store's rows carry no column for it.
+	Replicated bool
 	// Delivers is the provision the seat's holder answers for, or empty. A seat that delivers a
 	// provision may only be held by a module providing it at the seat's scope, and its holder is
 	// what a requirement for that provision resolves to when several modules provide it.
@@ -140,10 +148,14 @@ var defaultSeats = append([]Seat{
 	// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
 	{Name: "mesh-build-machine", Scope: ScopeMesh,
 		Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
-	// **The mesh's one resolver** (novox/hq ADR 0194, 0196): every node's internal domain, held in one
-	// place, and every node and container asks it first. Delivers what a machine's resolver
-	// configuration requires, so that requirement resolves to the holder wherever it is placed.
-	{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Decision: "novox/hq ADR 0194"},
+	// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
+	// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
+	// than one machine, each answering the same names from the same roster, and every machine's
+	// resolver file lists every holder — its own first — and no public resolver, so whichever answers
+	// first gives the one answer. Delivers what a machine's resolver configuration requires, so that
+	// requirement resolves to a holder wherever they are placed.
+	{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Replicated: true,
+		Decision: "novox/hq ADR 0194, ADR 0223"},
 	// **A machine's /etc/hosts is one module's** (novox/hq ADR 0199): its holder writes the machine's
 	// own lines and keeps every other line as the operator's, changed through these three verbs on that
 	// machine alone. The controller holds none of it.
@@ -314,11 +326,12 @@ func UseSeats(s []Seat) {
 				row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
 			}
 		}
-		// What a seat receives and what its holder needs are never stored (novox/hq ADR 0212, ADR
-		// 0220), so they are always the compiled ones.
+		// What a seat receives, what its holder needs and whether it is replicated are never stored
+		// (novox/hq ADR 0212, ADR 0220, ADR 0223), so they are always the compiled ones.
 		if d, known := byName[row.Name]; known {
 			row.Receives = d.Receives
 			row.Needs = d.Needs
+			row.Replicated = d.Replicated
 		}
 		merged = append(merged, row)
 	}
@@ -498,19 +511,24 @@ func seatNames() string {
 // two modules on one node could both provide a provision, and only the one holding the seat
 // answers for it. Nothing when no seat delivers the provision, when nobody holds
 // it, or when the holder is not among the providers offered.
+//
+// **The first holder in the providers' own order** (novox/hq ADR 0223). A replicated seat has
+// several, and the answer must not depend on the order the mesh happened to resolve its machines
+// in: the providers come sorted by machine, so every consumer is bound to the same one. A seat with
+// one holder gets the same answer as before.
 func HolderAmong(provision string, providers []Provider, held []Held) (Provider, bool) {
 	seat, delivered := SeatDelivering(provision)
 	if !delivered {
 		return Provider{}, false
 	}
-	for _, h := range held {
-		// Resolve the held claim to a seat rather than comparing names, so a record naming a seat's
-		// former name still matches it after a rename (novox/hq ADR 0122).
-		hs, ok := SeatNamed(h.Claim)
-		if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
-			continue
-		}
-		for _, p := range providers {
+	for _, p := range providers {
+		for _, h := range held {
+			// Resolve the held claim to a seat rather than comparing names, so a record naming a
+			// seat's former name still matches it after a rename (novox/hq ADR 0122).
+			hs, ok := SeatNamed(h.Claim)
+			if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
+				continue
+			}
 			if p.Node == h.Node && p.Module == h.Module {
 				return p, true
 			}

internal/catalogue/two_resolvers_test.go

@@ -0,0 +1,199 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// The mesh has two resolvers (novox/hq ADR 0223): `mesh-dns-resolver` is replicated, held on the
+// anchor and on the home server, each answering the same names; every machine's resolver file lists
+// every holder — its own first when it is one — and no public resolver. ADR 0196 listed the mesh's
+// resolver then a public one, and musl asks both at once and takes the first reply: from the home
+// server the public "no such name" for the anchor's mesh name won, every time, in every Alpine build.
+
+// resolverMachines is the anchor and the home server holding the resolver, and a laptop holding nothing.
+var resolverMachines = map[string]string{
+	"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2", "home.internal": "10.42.0.3"}
+
+// bothResolvers is the two holders on record, as `seat mesh-dns-resolver --add` leaves them.
+var bothResolvers = []Held{
+	{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "anchor", Module: "dnsmasq"},
+	{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "home", Module: "dnsmasq"},
+}
+
+// twoResolverShelf is the resolver, what asks it, and a stand-in answering `mesh-addressing`.
+func twoResolverShelf(t *testing.T) map[string]Manifest {
+	t.Helper()
+	return map[string]Manifest{
+		"net":         {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
+		"dnsmasq":     catalogueManifest(t, "dnsmasq"),
+		"resolv-conf": catalogueManifest(t, "resolv-conf"),
+	}
+}
+
+// worldWithout is the rest of the mesh as a plan for one machine sees it: every other holder's claim
+// and offer, and both holders on record.
+func worldWithout(node string) World {
+	w := World{Holdings: bothResolvers, Offered: map[string][]Provider{}}
+	for _, h := range bothResolvers {
+		if h.Node == node {
+			continue
+		}
+		w.Held = append(w.Held, h)
+		w.Offered["wildcard-resolution"] = append(w.Offered["wildcard-resolution"],
+			Provider{Node: h.Node, At: h.Node + ".internal", Module: h.Module})
+	}
+	return w
+}
+
+// resolvConfOn resolves and composes one machine and answers with the nameservers its resolver file
+// lists, in order, and the file.
+func resolvConfOn(t *testing.T, node string, assigned []string) ([]string, string) {
+	t.Helper()
+	got, err := Resolve(twoResolverShelf(t), assigned, Node{Name: node, At: node + ".internal"}, worldWithout(node))
+	if err != nil {
+		t.Fatalf("%s does not resolve with two resolvers on record: %v", node, err)
+	}
+	out, err := got.Declaration(Rendering{
+		Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
+		Holders: map[string]map[string]string{"mesh-dns-resolver": {
+			"anchor.internal": "10.42.0.1", "home.internal": "10.42.0.3"}},
+		Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
+	})
+	if err != nil {
+		t.Fatalf("%s does not compose: %v", node, err)
+	}
+	file := byID(out)["resolv-conf.fact-resolvers"]
+	if file == nil || file["path"] != "/etc/resolv.conf" {
+		t.Fatalf("%s was given no resolver file: %v", node, file)
+	}
+	content, _ := file["content"].(string)
+	var servers []string
+	for _, line := range strings.Split(content, "\n") {
+		if strings.HasPrefix(line, "nameserver ") {
+			servers = append(servers, strings.TrimPrefix(line, "nameserver "))
+		}
+	}
+	return servers, content
+}
+
+func TestTwoResolversComposeOnBothHoldersAndEachListsItselfFirst(t *testing.T) {
+	for node, want := range map[string][]string{
+		"anchor": {"10.42.0.1", "10.42.0.3"},
+		"home":   {"10.42.0.3", "10.42.0.1"},
+	} {
+		servers, content := resolvConfOn(t, node, []string{"dnsmasq", "resolv-conf"})
+		if strings.Join(servers, " ") != strings.Join(want, " ") {
+			t.Errorf("%s lists %v; itself first, then the other holder: %v\n%s", node, servers, want, content)
+		}
+	}
+}
+
+func TestAMachineHoldingNoResolverListsBothAndNoPublicOne(t *testing.T) {
+	servers, content := resolvConfOn(t, "laptop", []string{"resolv-conf"})
+	if strings.Join(servers, " ") != "10.42.0.1 10.42.0.3" {
+		t.Errorf("the laptop lists %v; every holder, by name, and nothing else:\n%s", servers, content)
+	}
+	for _, public := range []string{"1.1.1.1", "8.8.8.8", "9.9.9.9"} {
+		if strings.Contains(content, public) {
+			t.Errorf("a public resolver is listed beside the mesh's (ADR 0223):\n%s", content)
+		}
+	}
+}
+
+// A holder answers its own requirement, even though the other holder sorts first (issue 258 kept).
+func TestAHolderAnswersItsOwnRequirement(t *testing.T) {
+	got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq", "resolv-conf"},
+		Node{Name: "home", At: "home.internal"}, worldWithout("home"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, n := range got.Needs {
+		if n.Name == "wildcard-resolution" && n.From != "home" {
+			t.Errorf("the home server's resolver configuration is bound to %s; it holds the seat itself", n.From)
+		}
+	}
+}
+
+// A seat held once is still held once: a second claimant on another machine is refused while
+// nothing is on record, and a store recording two holders of it is refused, naming the seat.
+func TestASingleHolderMeshSeatStillRefusesASecondHolder(t *testing.T) {
+	store := shelf(mod("postgres", nil, nil, nil, Claim{Name: "mesh-store", Scope: ScopeMesh}))
+	other := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: "anchor", Module: "postgres"}
+	if _, err := Resolve(store, []string{"postgres"}, workstation(), World{Held: []Held{other}}); err == nil ||
+		!strings.Contains(err.Error(), "one per mesh") {
+		t.Errorf("a second claimant of a seat held once was not refused: %v", err)
+	}
+	here := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: workstation().Name, Module: "postgres"}
+	_, err := Resolve(store, []string{"postgres"}, workstation(),
+		World{Held: []Held{other}, Holdings: []Held{other, here}})
+	if err == nil || !strings.Contains(err.Error(), "on record as held by 2") {
+		t.Errorf("two holders on record for a seat held once were not refused: %v", err)
+	}
+}
+
+// Replicated is not "whoever is assigned": two claimants with nothing on record are refused, as for
+// any mesh seat, and each holder is added by an act.
+func TestTwoUnrecordedClaimantsOfTheReplicatedSeatAreRefused(t *testing.T) {
+	w := worldWithout("home")
+	w.Holdings = nil
+	_, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "home", At: "home.internal"}, w)
+	if err == nil || !strings.Contains(err.Error(), "seat mesh-dns-resolver --to") {
+		t.Errorf("a second resolver with nothing on record was not refused, naming the handover: %v", err)
+	}
+}
+
+// Only the mesh's resolver is replicated: a seat being replicated is a decision, recorded.
+func TestOnlyTheResolverIsReplicated(t *testing.T) {
+	for _, s := range Seats() {
+		if s.Replicated != (s.Name == "mesh-dns-resolver") {
+			t.Errorf("%s replicated = %v; only mesh-dns-resolver is (ADR 0223)", s.Name, s.Replicated)
+		}
+	}
+	UseSeats([]Seat{{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution"}})
+	defer UseSeats(DefaultSeats())
+	if s, _ := SeatNamed("mesh-dns-resolver"); !s.Replicated {
+		t.Error("loading the set from the store, which has no column for it, lost the resolver's replication")
+	}
+}
+
+// Every consumer is bound to the same holder whatever order the mesh resolved its machines in.
+func TestTheFirstHolderIsTheFirstProvider(t *testing.T) {
+	providers := []Provider{{Node: "anchor", Module: "dnsmasq"}, {Node: "home", Module: "dnsmasq"}}
+	for _, held := range [][]Held{bothResolvers, {bothResolvers[1], bothResolvers[0]}} {
+		if p, ok := HolderAmong("wildcard-resolution", providers, held); !ok || p.Node != "anchor" {
+			t.Errorf("held in order %v answered %v", held, p)
+		}
+	}
+}
+
+// One host record per machine, beside its wildcard (novox/hq issue 262): a name with a host record
+// says it exists and has no IPv6 address, where the wildcard alone said there is no such name, and
+// musl reads that as final.
+func TestTheResolverHasOneHostRecordPerMachine(t *testing.T) {
+	got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "anchor", At: "anchor.internal"},
+		World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := got.Declaration(Rendering{Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
+		Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	content, _ := byID(out)["dnsmasq.fact-node-zones"]["content"].(string)
+	records := map[string]int{}
+	for _, line := range strings.Split(content, "\n") {
+		if strings.HasPrefix(line, "host-record=") {
+			records[strings.TrimPrefix(line, "host-record=")]++
+		}
+	}
+	for name, at := range resolverMachines {
+		if records[name+","+at] != 1 {
+			t.Errorf("%s has %d host records at %s, and has one:\n%s", name, records[name+","+at], at, content)
+		}
+	}
+	if len(records) != len(resolverMachines) {
+		t.Errorf("%d host records for %d machines:\n%s", len(records), len(resolverMachines), content)
+	}
+}

internal/inventory/catalogue.go

@@ -1158,6 +1158,37 @@ func (i *Inventory) SetUpgradeOf(ctx context.Context, module string, u Upgrade)
 	return nil
 }
 
+// CurrentBuild is the build a module is at and whether its policy rolls a new one out (novox/hq
+// issue 259).
+type CurrentBuild struct {
+	// Commit is the commit the module's manifest was read at — its `built_from` — and empty for a
+	// module held without a source.
+	Commit string
+	// RollOut is the module's upgrade policy, as Upgrade.RollOut.
+	RollOut bool
+}
+
+// CurrentBuilds is every module's current build and upgrade policy, in one read: what a send records
+// it carried, and what a push compares a machine's last send against.
+func (i *Inventory) CurrentBuilds(ctx context.Context) (map[string]CurrentBuild, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select name, coalesce(built_from, ''), upgrade = 'roll-out' from module`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string]CurrentBuild{}
+	for rows.Next() {
+		var name string
+		var b CurrentBuild
+		if err := rows.Scan(&name, &b.Commit, &b.RollOut); err != nil {
+			return nil, err
+		}
+		out[name] = b
+	}
+	return out, rows.Err()
+}
+
 // Running is every machine assigned a module, in a stable order.
 //
 // **Assigned, not reported.** A machine that is assigned the module and has not applied it yet is

internal/inventory/doing_test.go

@@ -189,7 +189,7 @@ func TestAMachineIsWaitingWhenWhatItWasSentIsNotWhatItShouldBe(t *testing.T) {
 	}
 
 	// Sent what it should be: not waiting.
-	if err := inv.RecordSent(ctx, anchor.ID, "aaa"); err != nil {
+	if err := inv.RecordSent(ctx, anchor.ID, "aaa", nil); err != nil {
 		t.Fatal(err)
 	}
 	waiting, err = inv.Waiting(ctx, map[string]string{"anchor": "aaa", "laptop": "bbb"})
@@ -234,7 +234,7 @@ func TestAMachineWithNothingComputedForItIsNotWaiting(t *testing.T) {
 	// It has been sent something before, which is what makes this the case the guard is for: a
 	// machine with a digest and nothing computed for it would compare against the empty string
 	// and look out of date, when the truth is that nobody worked out what it should be.
-	if err := inv.RecordSent(ctx, node.ID, "what-it-got-last-time"); err != nil {
+	if err := inv.RecordSent(ctx, node.ID, "what-it-got-last-time", nil); err != nil {
 		t.Fatal(err)
 	}
 	waiting, err := inv.Waiting(ctx, map[string]string{})

internal/inventory/holdings_test.go

@@ -97,3 +97,57 @@ func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T)
 		t.Fatalf("a re-told membership did not replace the first: %v", got)
 	}
 }
+
+// A replicated seat has several holders on record (novox/hq ADR 0223): each is added beside the
+// others, adding one twice changes nothing, a handover still leaves exactly one, and unassigning one
+// takes only its own row.
+func TestAReplicatedSeatHasSeveralHoldersOnRecord(t *testing.T) {
+	resolver := catalogue.Manifest{Module: "resolver", Version: "1",
+		Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
+		Claims:   []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}
+	inv, ctx := aMeshWith(t, resolver)
+	if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
+		t.Fatal(err)
+	}
+	for _, n := range []string{"anchor", "home"} {
+		if _, err := inv.AddNode(ctx, n); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := inv.Assign(ctx, n, "resolver"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "anchor", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	for range 2 {
+		if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	held, err := inv.Holdings(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(held) != 2 || held[0].Node != "anchor" || held[1].Node != "home" {
+		t.Fatalf("the two holders are not both on record, once each: %+v", held)
+	}
+	if err := inv.Unassign(ctx, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "anchor" {
+		t.Fatalf("unassigning one holder took more or less than its own row: %+v", held)
+	}
+	if _, err := inv.Assign(ctx, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "home" {
+		t.Fatalf("a handover left other holders on record: %+v", held)
+	}
+}

internal/inventory/migrations/0061-a-send-records-the-builds-it-carried.sql

@@ -0,0 +1,14 @@
+-- A send records the build of each module it carried (novox/hq issue 259, ADR 0221).
+--
+-- A named push ends by sending every other machine whose declaration differs from what it was last
+-- sent (ADR 0083). Read from the declaration's digest alone, a module whose upgrade policy records
+-- rather than rolls out, or whose plan sends one machine first (ADR 0218), made every machine running
+-- it differ, so `push <one machine>` sent all of them the build the policy was holding back. Which
+-- build of each module a machine was last sent is what tells a held upgrade from a consequence of the
+-- push, and it is not in a digest.
+--
+-- Module name to the commit its build was made from — the module's `built_from` when the declaration
+-- was composed, empty for a module the mesh holds without a source. NULL for a machine last sent
+-- before this was kept, or sent a declaration by hand: what it carried is not known, and the push
+-- treats such a machine as held until it is pushed by name.
+alter table node add column sent_builds jsonb;

internal/inventory/migrations/0062-a-replicated-seat-has-several-holders-on-record.sql

@@ -0,0 +1,14 @@
+-- A replicated seat has several holders on record (novox/hq ADR 0223).
+--
+-- 0039 recorded one holder per seat, keyed by the seat: a handover replaced the row, so the seat was
+-- never without a holder in between. The mesh's resolver is now held on more than one machine, each
+-- answering the same names, and each of those holders is recorded — added by `seat <name> --add`,
+-- never by being assigned. So a holding is keyed by the seat and the assignment holding it.
+--
+-- Nothing else changes. A handover (`seat <name> --to`) still leaves exactly one row, replacing every
+-- holder in one transaction; whether a seat may have more than one is the seat's compiled definition,
+-- judged by the controller before a row is added, and a store holding two for any other seat is
+-- refused at resolution, naming the seat. Every existing row is one per seat, so it satisfies the new
+-- key as it stands.
+alter table seat_holding drop constraint seat_holding_pkey;
+alter table seat_holding add primary key (seat, node, module);

internal/inventory/nodes.go

@@ -909,18 +909,53 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
 	return out, rows.Err()
 }
 
-// RecordSent keeps a digest of the declaration a machine was last sent.
+// RecordSent keeps a digest of the declaration a machine was last sent, and the build of each module
+// it carried.
 //
 // **A digest rather than the declaration.** The mesh can compute what a machine should be at any
 // moment; keeping a copy would be a second account of it, able to disagree with the first. What
 // cannot be recomputed is what was *actually sent*, and that is the whole difference between a
 // machine that is out of date and one that has never been told.
-func (i *Inventory) RecordSent(ctx context.Context, node, digest string) error {
+//
+// **And which build of each module** (novox/hq issue 259, ADR 0221): module name to the commit its
+// build was made from. A digest cannot say whether a machine differs because a module moved to a build
+// its upgrade policy holds back, or because of something a push made — a grant — and only the second
+// is a push's to send to a machine it did not name. Nil records that it is not known, as for a
+// declaration sent by hand.
+func (i *Inventory) RecordSent(ctx context.Context, node, digest string, builds map[string]string) error {
+	var carried *string
+	if builds != nil {
+		raw, err := json.Marshal(builds)
+		if err != nil {
+			return err
+		}
+		text := string(raw)
+		carried = &text
+	}
 	_, err := i.store.Pool().Exec(ctx,
-		`update node set sent = $2, sent_at = now() where id = $1`, node, digest)
+		`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb where id = $1`, node, digest, carried)
 	return err
 }
 
+// SentBuilds is the build of each module a machine was last sent, by its name: module to the commit
+// its build was made from (novox/hq issue 259). Known is false when that was not kept — a machine
+// last sent before it was, sent a declaration by hand, or one the mesh does not know.
+func (i *Inventory) SentBuilds(ctx context.Context, name string) (builds map[string]string, known bool, err error) {
+	var raw []byte
+	err = i.store.Pool().QueryRow(ctx, `select sent_builds from node where name = $1`, name).Scan(&raw)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return nil, false, nil
+	}
+	if err != nil || raw == nil {
+		return nil, false, err
+	}
+	builds = map[string]string{}
+	if err := json.Unmarshal(raw, &builds); err != nil {
+		return nil, false, err
+	}
+	return builds, true, nil
+}
+
 // RecordSentBusUsers keeps a digest of the bus's user list a machine was just sent, by its name
 // (novox/hq issue 249): whether the machine holding the bus must go first is whether this differs
 // from the list composed now.

internal/inventory/seats.go

@@ -215,23 +215,50 @@ func (i *Inventory) RenameSeat(ctx context.Context, from, to string) error {
 	return nil
 }
 
-// HoldSeat records that one assignment holds a seat, replacing whoever held it — as one write, so
-// the seat is never without a holder in between (novox/hq ADR 0131, design 28 task 5.3). The
-// assignment must exist; the store refuses otherwise, and that refusal is the right one: a seat
-// cannot be handed to something that is not running anywhere.
+// HoldSeat records that one assignment holds a seat, replacing whoever held it — every holder, for a
+// replicated seat (novox/hq ADR 0223) — as one transaction, so the seat is never without a holder in
+// between (novox/hq ADR 0131, design 28 task 5.3). The assignment must exist; the store refuses
+// otherwise, and that refusal is the right one: a seat cannot be handed to something that is not
+// running anywhere.
 func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module string) error {
 	node, err := i.NodeByName(ctx, nodeName)
 	if err != nil {
 		return err
 	}
-	_, err = i.store.Pool().Exec(ctx,
-		`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
-		 on conflict (seat) do update set scope = excluded.scope, node = excluded.node,
-		   module = excluded.module, since = now()`,
-		seat, scope, node.ID, module)
+	tx, err := i.store.Pool().Begin(ctx)
 	if err != nil {
+		return err
+	}
+	defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
+	if _, err := tx.Exec(ctx,
+		`delete from seat_holding where seat = $1 and not (node = $2 and module = $3)`,
+		seat, node.ID, module); err != nil {
+		return fmt.Errorf("handing %s to %s on %s: %w", seat, module, nodeName, err)
+	}
+	if _, err := tx.Exec(ctx,
+		`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
+		 on conflict (seat, node, module) do update set scope = excluded.scope, since = now()`,
+		seat, scope, node.ID, module); err != nil {
 		return fmt.Errorf("recording %s on %s as the holder of %s: %w", module, nodeName, seat, err)
 	}
+	return tx.Commit(ctx)
+}
+
+// AddSeatHolder records one more assignment holding a replicated seat, beside those already on
+// record (novox/hq ADR 0223). Whether the seat may have several holders is the caller's to judge —
+// the seat's definition is compiled, and the store holds no column for it. Recording a holder
+// already on record changes nothing.
+func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, module string) error {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return err
+	}
+	if _, err := i.store.Pool().Exec(ctx,
+		`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
+		 on conflict (seat, node, module) do nothing`,
+		seat, scope, node.ID, module); err != nil {
+		return fmt.Errorf("adding %s on %s as a holder of %s: %w", module, nodeName, seat, err)
+	}
 	return nil
 }
 
@@ -240,7 +267,7 @@ func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module
 func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
 	rows, err := i.store.Pool().Query(ctx,
 		`select h.seat, h.scope, n.name, h.module, coalesce(n.site, '')
-		   from seat_holding h join node n on n.id = h.node order by h.seat`)
+		   from seat_holding h join node n on n.id = h.node order by h.seat, n.name, h.module`)
 	if err != nil {
 		return nil, err
 	}

internal/inventory/sent_builds_test.go

@@ -0,0 +1,81 @@
+package inventory
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// novox/hq issue 259: a send keeps which build of each module it carried. A machine never sent
+// anything, or sent with nothing recorded — before this was kept, or by hand — is not known, which
+// is not the same as having been sent nothing.
+func TestASendKeepsTheBuildsItCarried(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	node, err := inv.AddNode(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil {
+		t.Fatalf("a machine never sent anything has known builds %v (%v): %v", builds, known, err)
+	}
+
+	carried := map[string]string{"resolver": "abc123", "network": ""}
+	if err := inv.RecordSent(ctx, node.ID, "d1", carried); err != nil {
+		t.Fatal(err)
+	}
+	builds, known, err := inv.SentBuilds(ctx, "anchor")
+	if err != nil || !known || !reflect.DeepEqual(builds, carried) {
+		t.Fatalf("the builds sent were not kept: %v %v %v", builds, known, err)
+	}
+
+	// Sent with nothing carried: known, and empty.
+	if err := inv.RecordSent(ctx, node.ID, "d2", map[string]string{}); err != nil {
+		t.Fatal(err)
+	}
+	if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || !known || len(builds) != 0 {
+		t.Fatalf("an empty send: %v %v %v", builds, known, err)
+	}
+
+	// Sent by hand: not known, and the digest still recorded.
+	if err := inv.RecordSent(ctx, node.ID, "d3", nil); err != nil {
+		t.Fatal(err)
+	}
+	if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil {
+		t.Fatalf("a send whose builds are not known read as %v %v: %v", builds, known, err)
+	}
+	if sent, err := inv.Outstanding(ctx, "anchor"); err != nil || sent != "d3" {
+		t.Fatalf("the digest was not recorded with it: %q %v", sent, err)
+	}
+
+	if _, known, err := inv.SentBuilds(ctx, "nobody"); err != nil || known {
+		t.Fatalf("a machine the mesh does not know: %v %v", known, err)
+	}
+}
+
+// A module's current build is the commit its manifest was read at, with its upgrade policy.
+func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "resolver", Version: "1"},
+		Source{Repository: "novox/mesh-catalog", BuiltFrom: "c1"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "by-hand", Version: "1"}, Source{}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.SetUpgradeOf(ctx, "by-hand", Upgrade{RollOut: true}); err != nil {
+		t.Fatal(err)
+	}
+	current, err := inv.CurrentBuilds(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := current["resolver"]; got != (CurrentBuild{Commit: "c1"}) {
+		t.Errorf("resolver is at %+v", got)
+	}
+	if got := current["by-hand"]; got != (CurrentBuild{RollOut: true}) {
+		t.Errorf("a module with no source is at %+v", got)
+	}
+}

internal/link/heard_test.go

@@ -100,7 +100,7 @@ func TestABareAliveDoesNotWipeTheDeclarationThatSaysANodeIsCurrent(t *testing.T)
 	}
 	// The mesh sent this node a declaration, and the node applied it and named which by digest.
 	const digest = "d640d1b6a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f6071829304152"
-	if err := inv.RecordSent(ctx, node.ID, digest); err != nil {
+	if err := inv.RecordSent(ctx, node.ID, digest, nil); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := (link.Enrolment{Inventory: inv}).Heard(ctx, link.Report{

internal/overlay/generator.go

@@ -78,18 +78,13 @@ type Generator struct {
 	// keyPath is where each node keeps the private half it generated. Named rather than carried:
 	// the mesh has never seen it and never will.
 	keyPath string
-	// registry is the mesh's artifact store as the network reaches it (host:port), or empty when
-	// the mesh has none. Being on the network is what grants a machine the right to pull from it
-	// (novox/hq ADR 0082), so the module that puts a machine on the network is what writes the
-	// runtime's trust. (That is still a write into another module's file, the container runtime's;
-	// novox/hq issue 190 has it handed to that module as a value.)
-	registry string
+	// No registry. Being on the network is still what grants a machine the right to pull from the
+	// mesh's artifact store in the clear (novox/hq ADR 0082), but the runtime's file is the runtime's
+	// module's: the private network writes nothing into it, and that module states the registry
+	// itself, told where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR
+	// 0222, issue 190).
 }
 
-// TrustRegistry names the artifact store this network's machines pull from in the clear —
-// the overlay is the transport security (ADR 0082).
-func (g *Generator) TrustRegistry(hostPort string) { g.registry = hostPort }
-
 // From builds a generator over the machines that are part of the network.
 //
 // The nodes given are the ones assigned the module — not every node the mesh knows. A machine
@@ -128,31 +123,7 @@ func (g *Generator) Resources(node string) ([]map[string]any, bool, error) {
 	if err := json.Unmarshal(raw, &parsed); err != nil {
 		return nil, false, err
 	}
-	resources := parsed.Resources
-	if g.registry != "" {
-		trust, err := json.Marshal(map[string]any{"insecure-registries": []string{g.registry}})
-		if err != nil {
-			return nil, false, err
-		}
-		resources = append(resources,
-			map[string]any{
-				// Written into, not over (novox/hq ADR 0102): the runtime's daemon file is the
-				// machine's — its data directory, its logging, whatever a predecessor set — and
-				// this states one fact in it. The host sets this key and keeps every other.
-				// ("merge" is the operator's settings merged into this content; "into" is the
-				// content written into the machine's file.) The registry speaks plain HTTP
-				// because every path to it is already inside the overlay's encryption (ADR 0082).
-				"id": "registry-trust", "type": "file", "path": "/etc/docker/daemon.json",
-				"content": string(trust) + "\n", "mode": "0644", "merge": "json", "into": "json",
-			},
-			map[string]any{
-				// Reloaded, not restarted: the runtime re-reads its trusted registries on a reload,
-				// and a restart stops every container on the machine (measured; ADR 0102).
-				"id": "registry-trust-reload", "type": "service", "unit": "docker.service",
-				"state": "running", "reload-on": []string{"registry-trust"},
-			})
-	}
-	return resources, true, nil
+	return parsed.Resources, true, nil
 }
 
 // Nodes are the machines this generator was built over, so a caller can say who is on the network.

internal/overlay/generator_test.go

@@ -1,15 +1,13 @@
 package overlay
 
 import (
-	"fmt"
-	"strings"
 	"testing"
 )
 
-func TestTheNetworkCarriesRegistryTrust(t *testing.T) {
-	// novox/hq ADR 0082: being on the network is what grants a machine the right to pull from the
-	// mesh's artifact store in the clear, so the network module writes the runtime's trust — and
-	// writes nothing when the mesh has no store to trust.
+// novox/hq ADR 0222, issue 190: the runtime's file and service are the runtime's module's. The
+// private network writes nothing into either — being on it still grants the right to pull from the
+// mesh's store in the clear (ADR 0082), and the runtime's module states that trust itself.
+func TestTheNetworkWritesNothingOfTheRuntimes(t *testing.T) {
 	nodes := []Node{
 		{Name: "anchor", Site: "lab", Hub: true, Endpoint: "192.0.2.10:51820", Key: "k1", Address: "10.42.0.1"},
 		{Name: "node2", Site: "lab", Key: "k2", Address: "10.42.0.2"},
@@ -18,47 +16,15 @@ func TestTheNetworkCarriesRegistryTrust(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	plain, _, err := g.Resources("node2")
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, r := range plain {
-		if r["id"] == "registry-trust" {
-			t.Fatal("trust was written with no artifact store to trust")
+	for _, node := range []string{"anchor", "node2"} {
+		resources, part, err := g.Resources(node)
+		if err != nil || !part {
+			t.Fatalf("%s: resources: %v part=%v", node, err, part)
 		}
-	}
-
-	g.TrustRegistry("anchor.internal:5000")
-	trusted, part, err := g.Resources("node2")
-	if err != nil || !part {
-		t.Fatalf("resources: %v part=%v", err, part)
-	}
-	var file, service map[string]any
-	for _, r := range trusted {
-		switch r["id"] {
-		case "registry-trust":
-			file = r
-		case "registry-trust-reload":
-			service = r
+		for _, r := range resources {
+			if r["path"] == "/etc/docker/daemon.json" || r["unit"] == "docker.service" {
+				t.Errorf("%s: the private network declares the runtime's %v", node, r)
+			}
 		}
 	}
-	if file == nil || service == nil {
-		t.Fatalf("the trust file or its reload is missing: %v", trusted)
-	}
-	if file["path"] != "/etc/docker/daemon.json" || file["merge"] != "json" || file["into"] != "json" {
-		t.Fatalf("the trust is not written into daemon.json (ADR 0102): %v", file)
-	}
-	if content, _ := file["content"].(string); !strings.Contains(content, `"anchor.internal:5000"`) {
-		t.Fatalf("the trust does not name the store: %v", file["content"])
-	}
-	if service["unit"] != "docker.service" {
-		t.Fatalf("the reload is not the runtime's: %v", service)
-	}
-	// Reloaded, never restarted: a restart stops every container on the machine (ADR 0102).
-	if _, restarts := service["restart-on"]; restarts {
-		t.Fatalf("the runtime is restarted for its trust: %v", service)
-	}
-	if fmt.Sprint(service["reload-on"]) != "[registry-trust]" {
-		t.Fatalf("the runtime is not reloaded for its trust: %v", service)
-	}
 }