Let the mesh's resolver seat have several holders on record

musl takes the first reply from any listed nameserver, so a public fallback
beside the mesh's resolver answered NXDOMAIN for mesh names in every Alpine
container (hq ADR 0223). The fix is two mesh resolvers and no public one, which
needs mesh-dns-resolver held on two machines: a seat can now be replicated,
each holder recorded by 'seat <name> --add', checkClaims accepts every holder
on record and still refuses a second holder of any other mesh seat, a holder
answers its own requirement, and a roster fact gives each replicated seat's
holders, this machine first, so resolv-conf can list them. Migration 0062 keys
a holding by seat and assignment.

cmd/mesh-controller/holdings.go

@@ -6,6 +6,8 @@ import (
 	"sort"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/overlay"
 )
 
 // recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
@@ -90,3 +92,72 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
 	}
 	return said, nil
 }
+
+// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
+// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
+// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
+//
+// **The holders on record, and only the sole claimant when there are none** — the same answer the
+// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
+// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
+// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
+// listed here. A holder off the private network is left out: a resolver named at an address nothing
+// answers is a lookup that waits out its timeout on every name.
+func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
+	shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
+	var replicated []catalogue.Seat
+	for _, s := range catalogue.Seats() {
+		if s.Replicated && s.Scope == catalogue.ScopeMesh {
+			replicated = append(replicated, s)
+		}
+	}
+	if len(replicated) == 0 {
+		return nil, nil
+	}
+	recorded, err := inv.Holdings(ctx)
+	if err != nil {
+		return nil, err
+	}
+	places, err := onTheNetwork(ctx, inv, shelf)
+	if err != nil {
+		return nil, err
+	}
+	address := map[string]string{}
+	for _, p := range places {
+		address[p.Name] = p.Address
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return nil, err
+	}
+	out := map[string]map[string]string{}
+	for _, seat := range replicated {
+		var nodes []string
+		for _, h := range recorded {
+			if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
+				nodes = append(nodes, h.Node)
+			}
+		}
+		if len(nodes) == 0 {
+			var derived []string
+			for _, e := range entries {
+				for _, c := range e.Manifest.Claims {
+					if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
+						derived = append(derived, e.On...)
+					}
+				}
+			}
+			if len(derived) == 1 {
+				nodes = derived
+			}
+		}
+		at := map[string]string{}
+		for _, n := range nodes {
+			if address[n] != "" {
+				at[overlay.InternalName(n)] = address[n]
+			}
+		}
+		out[seat.Name] = at
+	}
+	return out, nil
+}

cmd/mesh-controller/main.go

@@ -194,6 +194,7 @@ func usage() {
   seats [--json]                 every seat this mesh defines, what it delivers, and who holds it
   seat rename <from> <to>        rename a seat; its former name still resolves (ADR 0122)
   seat <name> --to <node>/<module>   hand a seat to that assignment as one act; never empty in between (ADR 0131)
+  seat <name> --add <node>/<module>  add a holder beside the others, for a replicated seat (ADR 0223)
   board [--listen ADDR]          the same three questions, as a page that holds nothing
   api --issuer URL [--listen A]  assign and unassign over http, for a surface that is not here
   assign <node> <module>...      put modules on a node, judged together (ADR 0207)

cmd/mesh-controller/plan.go

@@ -707,6 +707,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
 		return catalogue.Rendering{}, inventory.Node{}, err
 	}
 
+	// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
+	// lists every holder of the mesh's resolver.
+	replicas, err := replicatedHolders(ctx, inv, shelf)
+	if err != nil {
+		return catalogue.Rendering{}, inventory.Node{}, err
+	}
+
 	// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
 	// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
 	// before it had an address on the private network. A machine joins through the tunnel now, and
@@ -783,7 +790,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
 		BusMembership: memberships[node],
 		Settings:      settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
 		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
-		Machines: machines, Zones: zones,
+		Machines: machines, Zones: zones, Holders: replicas,
 		Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
 		Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
 		Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,

cmd/mesh-controller/seats.go

@@ -29,11 +29,13 @@ type seatHolder struct {
 
 // seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
 type seatRow struct {
-	Seat     string       `json:"seat"`
-	Scope    string       `json:"scope"`
-	Delivers string       `json:"delivers,omitempty"`
-	Decision string       `json:"decision"`
-	Holders  []seatHolder `json:"holders"`
+	Seat     string `json:"seat"`
+	Scope    string `json:"scope"`
+	Delivers string `json:"delivers,omitempty"`
+	Decision string `json:"decision"`
+	// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
+	Replicated bool         `json:"replicated,omitempty"`
+	Holders    []seatHolder `json:"holders"`
 }
 
 // seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
@@ -47,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
 	rows := make([]seatRow, 0, len(seats))
 	for _, s := range seats {
 		row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
-			Holders: []seatHolder{}}
+			Replicated: s.Replicated, Holders: []seatHolder{}}
 		seen := map[seatHolder]bool{}
 		for _, h := range held {
 			// Resolve the held claim to a seat rather than comparing names, so a record naming a
@@ -104,9 +106,13 @@ func seatCommand(ctx context.Context, args []string) error {
 		return nil
 	}
 	if len(args) == 3 && args[1] == "--to" {
-		return handOver(ctx, args[0], args[2])
+		return handOver(ctx, args[0], args[2], false)
 	}
-	return fmt.Errorf("seat rename <from> <to>  |  seat <name> --to <node>/<module>")
+	if len(args) == 3 && args[1] == "--add" {
+		return handOver(ctx, args[0], args[2], true)
+	}
+	return fmt.Errorf("seat rename <from> <to>  |  seat <name> --to <node>/<module>  |  " +
+		"seat <name> --add <node>/<module>")
 }
 
 // handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
@@ -119,7 +125,12 @@ func seatCommand(ctx context.Context, args []string) error {
 // **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
 // and refusing to record a handover to a module the node has not started would make the handover
 // impossible to do before the switch instead of as the switch.
-func handOver(ctx context.Context, seatName, to string) error {
+//
+// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
+// records the named assignment as a further holder and leaves every holder on record as it is. A
+// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
+// the one named, as it always did.
+func handOver(ctx context.Context, seatName, to string, adding bool) error {
 	nodeName, module, ok := strings.Cut(to, "/")
 	if !ok || nodeName == "" || module == "" {
 		return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
@@ -135,6 +146,10 @@ func handOver(ctx context.Context, seatName, to string) error {
 	if !known {
 		return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
 	}
+	if adding && !seat.Replicated {
+		return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
+			"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
+	}
 	assigned, err := inv.Assigned(ctx, nodeName)
 	if err != nil {
 		return err
@@ -157,6 +172,7 @@ func handOver(ctx context.Context, seatName, to string) error {
 		return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
 	}
 	var was string
+	var held []string
 	holdings, err := inv.Holdings(ctx)
 	if err != nil {
 		return err
@@ -164,6 +180,7 @@ func handOver(ctx context.Context, seatName, to string) error {
 	for _, h := range holdings {
 		if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
 			was = h.Node
+			held = append(held, h.Node)
 		}
 	}
 
@@ -187,6 +204,15 @@ func handOver(ctx context.Context, seatName, to string) error {
 	} else if err := catalogue.CanHold(*m, seat); err != nil {
 		return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
 	}
+	if adding {
+		if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
+			return err
+		}
+		fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
+			strings.Join(held, ", "))
+		fmt.Printf("  `push --behind` re-declares every machine that reads the seat's holders\n")
+		return nil
+	}
 	if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
 		return err
 	}

internal/catalogue/declaration.go

@@ -159,6 +159,11 @@ type Rendering struct {
 	// 0199): the mesh's resolver forwards each one there.
 	Zones []ZoneAt
 
+	// Holders is, for each replicated mesh seat, every machine holding it, by internal name and
+	// private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver
+	// file lists: every holder of the mesh's resolver, this machine first if it is one.
+	Holders map[string]map[string]string
+
 	Settings   SettingsBy
 	Generators map[string]Generator
 	// Grants are the credentials this node must create, for the provisions it offers. Passed in
@@ -980,7 +985,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 		// plane's; making a name resolve is the module's software. Emitted as ordinary files under
 		// this module's name, so they are applied, reported and removed exactly as anything else
 		// it declares.
-		given, err := FactsWithZonesInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix, with.Zones)
+		given, err := FactsFrom(m, r, with)
 		if err != nil {
 			return nil, err
 		}

internal/catalogue/resolve.go

@@ -767,16 +767,27 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
 	var problems []string
 	var held []Held
 
-	// onRecord is the recorded holder of a seat, if a handover ever named one.
-	onRecord := func(claim, scope string) (Held, bool) {
+	// onRecord is every recorded holder of a seat, if a handover ever named one: one for most seats,
+	// and as many as were added for a replicated one (novox/hq ADR 0223).
+	onRecord := func(claim, scope string) []Held {
+		var out []Held
 		for _, h := range holdings {
 			hs, ok := SeatNamed(h.Claim)
 			cs, cok := SeatNamed(claim)
 			if ok && cok && hs.Name == cs.Name && h.Scope == scope {
-				return h, true
+				out = append(out, h)
 			}
 		}
-		return Held{}, false
+		return out
+	}
+	// recordedHere says this node's module is one of a seat's holders on record.
+	recordedHere := func(claim, scope, module string) bool {
+		for _, rec := range onRecord(claim, scope) {
+			if rec.Node == node.Name && rec.Module == module {
+				return true
+			}
+		}
+		return false
 	}
 
 	byScope := map[string]map[string]string{} // scope → claim → module
@@ -787,8 +798,18 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
 			// the seat but is not the one on record is eligible, and that is all: it is not a second
 			// holder, so it is not refused, and it does not hold (novox/hq ADR 0131). This is what
 			// lets the next holder stand beside the current one until the seat is handed over.
-			if rec, recorded := onRecord(c.Name, scope); recorded {
-				if rec.Node != node.Name || rec.Module != m.Module {
+			if recs := onRecord(c.Name, scope); len(recs) > 0 {
+				// **A seat held once is on record once** (novox/hq ADR 0223). Only a replicated seat
+				// may have several holders on record; several for any other seat is a store that
+				// disagrees with the mesh's definition of the role, and it is refused, named, rather
+				// than letting two machines answer for what the mesh has one of.
+				if s, known := SeatNamed(c.Name); known && !s.Replicated && len(recs) > 1 {
+					problems = append(problems, fmt.Sprintf(
+						"%q is on record as held by %d assignments, and it is held once per %s — "+
+							"`seat %s --to <node>/<module>` records one", c.Name, len(recs), scope, c.Name))
+					continue
+				}
+				if !recordedHere(c.Name, scope, m.Module) {
 					continue
 				}
 			}
@@ -815,6 +836,12 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
 			}
 			switch h.Scope {
 			case ScopeMesh:
+				// **A holder on record is never a second claimant** (novox/hq ADR 0223). Records are
+				// the mesh's settled answer: one for most seats, several only for a replicated seat,
+				// each added by an act. Two holders here are two records, and both hold.
+				if recordedHere(h.Claim, h.Scope, h.Module) {
+					continue
+				}
 				// Both claim and nobody is on record, or this refusal could not have happened.
 				// The remedy is the handover that records the holder (novox/hq ADR 0131,
 				// 04-ISSUES/170), so it is named here rather than left to be found.
@@ -1156,6 +1183,18 @@ func answeredElsewhere(want string, node Node, world World, brokered map[string]
 	if c, pinned := world.Pinned[want]; pinned {
 		return c.Node != node.Name
 	}
+	// **A machine holding the seat answers itself** (novox/hq ADR 0223). With a replicated seat
+	// another machine holds it too, and the first holder in the providers' order may be that one; a
+	// holder is still where this machine's own requirement is answered, so its resolver file lists
+	// itself first.
+	if seat, delivered := SeatDelivering(want); delivered {
+		for _, h := range append(append([]Held(nil), world.Holdings...), world.Held...) {
+			if hs, ok := SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope &&
+				h.Node == node.Name {
+				return false
+			}
+		}
+	}
 	holder, held := HolderAmong(want, world.Offered[want], world.Held)
 	return held && holder.Node != node.Name
 }

internal/catalogue/resolver_manifests_test.go

@@ -84,25 +84,24 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
 			t.Errorf("the mesh's resolver still reads or listens on %q", never)
 		}
 	}
-	// And the file that decides what the machine asks names the mesh's resolver first, by address,
-	// and a public one second, asked only when the first is silent (ADR 0196).
-	var resolv string
-	for _, r := range catalogueManifest(t, "resolv-conf").Resources {
-		if r["path"] == "/etc/resolv.conf" {
-			resolv, _ = r["content"].(string)
+	// And the file that decides what the machine asks names every one of the mesh's resolvers, by
+	// address, from the seat's holders, and no public one (ADR 0223): a resolver library that asks
+	// every listed server at once takes the first reply, and a public "no such name" for a mesh name
+	// won it.
+	fact, ok := catalogueManifest(t, "resolv-conf").Facts["resolvers"]
+	if !ok || fact.Path != "/etc/resolv.conf" {
+		t.Fatalf("the machine's resolver file is not rendered from the roster: %+v", fact)
+	}
+	if !strings.Contains(fact.Template, `{{range index .Holders "mesh-dns-resolver"}}nameserver {{.Address}}`) {
+		t.Errorf("resolv.conf does not list every holder of the mesh's resolver:\n%s", fact.Template)
+	}
+	for _, line := range strings.Split(fact.Template, "\n") {
+		if strings.HasPrefix(line, "nameserver ") && !strings.Contains(line, "{{") {
+			t.Errorf("resolv.conf names a resolver of its own beside the mesh's: %s", line)
 		}
 	}
-	var nameservers []string
-	for _, line := range strings.Split(resolv, "\n") {
-		if strings.HasPrefix(line, "nameserver ") {
-			nameservers = append(nameservers, strings.TrimPrefix(line, "nameserver "))
-		}
-	}
-	if len(nameservers) != 2 || nameservers[0] != "${bound:wildcard-resolution:address}" || nameservers[1] != "1.1.1.1" {
-		t.Errorf("resolv.conf names %v; the mesh's resolver by address first, a public one second", nameservers)
-	}
-	if !strings.Contains(resolv, "\noptions timeout:1 attempts:1") {
-		t.Errorf("the fallback is not reached after one short attempt:\n%s", resolv)
+	if !strings.Contains(fact.Template, "options timeout:1 attempts:2 edns0\n") {
+		t.Errorf("a silent resolver is not passed over after one short wait:\n%s", fact.Template)
 	}
 }
 
@@ -126,6 +125,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
 		// happen to be the same map, since nothing routed is part of it.
 		Names: twoMachines, Machines: twoMachines, Suffix: "internal",
 		Zones:    []ZoneAt{{Zone: "incus", Address: "10.42.0.2", Port: 5353}},
+		Holders:  map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1"}},
 		Needed:   map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
 		Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
 	})
@@ -205,9 +205,9 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
 		t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
 	}
 
-	resolv := ids["resolv-conf.resolv"]
-	if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\nnameserver 1.1.1.1\n") {
-		t.Fatalf("the machine is not pointed at the resolver by address, with the public fallback: %v", resolv)
+	resolv := ids["resolv-conf.fact-resolvers"]
+	if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\noptions ") {
+		t.Fatalf("the machine is not pointed at the resolver by address, and only at it: %v", resolv)
 	}
 }
 

internal/catalogue/roster.go

@@ -64,6 +64,12 @@ type rosterView struct {
 	// Zones is every zone a module in the mesh answers itself, with where its answerer is (novox/hq
 	// ADR 0199) — what the mesh's resolver forwards. Ordered by zone.
 	Zones []rosterZone
+	// Holders is the machines holding each replicated mesh seat, by seat (novox/hq ADR 0223) — what
+	// a machine's resolver file lists for `mesh-dns-resolver`. **This machine first when it is one
+	// of them**, then the rest by name: the nearest holder is asked first, and two renderings of
+	// one mesh on one machine are one file. A template reads one seat's with
+	// `index .Holders "<seat>"`; a seat nobody holds ranges over nothing.
+	Holders map[string][]rosterEntry
 }
 
 // rosterZone is one zone as a template sees it: the zone, and the address and port answering it.
@@ -96,6 +102,13 @@ func FactsInto(m Manifest, r Resolution, every, machines, accounts map[string]st
 // FactsWithZonesInto is FactsInto with the mesh's zones in the view, for a template that ranges them.
 func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[string]string, suffix string,
 	zones []ZoneAt) ([]map[string]any, error) {
+	return FactsFrom(m, r, Rendering{Names: every, Machines: machines, Accounts: accounts, Suffix: suffix,
+		Zones: zones})
+}
+
+// FactsFrom renders the roster files a module asked for from everything the mesh composed for this
+// machine: its machines, zones and the holders of each replicated seat.
+func FactsFrom(m Manifest, r Resolution, with Rendering) ([]map[string]any, error) {
 	if len(m.Facts) == 0 {
 		return nil, nil
 	}
@@ -107,10 +120,11 @@ func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[
 
 	view := rosterView{
 		Node:     r.Node,
-		Suffix:   strings.TrimPrefix(suffixOr(suffix), "."),
-		Names:    entriesFrom(every, accounts, suffix),
-		Machines: entriesFrom(machines, accounts, suffix),
-		Zones:    zonesFrom(zones),
+		Suffix:   strings.TrimPrefix(suffixOr(with.Suffix), "."),
+		Names:    entriesFrom(with.Names, with.Accounts, with.Suffix),
+		Machines: entriesFrom(with.Machines, with.Accounts, with.Suffix),
+		Zones:    zonesFrom(with.Zones),
+		Holders:  holdersFrom(with.Holders, with.Accounts, with.Suffix, r.Node),
 	}
 
 	out := make([]map[string]any, 0, len(names))
@@ -250,3 +264,18 @@ func zonesFrom(zones []ZoneAt) []rosterZone {
 	sort.Slice(out, func(i, j int) bool { return out[i].Zone < out[j].Zone })
 	return out
 }
+
+// holdersFrom is each replicated seat's holders as a template ranges them: this machine first when
+// it holds the seat, then the others by name (novox/hq ADR 0223). A machine with no address is left
+// out, as everywhere in the roster — a resolver named at nothing is a lookup that hangs.
+func holdersFrom(holders map[string]map[string]string, accounts map[string]string, suffix, node string) map[string][]rosterEntry {
+	out := make(map[string][]rosterEntry, len(holders))
+	for seat, at := range holders {
+		entries := entriesFrom(at, accounts, suffix)
+		sort.SliceStable(entries, func(i, j int) bool {
+			return entries[i].Name == node && entries[j].Name != node
+		})
+		out[seat] = entries
+	}
+	return out
+}

internal/catalogue/seats.go

@@ -21,8 +21,16 @@ import (
 type Seat struct {
 	// Name is what a manifest claims.
 	Name string
-	// Scope is where there may be only one holder.
+	// Scope is where there may be only one holder — unless the seat is Replicated.
 	Scope string
+	// Replicated says a mesh seat may be held on several machines at once, each holder answering the
+	// same thing (novox/hq ADR 0223): the mesh's resolver, held on the anchor and the home server so a
+	// machine's resolver file lists two that give one answer. Each holder is on record, added by an
+	// act (`seat <name> --add <node>/<module>`), never by being assigned: two claimants with nothing on
+	// record are refused exactly as for any mesh seat. One per machine still — two modules on one
+	// node claiming it are refused. Compiled, never stored, like Needs: it is the mesh's definition of
+	// the role, and the store's rows carry no column for it.
+	Replicated bool
 	// Delivers is the provision the seat's holder answers for, or empty. A seat that delivers a
 	// provision may only be held by a module providing it at the seat's scope, and its holder is
 	// what a requirement for that provision resolves to when several modules provide it.
@@ -140,10 +148,14 @@ var defaultSeats = append([]Seat{
 	// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
 	{Name: "mesh-build-machine", Scope: ScopeMesh,
 		Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
-	// **The mesh's one resolver** (novox/hq ADR 0194, 0196): every node's internal domain, held in one
-	// place, and every node and container asks it first. Delivers what a machine's resolver
-	// configuration requires, so that requirement resolves to the holder wherever it is placed.
-	{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Decision: "novox/hq ADR 0194"},
+	// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
+	// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
+	// than one machine, each answering the same names from the same roster, and every machine's
+	// resolver file lists every holder — its own first — and no public resolver, so whichever answers
+	// first gives the one answer. Delivers what a machine's resolver configuration requires, so that
+	// requirement resolves to a holder wherever they are placed.
+	{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Replicated: true,
+		Decision: "novox/hq ADR 0194, ADR 0223"},
 	// **A machine's /etc/hosts is one module's** (novox/hq ADR 0199): its holder writes the machine's
 	// own lines and keeps every other line as the operator's, changed through these three verbs on that
 	// machine alone. The controller holds none of it.
@@ -314,11 +326,12 @@ func UseSeats(s []Seat) {
 				row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
 			}
 		}
-		// What a seat receives and what its holder needs are never stored (novox/hq ADR 0212, ADR
-		// 0220), so they are always the compiled ones.
+		// What a seat receives, what its holder needs and whether it is replicated are never stored
+		// (novox/hq ADR 0212, ADR 0220, ADR 0223), so they are always the compiled ones.
 		if d, known := byName[row.Name]; known {
 			row.Receives = d.Receives
 			row.Needs = d.Needs
+			row.Replicated = d.Replicated
 		}
 		merged = append(merged, row)
 	}
@@ -498,19 +511,24 @@ func seatNames() string {
 // two modules on one node could both provide a provision, and only the one holding the seat
 // answers for it. Nothing when no seat delivers the provision, when nobody holds
 // it, or when the holder is not among the providers offered.
+//
+// **The first holder in the providers' own order** (novox/hq ADR 0223). A replicated seat has
+// several, and the answer must not depend on the order the mesh happened to resolve its machines
+// in: the providers come sorted by machine, so every consumer is bound to the same one. A seat with
+// one holder gets the same answer as before.
 func HolderAmong(provision string, providers []Provider, held []Held) (Provider, bool) {
 	seat, delivered := SeatDelivering(provision)
 	if !delivered {
 		return Provider{}, false
 	}
-	for _, h := range held {
-		// Resolve the held claim to a seat rather than comparing names, so a record naming a seat's
-		// former name still matches it after a rename (novox/hq ADR 0122).
-		hs, ok := SeatNamed(h.Claim)
-		if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
-			continue
-		}
-		for _, p := range providers {
+	for _, p := range providers {
+		for _, h := range held {
+			// Resolve the held claim to a seat rather than comparing names, so a record naming a
+			// seat's former name still matches it after a rename (novox/hq ADR 0122).
+			hs, ok := SeatNamed(h.Claim)
+			if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
+				continue
+			}
 			if p.Node == h.Node && p.Module == h.Module {
 				return p, true
 			}

internal/catalogue/two_resolvers_test.go

@@ -0,0 +1,199 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// The mesh has two resolvers (novox/hq ADR 0223): `mesh-dns-resolver` is replicated, held on the
+// anchor and on the home server, each answering the same names; every machine's resolver file lists
+// every holder — its own first when it is one — and no public resolver. ADR 0196 listed the mesh's
+// resolver then a public one, and musl asks both at once and takes the first reply: from the home
+// server the public "no such name" for the anchor's mesh name won, every time, in every Alpine build.
+
+// resolverMachines is the anchor and the home server holding the resolver, and a laptop holding nothing.
+var resolverMachines = map[string]string{
+	"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2", "home.internal": "10.42.0.3"}
+
+// bothResolvers is the two holders on record, as `seat mesh-dns-resolver --add` leaves them.
+var bothResolvers = []Held{
+	{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "anchor", Module: "dnsmasq"},
+	{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "home", Module: "dnsmasq"},
+}
+
+// twoResolverShelf is the resolver, what asks it, and a stand-in answering `mesh-addressing`.
+func twoResolverShelf(t *testing.T) map[string]Manifest {
+	t.Helper()
+	return map[string]Manifest{
+		"net":         {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
+		"dnsmasq":     catalogueManifest(t, "dnsmasq"),
+		"resolv-conf": catalogueManifest(t, "resolv-conf"),
+	}
+}
+
+// worldWithout is the rest of the mesh as a plan for one machine sees it: every other holder's claim
+// and offer, and both holders on record.
+func worldWithout(node string) World {
+	w := World{Holdings: bothResolvers, Offered: map[string][]Provider{}}
+	for _, h := range bothResolvers {
+		if h.Node == node {
+			continue
+		}
+		w.Held = append(w.Held, h)
+		w.Offered["wildcard-resolution"] = append(w.Offered["wildcard-resolution"],
+			Provider{Node: h.Node, At: h.Node + ".internal", Module: h.Module})
+	}
+	return w
+}
+
+// resolvConfOn resolves and composes one machine and answers with the nameservers its resolver file
+// lists, in order, and the file.
+func resolvConfOn(t *testing.T, node string, assigned []string) ([]string, string) {
+	t.Helper()
+	got, err := Resolve(twoResolverShelf(t), assigned, Node{Name: node, At: node + ".internal"}, worldWithout(node))
+	if err != nil {
+		t.Fatalf("%s does not resolve with two resolvers on record: %v", node, err)
+	}
+	out, err := got.Declaration(Rendering{
+		Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
+		Holders: map[string]map[string]string{"mesh-dns-resolver": {
+			"anchor.internal": "10.42.0.1", "home.internal": "10.42.0.3"}},
+		Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
+	})
+	if err != nil {
+		t.Fatalf("%s does not compose: %v", node, err)
+	}
+	file := byID(out)["resolv-conf.fact-resolvers"]
+	if file == nil || file["path"] != "/etc/resolv.conf" {
+		t.Fatalf("%s was given no resolver file: %v", node, file)
+	}
+	content, _ := file["content"].(string)
+	var servers []string
+	for _, line := range strings.Split(content, "\n") {
+		if strings.HasPrefix(line, "nameserver ") {
+			servers = append(servers, strings.TrimPrefix(line, "nameserver "))
+		}
+	}
+	return servers, content
+}
+
+func TestTwoResolversComposeOnBothHoldersAndEachListsItselfFirst(t *testing.T) {
+	for node, want := range map[string][]string{
+		"anchor": {"10.42.0.1", "10.42.0.3"},
+		"home":   {"10.42.0.3", "10.42.0.1"},
+	} {
+		servers, content := resolvConfOn(t, node, []string{"dnsmasq", "resolv-conf"})
+		if strings.Join(servers, " ") != strings.Join(want, " ") {
+			t.Errorf("%s lists %v; itself first, then the other holder: %v\n%s", node, servers, want, content)
+		}
+	}
+}
+
+func TestAMachineHoldingNoResolverListsBothAndNoPublicOne(t *testing.T) {
+	servers, content := resolvConfOn(t, "laptop", []string{"resolv-conf"})
+	if strings.Join(servers, " ") != "10.42.0.1 10.42.0.3" {
+		t.Errorf("the laptop lists %v; every holder, by name, and nothing else:\n%s", servers, content)
+	}
+	for _, public := range []string{"1.1.1.1", "8.8.8.8", "9.9.9.9"} {
+		if strings.Contains(content, public) {
+			t.Errorf("a public resolver is listed beside the mesh's (ADR 0223):\n%s", content)
+		}
+	}
+}
+
+// A holder answers its own requirement, even though the other holder sorts first (issue 258 kept).
+func TestAHolderAnswersItsOwnRequirement(t *testing.T) {
+	got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq", "resolv-conf"},
+		Node{Name: "home", At: "home.internal"}, worldWithout("home"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, n := range got.Needs {
+		if n.Name == "wildcard-resolution" && n.From != "home" {
+			t.Errorf("the home server's resolver configuration is bound to %s; it holds the seat itself", n.From)
+		}
+	}
+}
+
+// A seat held once is still held once: a second claimant on another machine is refused while
+// nothing is on record, and a store recording two holders of it is refused, naming the seat.
+func TestASingleHolderMeshSeatStillRefusesASecondHolder(t *testing.T) {
+	store := shelf(mod("postgres", nil, nil, nil, Claim{Name: "mesh-store", Scope: ScopeMesh}))
+	other := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: "anchor", Module: "postgres"}
+	if _, err := Resolve(store, []string{"postgres"}, workstation(), World{Held: []Held{other}}); err == nil ||
+		!strings.Contains(err.Error(), "one per mesh") {
+		t.Errorf("a second claimant of a seat held once was not refused: %v", err)
+	}
+	here := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: workstation().Name, Module: "postgres"}
+	_, err := Resolve(store, []string{"postgres"}, workstation(),
+		World{Held: []Held{other}, Holdings: []Held{other, here}})
+	if err == nil || !strings.Contains(err.Error(), "on record as held by 2") {
+		t.Errorf("two holders on record for a seat held once were not refused: %v", err)
+	}
+}
+
+// Replicated is not "whoever is assigned": two claimants with nothing on record are refused, as for
+// any mesh seat, and each holder is added by an act.
+func TestTwoUnrecordedClaimantsOfTheReplicatedSeatAreRefused(t *testing.T) {
+	w := worldWithout("home")
+	w.Holdings = nil
+	_, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "home", At: "home.internal"}, w)
+	if err == nil || !strings.Contains(err.Error(), "seat mesh-dns-resolver --to") {
+		t.Errorf("a second resolver with nothing on record was not refused, naming the handover: %v", err)
+	}
+}
+
+// Only the mesh's resolver is replicated: a seat being replicated is a decision, recorded.
+func TestOnlyTheResolverIsReplicated(t *testing.T) {
+	for _, s := range Seats() {
+		if s.Replicated != (s.Name == "mesh-dns-resolver") {
+			t.Errorf("%s replicated = %v; only mesh-dns-resolver is (ADR 0223)", s.Name, s.Replicated)
+		}
+	}
+	UseSeats([]Seat{{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution"}})
+	defer UseSeats(DefaultSeats())
+	if s, _ := SeatNamed("mesh-dns-resolver"); !s.Replicated {
+		t.Error("loading the set from the store, which has no column for it, lost the resolver's replication")
+	}
+}
+
+// Every consumer is bound to the same holder whatever order the mesh resolved its machines in.
+func TestTheFirstHolderIsTheFirstProvider(t *testing.T) {
+	providers := []Provider{{Node: "anchor", Module: "dnsmasq"}, {Node: "home", Module: "dnsmasq"}}
+	for _, held := range [][]Held{bothResolvers, {bothResolvers[1], bothResolvers[0]}} {
+		if p, ok := HolderAmong("wildcard-resolution", providers, held); !ok || p.Node != "anchor" {
+			t.Errorf("held in order %v answered %v", held, p)
+		}
+	}
+}
+
+// One host record per machine, beside its wildcard (novox/hq issue 262): a name with a host record
+// says it exists and has no IPv6 address, where the wildcard alone said there is no such name, and
+// musl reads that as final.
+func TestTheResolverHasOneHostRecordPerMachine(t *testing.T) {
+	got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "anchor", At: "anchor.internal"},
+		World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := got.Declaration(Rendering{Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
+		Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	content, _ := byID(out)["dnsmasq.fact-node-zones"]["content"].(string)
+	records := map[string]int{}
+	for _, line := range strings.Split(content, "\n") {
+		if strings.HasPrefix(line, "host-record=") {
+			records[strings.TrimPrefix(line, "host-record=")]++
+		}
+	}
+	for name, at := range resolverMachines {
+		if records[name+","+at] != 1 {
+			t.Errorf("%s has %d host records at %s, and has one:\n%s", name, records[name+","+at], at, content)
+		}
+	}
+	if len(records) != len(resolverMachines) {
+		t.Errorf("%d host records for %d machines:\n%s", len(records), len(resolverMachines), content)
+	}
+}

internal/inventory/holdings_test.go

@@ -97,3 +97,57 @@ func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T)
 		t.Fatalf("a re-told membership did not replace the first: %v", got)
 	}
 }
+
+// A replicated seat has several holders on record (novox/hq ADR 0223): each is added beside the
+// others, adding one twice changes nothing, a handover still leaves exactly one, and unassigning one
+// takes only its own row.
+func TestAReplicatedSeatHasSeveralHoldersOnRecord(t *testing.T) {
+	resolver := catalogue.Manifest{Module: "resolver", Version: "1",
+		Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
+		Claims:   []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}
+	inv, ctx := aMeshWith(t, resolver)
+	if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
+		t.Fatal(err)
+	}
+	for _, n := range []string{"anchor", "home"} {
+		if _, err := inv.AddNode(ctx, n); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := inv.Assign(ctx, n, "resolver"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "anchor", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	for range 2 {
+		if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	held, err := inv.Holdings(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(held) != 2 || held[0].Node != "anchor" || held[1].Node != "home" {
+		t.Fatalf("the two holders are not both on record, once each: %+v", held)
+	}
+	if err := inv.Unassign(ctx, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "anchor" {
+		t.Fatalf("unassigning one holder took more or less than its own row: %+v", held)
+	}
+	if _, err := inv.Assign(ctx, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "home" {
+		t.Fatalf("a handover left other holders on record: %+v", held)
+	}
+}

internal/inventory/migrations/0062-a-replicated-seat-has-several-holders-on-record.sql

@@ -0,0 +1,14 @@
+-- A replicated seat has several holders on record (novox/hq ADR 0223).
+--
+-- 0039 recorded one holder per seat, keyed by the seat: a handover replaced the row, so the seat was
+-- never without a holder in between. The mesh's resolver is now held on more than one machine, each
+-- answering the same names, and each of those holders is recorded — added by `seat <name> --add`,
+-- never by being assigned. So a holding is keyed by the seat and the assignment holding it.
+--
+-- Nothing else changes. A handover (`seat <name> --to`) still leaves exactly one row, replacing every
+-- holder in one transaction; whether a seat may have more than one is the seat's compiled definition,
+-- judged by the controller before a row is added, and a store holding two for any other seat is
+-- refused at resolution, naming the seat. Every existing row is one per seat, so it satisfies the new
+-- key as it stands.
+alter table seat_holding drop constraint seat_holding_pkey;
+alter table seat_holding add primary key (seat, node, module);

internal/inventory/seats.go

@@ -215,23 +215,50 @@ func (i *Inventory) RenameSeat(ctx context.Context, from, to string) error {
 	return nil
 }
 
-// HoldSeat records that one assignment holds a seat, replacing whoever held it — as one write, so
-// the seat is never without a holder in between (novox/hq ADR 0131, design 28 task 5.3). The
-// assignment must exist; the store refuses otherwise, and that refusal is the right one: a seat
-// cannot be handed to something that is not running anywhere.
+// HoldSeat records that one assignment holds a seat, replacing whoever held it — every holder, for a
+// replicated seat (novox/hq ADR 0223) — as one transaction, so the seat is never without a holder in
+// between (novox/hq ADR 0131, design 28 task 5.3). The assignment must exist; the store refuses
+// otherwise, and that refusal is the right one: a seat cannot be handed to something that is not
+// running anywhere.
 func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module string) error {
 	node, err := i.NodeByName(ctx, nodeName)
 	if err != nil {
 		return err
 	}
-	_, err = i.store.Pool().Exec(ctx,
-		`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
-		 on conflict (seat) do update set scope = excluded.scope, node = excluded.node,
-		   module = excluded.module, since = now()`,
-		seat, scope, node.ID, module)
+	tx, err := i.store.Pool().Begin(ctx)
 	if err != nil {
+		return err
+	}
+	defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
+	if _, err := tx.Exec(ctx,
+		`delete from seat_holding where seat = $1 and not (node = $2 and module = $3)`,
+		seat, node.ID, module); err != nil {
+		return fmt.Errorf("handing %s to %s on %s: %w", seat, module, nodeName, err)
+	}
+	if _, err := tx.Exec(ctx,
+		`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
+		 on conflict (seat, node, module) do update set scope = excluded.scope, since = now()`,
+		seat, scope, node.ID, module); err != nil {
 		return fmt.Errorf("recording %s on %s as the holder of %s: %w", module, nodeName, seat, err)
 	}
+	return tx.Commit(ctx)
+}
+
+// AddSeatHolder records one more assignment holding a replicated seat, beside those already on
+// record (novox/hq ADR 0223). Whether the seat may have several holders is the caller's to judge —
+// the seat's definition is compiled, and the store holds no column for it. Recording a holder
+// already on record changes nothing.
+func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, module string) error {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return err
+	}
+	if _, err := i.store.Pool().Exec(ctx,
+		`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
+		 on conflict (seat, node, module) do nothing`,
+		seat, scope, node.ID, module); err != nil {
+		return fmt.Errorf("adding %s on %s as a holder of %s: %w", module, nodeName, seat, err)
+	}
 	return nil
 }
 
@@ -240,7 +267,7 @@ func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module
 func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
 	rows, err := i.store.Pool().Query(ctx,
 		`select h.seat, h.scope, n.name, h.module, coalesce(n.site, '')
-		   from seat_holding h join node n on n.id = h.node order by h.seat`)
+		   from seat_holding h join node n on n.id = h.node order by h.seat, n.name, h.module`)
 	if err != nil {
 		return nil, err
 	}