Merge pull request 'A mesh seat may be replicated: the resolver held on two machines (hq ADR 0223)' (#65) from feat/the-mesh-has-two-resolvers into main

cmd/mesh-controller/addresses_test.go

@@ -111,6 +111,14 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
 	if _, err := assign(ctx, open, "laptop", "app"); err != nil {
 		t.Fatal(err)
 	}
+	// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
+	// asks where this machine reaches the store, as the docker module does.
+	register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
+		Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
+			"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
+	if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
+		t.Fatal(err)
+	}
 
 	on := map[string]bool{"anchor": true, "laptop": true}
 	node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)

cmd/mesh-controller/holdings.go

@@ -6,6 +6,8 @@ import (
 	"sort"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/overlay"
 )
 
 // recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
@@ -90,3 +92,72 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
 	}
 	return said, nil
 }
+
+// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
+// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
+// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
+//
+// **The holders on record, and only the sole claimant when there are none** — the same answer the
+// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
+// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
+// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
+// listed here. A holder off the private network is left out: a resolver named at an address nothing
+// answers is a lookup that waits out its timeout on every name.
+func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
+	shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
+	var replicated []catalogue.Seat
+	for _, s := range catalogue.Seats() {
+		if s.Replicated && s.Scope == catalogue.ScopeMesh {
+			replicated = append(replicated, s)
+		}
+	}
+	if len(replicated) == 0 {
+		return nil, nil
+	}
+	recorded, err := inv.Holdings(ctx)
+	if err != nil {
+		return nil, err
+	}
+	places, err := onTheNetwork(ctx, inv, shelf)
+	if err != nil {
+		return nil, err
+	}
+	address := map[string]string{}
+	for _, p := range places {
+		address[p.Name] = p.Address
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return nil, err
+	}
+	out := map[string]map[string]string{}
+	for _, seat := range replicated {
+		var nodes []string
+		for _, h := range recorded {
+			if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
+				nodes = append(nodes, h.Node)
+			}
+		}
+		if len(nodes) == 0 {
+			var derived []string
+			for _, e := range entries {
+				for _, c := range e.Manifest.Claims {
+					if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
+						derived = append(derived, e.On...)
+					}
+				}
+			}
+			if len(derived) == 1 {
+				nodes = derived
+			}
+		}
+		at := map[string]string{}
+		for _, n := range nodes {
+			if address[n] != "" {
+				at[overlay.InternalName(n)] = address[n]
+			}
+		}
+		out[seat.Name] = at
+	}
+	return out, nil
+}

cmd/mesh-controller/main.go

@@ -194,6 +194,7 @@ func usage() {
   seats [--json]                 every seat this mesh defines, what it delivers, and who holds it
   seat rename <from> <to>        rename a seat; its former name still resolves (ADR 0122)
   seat <name> --to <node>/<module>   hand a seat to that assignment as one act; never empty in between (ADR 0131)
+  seat <name> --add <node>/<module>  add a holder beside the others, for a replicated seat (ADR 0223)
   board [--listen ADDR]          the same three questions, as a page that holds nothing
   api --issuer URL [--listen A]  assign and unassign over http, for a surface that is not here
   assign <node> <module>...      put modules on a node, judged together (ADR 0207)

cmd/mesh-controller/plan.go

@@ -707,6 +707,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
 		return catalogue.Rendering{}, inventory.Node{}, err
 	}
 
+	// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
+	// lists every holder of the mesh's resolver.
+	replicas, err := replicatedHolders(ctx, inv, shelf)
+	if err != nil {
+		return catalogue.Rendering{}, inventory.Node{}, err
+	}
+
 	// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
 	// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
 	// before it had an address on the private network. A machine joins through the tunnel now, and
@@ -783,7 +790,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
 		BusMembership: memberships[node],
 		Settings:      settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
 		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
-		Machines: machines, Zones: zones,
+		Machines: machines, Zones: zones, Holders: replicas,
 		Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
 		Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
 		Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,

cmd/mesh-controller/seats.go

@@ -29,11 +29,13 @@ type seatHolder struct {
 
 // seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
 type seatRow struct {
-	Seat     string       `json:"seat"`
-	Scope    string       `json:"scope"`
-	Delivers string       `json:"delivers,omitempty"`
-	Decision string       `json:"decision"`
-	Holders  []seatHolder `json:"holders"`
+	Seat     string `json:"seat"`
+	Scope    string `json:"scope"`
+	Delivers string `json:"delivers,omitempty"`
+	Decision string `json:"decision"`
+	// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
+	Replicated bool         `json:"replicated,omitempty"`
+	Holders    []seatHolder `json:"holders"`
 }
 
 // seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
@@ -47,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
 	rows := make([]seatRow, 0, len(seats))
 	for _, s := range seats {
 		row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
-			Holders: []seatHolder{}}
+			Replicated: s.Replicated, Holders: []seatHolder{}}
 		seen := map[seatHolder]bool{}
 		for _, h := range held {
 			// Resolve the held claim to a seat rather than comparing names, so a record naming a
@@ -104,9 +106,13 @@ func seatCommand(ctx context.Context, args []string) error {
 		return nil
 	}
 	if len(args) == 3 && args[1] == "--to" {
-		return handOver(ctx, args[0], args[2])
+		return handOver(ctx, args[0], args[2], false)
 	}
-	return fmt.Errorf("seat rename <from> <to>  |  seat <name> --to <node>/<module>")
+	if len(args) == 3 && args[1] == "--add" {
+		return handOver(ctx, args[0], args[2], true)
+	}
+	return fmt.Errorf("seat rename <from> <to>  |  seat <name> --to <node>/<module>  |  " +
+		"seat <name> --add <node>/<module>")
 }
 
 // handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
@@ -119,7 +125,12 @@ func seatCommand(ctx context.Context, args []string) error {
 // **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
 // and refusing to record a handover to a module the node has not started would make the handover
 // impossible to do before the switch instead of as the switch.
-func handOver(ctx context.Context, seatName, to string) error {
+//
+// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
+// records the named assignment as a further holder and leaves every holder on record as it is. A
+// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
+// the one named, as it always did.
+func handOver(ctx context.Context, seatName, to string, adding bool) error {
 	nodeName, module, ok := strings.Cut(to, "/")
 	if !ok || nodeName == "" || module == "" {
 		return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
@@ -135,6 +146,10 @@ func handOver(ctx context.Context, seatName, to string) error {
 	if !known {
 		return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
 	}
+	if adding && !seat.Replicated {
+		return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
+			"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
+	}
 	assigned, err := inv.Assigned(ctx, nodeName)
 	if err != nil {
 		return err
@@ -157,6 +172,7 @@ func handOver(ctx context.Context, seatName, to string) error {
 		return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
 	}
 	var was string
+	var held []string
 	holdings, err := inv.Holdings(ctx)
 	if err != nil {
 		return err
@@ -164,6 +180,7 @@ func handOver(ctx context.Context, seatName, to string) error {
 	for _, h := range holdings {
 		if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
 			was = h.Node
+			held = append(held, h.Node)
 		}
 	}
 
@@ -187,6 +204,15 @@ func handOver(ctx context.Context, seatName, to string) error {
 	} else if err := catalogue.CanHold(*m, seat); err != nil {
 		return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
 	}
+	if adding {
+		if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
+			return err
+		}
+		fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
+			strings.Join(held, ", "))
+		fmt.Printf("  `push --behind` re-declares every machine that reads the seat's holders\n")
+		return nil
+	}
 	if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
 		return err
 	}

internal/catalogue/declaration.go

@@ -159,6 +159,11 @@ type Rendering struct {
 	// 0199): the mesh's resolver forwards each one there.
 	Zones []ZoneAt
 
+	// Holders is, for each replicated mesh seat, every machine holding it, by internal name and
+	// private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver
+	// file lists: every holder of the mesh's resolver, this machine first if it is one.
+	Holders map[string]map[string]string
+
 	Settings   SettingsBy
 	Generators map[string]Generator
 	// Grants are the credentials this node must create, for the provisions it offers. Passed in
@@ -980,7 +985,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 		// plane's; making a name resolve is the module's software. Emitted as ordinary files under
 		// this module's name, so they are applied, reported and removed exactly as anything else
 		// it declares.
-		given, err := FactsWithZonesInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix, with.Zones)
+		given, err := FactsFrom(m, r, with)
 		if err != nil {
 			return nil, err
 		}

internal/catalogue/resolve.go

@@ -767,16 +767,27 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
 	var problems []string
 	var held []Held
 
-	// onRecord is the recorded holder of a seat, if a handover ever named one.
-	onRecord := func(claim, scope string) (Held, bool) {
+	// onRecord is every recorded holder of a seat, if a handover ever named one: one for most seats,
+	// and as many as were added for a replicated one (novox/hq ADR 0223).
+	onRecord := func(claim, scope string) []Held {
+		var out []Held
 		for _, h := range holdings {
 			hs, ok := SeatNamed(h.Claim)
 			cs, cok := SeatNamed(claim)
 			if ok && cok && hs.Name == cs.Name && h.Scope == scope {
-				return h, true
+				out = append(out, h)
 			}
 		}
-		return Held{}, false
+		return out
+	}
+	// recordedHere says this node's module is one of a seat's holders on record.
+	recordedHere := func(claim, scope, module string) bool {
+		for _, rec := range onRecord(claim, scope) {
+			if rec.Node == node.Name && rec.Module == module {
+				return true
+			}
+		}
+		return false
 	}
 
 	byScope := map[string]map[string]string{} // scope → claim → module
@@ -787,8 +798,18 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
 			// the seat but is not the one on record is eligible, and that is all: it is not a second
 			// holder, so it is not refused, and it does not hold (novox/hq ADR 0131). This is what
 			// lets the next holder stand beside the current one until the seat is handed over.
-			if rec, recorded := onRecord(c.Name, scope); recorded {
-				if rec.Node != node.Name || rec.Module != m.Module {
+			if recs := onRecord(c.Name, scope); len(recs) > 0 {
+				// **A seat held once is on record once** (novox/hq ADR 0223). Only a replicated seat
+				// may have several holders on record; several for any other seat is a store that
+				// disagrees with the mesh's definition of the role, and it is refused, named, rather
+				// than letting two machines answer for what the mesh has one of.
+				if s, known := SeatNamed(c.Name); known && !s.Replicated && len(recs) > 1 {
+					problems = append(problems, fmt.Sprintf(
+						"%q is on record as held by %d assignments, and it is held once per %s — "+
+							"`seat %s --to <node>/<module>` records one", c.Name, len(recs), scope, c.Name))
+					continue
+				}
+				if !recordedHere(c.Name, scope, m.Module) {
 					continue
 				}
 			}
@@ -815,6 +836,12 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
 			}
 			switch h.Scope {
 			case ScopeMesh:
+				// **A holder on record is never a second claimant** (novox/hq ADR 0223). Records are
+				// the mesh's settled answer: one for most seats, several only for a replicated seat,
+				// each added by an act. Two holders here are two records, and both hold.
+				if recordedHere(h.Claim, h.Scope, h.Module) {
+					continue
+				}
 				// Both claim and nobody is on record, or this refusal could not have happened.
 				// The remedy is the handover that records the holder (novox/hq ADR 0131,
 				// 04-ISSUES/170), so it is named here rather than left to be found.
@@ -1156,6 +1183,18 @@ func answeredElsewhere(want string, node Node, world World, brokered map[string]
 	if c, pinned := world.Pinned[want]; pinned {
 		return c.Node != node.Name
 	}
+	// **A machine holding the seat answers itself** (novox/hq ADR 0223). With a replicated seat
+	// another machine holds it too, and the first holder in the providers' order may be that one; a
+	// holder is still where this machine's own requirement is answered, so its resolver file lists
+	// itself first.
+	if seat, delivered := SeatDelivering(want); delivered {
+		for _, h := range append(append([]Held(nil), world.Holdings...), world.Held...) {
+			if hs, ok := SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope &&
+				h.Node == node.Name {
+				return false
+			}
+		}
+	}
 	holder, held := HolderAmong(want, world.Offered[want], world.Held)
 	return held && holder.Node != node.Name
 }

internal/catalogue/resolver_manifests_test.go

@@ -84,25 +84,24 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
 			t.Errorf("the mesh's resolver still reads or listens on %q", never)
 		}
 	}
-	// And the file that decides what the machine asks names the mesh's resolver first, by address,
-	// and a public one second, asked only when the first is silent (ADR 0196).
-	var resolv string
-	for _, r := range catalogueManifest(t, "resolv-conf").Resources {
-		if r["path"] == "/etc/resolv.conf" {
-			resolv, _ = r["content"].(string)
+	// And the file that decides what the machine asks names every one of the mesh's resolvers, by
+	// address, from the seat's holders, and no public one (ADR 0223): a resolver library that asks
+	// every listed server at once takes the first reply, and a public "no such name" for a mesh name
+	// won it.
+	fact, ok := catalogueManifest(t, "resolv-conf").Facts["resolvers"]
+	if !ok || fact.Path != "/etc/resolv.conf" {
+		t.Fatalf("the machine's resolver file is not rendered from the roster: %+v", fact)
+	}
+	if !strings.Contains(fact.Template, `{{range index .Holders "mesh-dns-resolver"}}nameserver {{.Address}}`) {
+		t.Errorf("resolv.conf does not list every holder of the mesh's resolver:\n%s", fact.Template)
+	}
+	for _, line := range strings.Split(fact.Template, "\n") {
+		if strings.HasPrefix(line, "nameserver ") && !strings.Contains(line, "{{") {
+			t.Errorf("resolv.conf names a resolver of its own beside the mesh's: %s", line)
 		}
 	}
-	var nameservers []string
-	for _, line := range strings.Split(resolv, "\n") {
-		if strings.HasPrefix(line, "nameserver ") {
-			nameservers = append(nameservers, strings.TrimPrefix(line, "nameserver "))
-		}
-	}
-	if len(nameservers) != 2 || nameservers[0] != "${bound:wildcard-resolution:address}" || nameservers[1] != "1.1.1.1" {
-		t.Errorf("resolv.conf names %v; the mesh's resolver by address first, a public one second", nameservers)
-	}
-	if !strings.Contains(resolv, "\noptions timeout:1 attempts:1") {
-		t.Errorf("the fallback is not reached after one short attempt:\n%s", resolv)
+	if !strings.Contains(fact.Template, "options timeout:1 attempts:2 edns0\n") {
+		t.Errorf("a silent resolver is not passed over after one short wait:\n%s", fact.Template)
 	}
 }
 
@@ -126,6 +125,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
 		// happen to be the same map, since nothing routed is part of it.
 		Names: twoMachines, Machines: twoMachines, Suffix: "internal",
 		Zones:    []ZoneAt{{Zone: "incus", Address: "10.42.0.2", Port: 5353}},
+		Holders:  map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1"}},
 		Needed:   map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
 		Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
 	})
@@ -205,9 +205,9 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
 		t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
 	}
 
-	resolv := ids["resolv-conf.resolv"]
-	if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\nnameserver 1.1.1.1\n") {
-		t.Fatalf("the machine is not pointed at the resolver by address, with the public fallback: %v", resolv)
+	resolv := ids["resolv-conf.fact-resolvers"]
+	if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\noptions ") {
+		t.Fatalf("the machine is not pointed at the resolver by address, and only at it: %v", resolv)
 	}
 }
 

internal/catalogue/roster.go

@@ -64,6 +64,12 @@ type rosterView struct {
 	// Zones is every zone a module in the mesh answers itself, with where its answerer is (novox/hq
 	// ADR 0199) — what the mesh's resolver forwards. Ordered by zone.
 	Zones []rosterZone
+	// Holders is the machines holding each replicated mesh seat, by seat (novox/hq ADR 0223) — what
+	// a machine's resolver file lists for `mesh-dns-resolver`. **This machine first when it is one
+	// of them**, then the rest by name: the nearest holder is asked first, and two renderings of
+	// one mesh on one machine are one file. A template reads one seat's with
+	// `index .Holders "<seat>"`; a seat nobody holds ranges over nothing.
+	Holders map[string][]rosterEntry
 }
 
 // rosterZone is one zone as a template sees it: the zone, and the address and port answering it.
@@ -96,6 +102,13 @@ func FactsInto(m Manifest, r Resolution, every, machines, accounts map[string]st
 // FactsWithZonesInto is FactsInto with the mesh's zones in the view, for a template that ranges them.
 func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[string]string, suffix string,
 	zones []ZoneAt) ([]map[string]any, error) {
+	return FactsFrom(m, r, Rendering{Names: every, Machines: machines, Accounts: accounts, Suffix: suffix,
+		Zones: zones})
+}
+
+// FactsFrom renders the roster files a module asked for from everything the mesh composed for this
+// machine: its machines, zones and the holders of each replicated seat.
+func FactsFrom(m Manifest, r Resolution, with Rendering) ([]map[string]any, error) {
 	if len(m.Facts) == 0 {
 		return nil, nil
 	}
@@ -107,10 +120,11 @@ func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[
 
 	view := rosterView{
 		Node:     r.Node,
-		Suffix:   strings.TrimPrefix(suffixOr(suffix), "."),
-		Names:    entriesFrom(every, accounts, suffix),
-		Machines: entriesFrom(machines, accounts, suffix),
-		Zones:    zonesFrom(zones),
+		Suffix:   strings.TrimPrefix(suffixOr(with.Suffix), "."),
+		Names:    entriesFrom(with.Names, with.Accounts, with.Suffix),
+		Machines: entriesFrom(with.Machines, with.Accounts, with.Suffix),
+		Zones:    zonesFrom(with.Zones),
+		Holders:  holdersFrom(with.Holders, with.Accounts, with.Suffix, r.Node),
 	}
 
 	out := make([]map[string]any, 0, len(names))
@@ -250,3 +264,18 @@ func zonesFrom(zones []ZoneAt) []rosterZone {
 	sort.Slice(out, func(i, j int) bool { return out[i].Zone < out[j].Zone })
 	return out
 }
+
+// holdersFrom is each replicated seat's holders as a template ranges them: this machine first when
+// it holds the seat, then the others by name (novox/hq ADR 0223). A machine with no address is left
+// out, as everywhere in the roster — a resolver named at nothing is a lookup that hangs.
+func holdersFrom(holders map[string]map[string]string, accounts map[string]string, suffix, node string) map[string][]rosterEntry {
+	out := make(map[string][]rosterEntry, len(holders))
+	for seat, at := range holders {
+		entries := entriesFrom(at, accounts, suffix)
+		sort.SliceStable(entries, func(i, j int) bool {
+			return entries[i].Name == node && entries[j].Name != node
+		})
+		out[seat] = entries
+	}
+	return out
+}

internal/catalogue/seats.go

@@ -21,8 +21,16 @@ import (
 type Seat struct {
 	// Name is what a manifest claims.
 	Name string
-	// Scope is where there may be only one holder.
+	// Scope is where there may be only one holder — unless the seat is Replicated.
 	Scope string
+	// Replicated says a mesh seat may be held on several machines at once, each holder answering the
+	// same thing (novox/hq ADR 0223): the mesh's resolver, held on the anchor and the home server so a
+	// machine's resolver file lists two that give one answer. Each holder is on record, added by an
+	// act (`seat <name> --add <node>/<module>`), never by being assigned: two claimants with nothing on
+	// record are refused exactly as for any mesh seat. One per machine still — two modules on one
+	// node claiming it are refused. Compiled, never stored, like Needs: it is the mesh's definition of
+	// the role, and the store's rows carry no column for it.
+	Replicated bool
 	// Delivers is the provision the seat's holder answers for, or empty. A seat that delivers a
 	// provision may only be held by a module providing it at the seat's scope, and its holder is
 	// what a requirement for that provision resolves to when several modules provide it.
@@ -140,10 +148,14 @@ var defaultSeats = append([]Seat{
 	// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
 	{Name: "mesh-build-machine", Scope: ScopeMesh,
 		Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
-	// **The mesh's one resolver** (novox/hq ADR 0194, 0196): every node's internal domain, held in one
-	// place, and every node and container asks it first. Delivers what a machine's resolver
-	// configuration requires, so that requirement resolves to the holder wherever it is placed.
-	{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Decision: "novox/hq ADR 0194"},
+	// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
+	// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
+	// than one machine, each answering the same names from the same roster, and every machine's
+	// resolver file lists every holder — its own first — and no public resolver, so whichever answers
+	// first gives the one answer. Delivers what a machine's resolver configuration requires, so that
+	// requirement resolves to a holder wherever they are placed.
+	{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Replicated: true,
+		Decision: "novox/hq ADR 0194, ADR 0223"},
 	// **A machine's /etc/hosts is one module's** (novox/hq ADR 0199): its holder writes the machine's
 	// own lines and keeps every other line as the operator's, changed through these three verbs on that
 	// machine alone. The controller holds none of it.
@@ -314,11 +326,12 @@ func UseSeats(s []Seat) {
 				row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
 			}
 		}
-		// What a seat receives and what its holder needs are never stored (novox/hq ADR 0212, ADR
-		// 0220), so they are always the compiled ones.
+		// What a seat receives, what its holder needs and whether it is replicated are never stored
+		// (novox/hq ADR 0212, ADR 0220, ADR 0223), so they are always the compiled ones.
 		if d, known := byName[row.Name]; known {
 			row.Receives = d.Receives
 			row.Needs = d.Needs
+			row.Replicated = d.Replicated
 		}
 		merged = append(merged, row)
 	}
@@ -498,19 +511,24 @@ func seatNames() string {
 // two modules on one node could both provide a provision, and only the one holding the seat
 // answers for it. Nothing when no seat delivers the provision, when nobody holds
 // it, or when the holder is not among the providers offered.
+//
+// **The first holder in the providers' own order** (novox/hq ADR 0223). A replicated seat has
+// several, and the answer must not depend on the order the mesh happened to resolve its machines
+// in: the providers come sorted by machine, so every consumer is bound to the same one. A seat with
+// one holder gets the same answer as before.
 func HolderAmong(provision string, providers []Provider, held []Held) (Provider, bool) {
 	seat, delivered := SeatDelivering(provision)
 	if !delivered {
 		return Provider{}, false
 	}
-	for _, h := range held {
-		// Resolve the held claim to a seat rather than comparing names, so a record naming a seat's
-		// former name still matches it after a rename (novox/hq ADR 0122).
-		hs, ok := SeatNamed(h.Claim)
-		if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
-			continue
-		}
-		for _, p := range providers {
+	for _, p := range providers {
+		for _, h := range held {
+			// Resolve the held claim to a seat rather than comparing names, so a record naming a
+			// seat's former name still matches it after a rename (novox/hq ADR 0122).
+			hs, ok := SeatNamed(h.Claim)
+			if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
+				continue
+			}
 			if p.Node == h.Node && p.Module == h.Module {
 				return p, true
 			}

internal/catalogue/two_resolvers_test.go

@@ -0,0 +1,199 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// The mesh has two resolvers (novox/hq ADR 0223): `mesh-dns-resolver` is replicated, held on the
+// anchor and on the home server, each answering the same names; every machine's resolver file lists
+// every holder — its own first when it is one — and no public resolver. ADR 0196 listed the mesh's
+// resolver then a public one, and musl asks both at once and takes the first reply: from the home
+// server the public "no such name" for the anchor's mesh name won, every time, in every Alpine build.
+
+// resolverMachines is the anchor and the home server holding the resolver, and a laptop holding nothing.
+var resolverMachines = map[string]string{
+	"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2", "home.internal": "10.42.0.3"}
+
+// bothResolvers is the two holders on record, as `seat mesh-dns-resolver --add` leaves them.
+var bothResolvers = []Held{
+	{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "anchor", Module: "dnsmasq"},
+	{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "home", Module: "dnsmasq"},
+}
+
+// twoResolverShelf is the resolver, what asks it, and a stand-in answering `mesh-addressing`.
+func twoResolverShelf(t *testing.T) map[string]Manifest {
+	t.Helper()
+	return map[string]Manifest{
+		"net":         {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
+		"dnsmasq":     catalogueManifest(t, "dnsmasq"),
+		"resolv-conf": catalogueManifest(t, "resolv-conf"),
+	}
+}
+
+// worldWithout is the rest of the mesh as a plan for one machine sees it: every other holder's claim
+// and offer, and both holders on record.
+func worldWithout(node string) World {
+	w := World{Holdings: bothResolvers, Offered: map[string][]Provider{}}
+	for _, h := range bothResolvers {
+		if h.Node == node {
+			continue
+		}
+		w.Held = append(w.Held, h)
+		w.Offered["wildcard-resolution"] = append(w.Offered["wildcard-resolution"],
+			Provider{Node: h.Node, At: h.Node + ".internal", Module: h.Module})
+	}
+	return w
+}
+
+// resolvConfOn resolves and composes one machine and answers with the nameservers its resolver file
+// lists, in order, and the file.
+func resolvConfOn(t *testing.T, node string, assigned []string) ([]string, string) {
+	t.Helper()
+	got, err := Resolve(twoResolverShelf(t), assigned, Node{Name: node, At: node + ".internal"}, worldWithout(node))
+	if err != nil {
+		t.Fatalf("%s does not resolve with two resolvers on record: %v", node, err)
+	}
+	out, err := got.Declaration(Rendering{
+		Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
+		Holders: map[string]map[string]string{"mesh-dns-resolver": {
+			"anchor.internal": "10.42.0.1", "home.internal": "10.42.0.3"}},
+		Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
+	})
+	if err != nil {
+		t.Fatalf("%s does not compose: %v", node, err)
+	}
+	file := byID(out)["resolv-conf.fact-resolvers"]
+	if file == nil || file["path"] != "/etc/resolv.conf" {
+		t.Fatalf("%s was given no resolver file: %v", node, file)
+	}
+	content, _ := file["content"].(string)
+	var servers []string
+	for _, line := range strings.Split(content, "\n") {
+		if strings.HasPrefix(line, "nameserver ") {
+			servers = append(servers, strings.TrimPrefix(line, "nameserver "))
+		}
+	}
+	return servers, content
+}
+
+func TestTwoResolversComposeOnBothHoldersAndEachListsItselfFirst(t *testing.T) {
+	for node, want := range map[string][]string{
+		"anchor": {"10.42.0.1", "10.42.0.3"},
+		"home":   {"10.42.0.3", "10.42.0.1"},
+	} {
+		servers, content := resolvConfOn(t, node, []string{"dnsmasq", "resolv-conf"})
+		if strings.Join(servers, " ") != strings.Join(want, " ") {
+			t.Errorf("%s lists %v; itself first, then the other holder: %v\n%s", node, servers, want, content)
+		}
+	}
+}
+
+func TestAMachineHoldingNoResolverListsBothAndNoPublicOne(t *testing.T) {
+	servers, content := resolvConfOn(t, "laptop", []string{"resolv-conf"})
+	if strings.Join(servers, " ") != "10.42.0.1 10.42.0.3" {
+		t.Errorf("the laptop lists %v; every holder, by name, and nothing else:\n%s", servers, content)
+	}
+	for _, public := range []string{"1.1.1.1", "8.8.8.8", "9.9.9.9"} {
+		if strings.Contains(content, public) {
+			t.Errorf("a public resolver is listed beside the mesh's (ADR 0223):\n%s", content)
+		}
+	}
+}
+
+// A holder answers its own requirement, even though the other holder sorts first (issue 258 kept).
+func TestAHolderAnswersItsOwnRequirement(t *testing.T) {
+	got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq", "resolv-conf"},
+		Node{Name: "home", At: "home.internal"}, worldWithout("home"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, n := range got.Needs {
+		if n.Name == "wildcard-resolution" && n.From != "home" {
+			t.Errorf("the home server's resolver configuration is bound to %s; it holds the seat itself", n.From)
+		}
+	}
+}
+
+// A seat held once is still held once: a second claimant on another machine is refused while
+// nothing is on record, and a store recording two holders of it is refused, naming the seat.
+func TestASingleHolderMeshSeatStillRefusesASecondHolder(t *testing.T) {
+	store := shelf(mod("postgres", nil, nil, nil, Claim{Name: "mesh-store", Scope: ScopeMesh}))
+	other := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: "anchor", Module: "postgres"}
+	if _, err := Resolve(store, []string{"postgres"}, workstation(), World{Held: []Held{other}}); err == nil ||
+		!strings.Contains(err.Error(), "one per mesh") {
+		t.Errorf("a second claimant of a seat held once was not refused: %v", err)
+	}
+	here := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: workstation().Name, Module: "postgres"}
+	_, err := Resolve(store, []string{"postgres"}, workstation(),
+		World{Held: []Held{other}, Holdings: []Held{other, here}})
+	if err == nil || !strings.Contains(err.Error(), "on record as held by 2") {
+		t.Errorf("two holders on record for a seat held once were not refused: %v", err)
+	}
+}
+
+// Replicated is not "whoever is assigned": two claimants with nothing on record are refused, as for
+// any mesh seat, and each holder is added by an act.
+func TestTwoUnrecordedClaimantsOfTheReplicatedSeatAreRefused(t *testing.T) {
+	w := worldWithout("home")
+	w.Holdings = nil
+	_, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "home", At: "home.internal"}, w)
+	if err == nil || !strings.Contains(err.Error(), "seat mesh-dns-resolver --to") {
+		t.Errorf("a second resolver with nothing on record was not refused, naming the handover: %v", err)
+	}
+}
+
+// Only the mesh's resolver is replicated: a seat being replicated is a decision, recorded.
+func TestOnlyTheResolverIsReplicated(t *testing.T) {
+	for _, s := range Seats() {
+		if s.Replicated != (s.Name == "mesh-dns-resolver") {
+			t.Errorf("%s replicated = %v; only mesh-dns-resolver is (ADR 0223)", s.Name, s.Replicated)
+		}
+	}
+	UseSeats([]Seat{{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution"}})
+	defer UseSeats(DefaultSeats())
+	if s, _ := SeatNamed("mesh-dns-resolver"); !s.Replicated {
+		t.Error("loading the set from the store, which has no column for it, lost the resolver's replication")
+	}
+}
+
+// Every consumer is bound to the same holder whatever order the mesh resolved its machines in.
+func TestTheFirstHolderIsTheFirstProvider(t *testing.T) {
+	providers := []Provider{{Node: "anchor", Module: "dnsmasq"}, {Node: "home", Module: "dnsmasq"}}
+	for _, held := range [][]Held{bothResolvers, {bothResolvers[1], bothResolvers[0]}} {
+		if p, ok := HolderAmong("wildcard-resolution", providers, held); !ok || p.Node != "anchor" {
+			t.Errorf("held in order %v answered %v", held, p)
+		}
+	}
+}
+
+// One host record per machine, beside its wildcard (novox/hq issue 262): a name with a host record
+// says it exists and has no IPv6 address, where the wildcard alone said there is no such name, and
+// musl reads that as final.
+func TestTheResolverHasOneHostRecordPerMachine(t *testing.T) {
+	got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "anchor", At: "anchor.internal"},
+		World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := got.Declaration(Rendering{Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
+		Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	content, _ := byID(out)["dnsmasq.fact-node-zones"]["content"].(string)
+	records := map[string]int{}
+	for _, line := range strings.Split(content, "\n") {
+		if strings.HasPrefix(line, "host-record=") {
+			records[strings.TrimPrefix(line, "host-record=")]++
+		}
+	}
+	for name, at := range resolverMachines {
+		if records[name+","+at] != 1 {
+			t.Errorf("%s has %d host records at %s, and has one:\n%s", name, records[name+","+at], at, content)
+		}
+	}
+	if len(records) != len(resolverMachines) {
+		t.Errorf("%d host records for %d machines:\n%s", len(records), len(resolverMachines), content)
+	}
+}

internal/inventory/holdings_test.go

@@ -97,3 +97,57 @@ func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T)
 		t.Fatalf("a re-told membership did not replace the first: %v", got)
 	}
 }
+
+// A replicated seat has several holders on record (novox/hq ADR 0223): each is added beside the
+// others, adding one twice changes nothing, a handover still leaves exactly one, and unassigning one
+// takes only its own row.
+func TestAReplicatedSeatHasSeveralHoldersOnRecord(t *testing.T) {
+	resolver := catalogue.Manifest{Module: "resolver", Version: "1",
+		Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
+		Claims:   []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}
+	inv, ctx := aMeshWith(t, resolver)
+	if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
+		t.Fatal(err)
+	}
+	for _, n := range []string{"anchor", "home"} {
+		if _, err := inv.AddNode(ctx, n); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := inv.Assign(ctx, n, "resolver"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "anchor", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	for range 2 {
+		if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	held, err := inv.Holdings(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(held) != 2 || held[0].Node != "anchor" || held[1].Node != "home" {
+		t.Fatalf("the two holders are not both on record, once each: %+v", held)
+	}
+	if err := inv.Unassign(ctx, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "anchor" {
+		t.Fatalf("unassigning one holder took more or less than its own row: %+v", held)
+	}
+	if _, err := inv.Assign(ctx, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "home" {
+		t.Fatalf("a handover left other holders on record: %+v", held)
+	}
+}

internal/inventory/migrations/0062-a-replicated-seat-has-several-holders-on-record.sql

@@ -0,0 +1,14 @@
+-- A replicated seat has several holders on record (novox/hq ADR 0223).
+--
+-- 0039 recorded one holder per seat, keyed by the seat: a handover replaced the row, so the seat was
+-- never without a holder in between. The mesh's resolver is now held on more than one machine, each
+-- answering the same names, and each of those holders is recorded — added by `seat <name> --add`,
+-- never by being assigned. So a holding is keyed by the seat and the assignment holding it.
+--
+-- Nothing else changes. A handover (`seat <name> --to`) still leaves exactly one row, replacing every
+-- holder in one transaction; whether a seat may have more than one is the seat's compiled definition,
+-- judged by the controller before a row is added, and a store holding two for any other seat is
+-- refused at resolution, naming the seat. Every existing row is one per seat, so it satisfies the new
+-- key as it stands.
+alter table seat_holding drop constraint seat_holding_pkey;
+alter table seat_holding add primary key (seat, node, module);

internal/inventory/seats.go

@@ -215,23 +215,50 @@ func (i *Inventory) RenameSeat(ctx context.Context, from, to string) error {
 	return nil
 }
 
-// HoldSeat records that one assignment holds a seat, replacing whoever held it — as one write, so
-// the seat is never without a holder in between (novox/hq ADR 0131, design 28 task 5.3). The
-// assignment must exist; the store refuses otherwise, and that refusal is the right one: a seat
-// cannot be handed to something that is not running anywhere.
+// HoldSeat records that one assignment holds a seat, replacing whoever held it — every holder, for a
+// replicated seat (novox/hq ADR 0223) — as one transaction, so the seat is never without a holder in
+// between (novox/hq ADR 0131, design 28 task 5.3). The assignment must exist; the store refuses
+// otherwise, and that refusal is the right one: a seat cannot be handed to something that is not
+// running anywhere.
 func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module string) error {
 	node, err := i.NodeByName(ctx, nodeName)
 	if err != nil {
 		return err
 	}
-	_, err = i.store.Pool().Exec(ctx,
-		`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
-		 on conflict (seat) do update set scope = excluded.scope, node = excluded.node,
-		   module = excluded.module, since = now()`,
-		seat, scope, node.ID, module)
+	tx, err := i.store.Pool().Begin(ctx)
 	if err != nil {
+		return err
+	}
+	defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
+	if _, err := tx.Exec(ctx,
+		`delete from seat_holding where seat = $1 and not (node = $2 and module = $3)`,
+		seat, node.ID, module); err != nil {
+		return fmt.Errorf("handing %s to %s on %s: %w", seat, module, nodeName, err)
+	}
+	if _, err := tx.Exec(ctx,
+		`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
+		 on conflict (seat, node, module) do update set scope = excluded.scope, since = now()`,
+		seat, scope, node.ID, module); err != nil {
 		return fmt.Errorf("recording %s on %s as the holder of %s: %w", module, nodeName, seat, err)
 	}
+	return tx.Commit(ctx)
+}
+
+// AddSeatHolder records one more assignment holding a replicated seat, beside those already on
+// record (novox/hq ADR 0223). Whether the seat may have several holders is the caller's to judge —
+// the seat's definition is compiled, and the store holds no column for it. Recording a holder
+// already on record changes nothing.
+func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, module string) error {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return err
+	}
+	if _, err := i.store.Pool().Exec(ctx,
+		`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
+		 on conflict (seat, node, module) do nothing`,
+		seat, scope, node.ID, module); err != nil {
+		return fmt.Errorf("adding %s on %s as a holder of %s: %w", module, nodeName, seat, err)
+	}
 	return nil
 }
 
@@ -240,7 +267,7 @@ func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module
 func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
 	rows, err := i.store.Pool().Query(ctx,
 		`select h.seat, h.scope, n.name, h.module, coalesce(n.site, '')
-		   from seat_holding h join node n on n.id = h.node order by h.seat`)
+		   from seat_holding h join node n on n.id = h.node order by h.seat, n.name, h.module`)
 	if err != nil {
 		return nil, err
 	}