Let the mesh's resolver seat have several holders on record

musl takes the first reply from any listed nameserver, so a public fallback
beside the mesh's resolver answered NXDOMAIN for mesh names in every Alpine
container (hq ADR 0223). The fix is two mesh resolvers and no public one, which
needs mesh-dns-resolver held on two machines: a seat can now be replicated,
each holder recorded by 'seat <name> --add', checkClaims accepts every holder
on record and still refuses a second holder of any other mesh seat, a holder
answers its own requirement, and a roster fact gives each replicated seat's
holders, this machine first, so resolv-conf can list them. Migration 0062 keys
a holding by seat and assignment.

cmd/mesh-controller/holdings.go

@@ -6,6 +6,8 @@ import (
 	"sort"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/overlay"
 )
 
 // recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
@@ -90,3 +92,72 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
 	}
 	return said, nil
 }
+
+// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
+// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
+// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
+//
+// **The holders on record, and only the sole claimant when there are none** — the same answer the
+// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
+// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
+// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
+// listed here. A holder off the private network is left out: a resolver named at an address nothing
+// answers is a lookup that waits out its timeout on every name.
+func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
+	shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
+	var replicated []catalogue.Seat
+	for _, s := range catalogue.Seats() {
+		if s.Replicated && s.Scope == catalogue.ScopeMesh {
+			replicated = append(replicated, s)
+		}
+	}
+	if len(replicated) == 0 {
+		return nil, nil
+	}
+	recorded, err := inv.Holdings(ctx)
+	if err != nil {
+		return nil, err
+	}
+	places, err := onTheNetwork(ctx, inv, shelf)
+	if err != nil {
+		return nil, err
+	}
+	address := map[string]string{}
+	for _, p := range places {
+		address[p.Name] = p.Address
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return nil, err
+	}
+	out := map[string]map[string]string{}
+	for _, seat := range replicated {
+		var nodes []string
+		for _, h := range recorded {
+			if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
+				nodes = append(nodes, h.Node)
+			}
+		}
+		if len(nodes) == 0 {
+			var derived []string
+			for _, e := range entries {
+				for _, c := range e.Manifest.Claims {
+					if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
+						derived = append(derived, e.On...)
+					}
+				}
+			}
+			if len(derived) == 1 {
+				nodes = derived
+			}
+		}
+		at := map[string]string{}
+		for _, n := range nodes {
+			if address[n] != "" {
+				at[overlay.InternalName(n)] = address[n]
+			}
+		}
+		out[seat.Name] = at
+	}
+	return out, nil
+}

cmd/mesh-controller/main.go

@@ -194,6 +194,7 @@ func usage() {
   seats [--json]                 every seat this mesh defines, what it delivers, and who holds it
   seat rename <from> <to>        rename a seat; its former name still resolves (ADR 0122)
   seat <name> --to <node>/<module>   hand a seat to that assignment as one act; never empty in between (ADR 0131)
+  seat <name> --add <node>/<module>  add a holder beside the others, for a replicated seat (ADR 0223)
   board [--listen ADDR]          the same three questions, as a page that holds nothing
   api --issuer URL [--listen A]  assign and unassign over http, for a surface that is not here
   assign <node> <module>...      put modules on a node, judged together (ADR 0207)

cmd/mesh-controller/plan.go

@@ -707,6 +707,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
 		return catalogue.Rendering{}, inventory.Node{}, err
 	}
 
+	// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
+	// lists every holder of the mesh's resolver.
+	replicas, err := replicatedHolders(ctx, inv, shelf)
+	if err != nil {
+		return catalogue.Rendering{}, inventory.Node{}, err
+	}
+
 	// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
 	// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
 	// before it had an address on the private network. A machine joins through the tunnel now, and
@@ -783,7 +790,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
 		BusMembership: memberships[node],
 		Settings:      settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
 		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
-		Machines: machines, Zones: zones,
+		Machines: machines, Zones: zones, Holders: replicas,
 		Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
 		Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
 		Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,

cmd/mesh-controller/seats.go

@@ -29,11 +29,13 @@ type seatHolder struct {
 
 // seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
 type seatRow struct {
-	Seat     string       `json:"seat"`
-	Scope    string       `json:"scope"`
-	Delivers string       `json:"delivers,omitempty"`
-	Decision string       `json:"decision"`
-	Holders  []seatHolder `json:"holders"`
+	Seat     string `json:"seat"`
+	Scope    string `json:"scope"`
+	Delivers string `json:"delivers,omitempty"`
+	Decision string `json:"decision"`
+	// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
+	Replicated bool         `json:"replicated,omitempty"`
+	Holders    []seatHolder `json:"holders"`
 }
 
 // seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
@@ -47,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
 	rows := make([]seatRow, 0, len(seats))
 	for _, s := range seats {
 		row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
-			Holders: []seatHolder{}}
+			Replicated: s.Replicated, Holders: []seatHolder{}}
 		seen := map[seatHolder]bool{}
 		for _, h := range held {
 			// Resolve the held claim to a seat rather than comparing names, so a record naming a
@@ -104,9 +106,13 @@ func seatCommand(ctx context.Context, args []string) error {
 		return nil
 	}
 	if len(args) == 3 && args[1] == "--to" {
-		return handOver(ctx, args[0], args[2])
+		return handOver(ctx, args[0], args[2], false)
 	}
-	return fmt.Errorf("seat rename <from> <to>  |  seat <name> --to <node>/<module>")
+	if len(args) == 3 && args[1] == "--add" {
+		return handOver(ctx, args[0], args[2], true)
+	}
+	return fmt.Errorf("seat rename <from> <to>  |  seat <name> --to <node>/<module>  |  " +
+		"seat <name> --add <node>/<module>")
 }
 
 // handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
@@ -119,7 +125,12 @@ func seatCommand(ctx context.Context, args []string) error {
 // **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
 // and refusing to record a handover to a module the node has not started would make the handover
 // impossible to do before the switch instead of as the switch.
-func handOver(ctx context.Context, seatName, to string) error {
+//
+// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
+// records the named assignment as a further holder and leaves every holder on record as it is. A
+// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
+// the one named, as it always did.
+func handOver(ctx context.Context, seatName, to string, adding bool) error {
 	nodeName, module, ok := strings.Cut(to, "/")
 	if !ok || nodeName == "" || module == "" {
 		return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
@@ -135,6 +146,10 @@ func handOver(ctx context.Context, seatName, to string) error {
 	if !known {
 		return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
 	}
+	if adding && !seat.Replicated {
+		return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
+			"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
+	}
 	assigned, err := inv.Assigned(ctx, nodeName)
 	if err != nil {
 		return err
@@ -157,6 +172,7 @@ func handOver(ctx context.Context, seatName, to string) error {
 		return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
 	}
 	var was string
+	var held []string
 	holdings, err := inv.Holdings(ctx)
 	if err != nil {
 		return err
@@ -164,6 +180,7 @@ func handOver(ctx context.Context, seatName, to string) error {
 	for _, h := range holdings {
 		if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
 			was = h.Node
+			held = append(held, h.Node)
 		}
 	}
 
@@ -187,6 +204,15 @@ func handOver(ctx context.Context, seatName, to string) error {
 	} else if err := catalogue.CanHold(*m, seat); err != nil {
 		return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
 	}
+	if adding {
+		if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
+			return err
+		}
+		fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
+			strings.Join(held, ", "))
+		fmt.Printf("  `push --behind` re-declares every machine that reads the seat's holders\n")
+		return nil
+	}
 	if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
 		return err
 	}